Merge pull request #897 from nushell/modernize_external_tokens

Modernize external tokens

.cargo/config

@@ -0,0 +1,3 @@
+[build]
+
+rustflags = "--cfg coloring_in_tokens"

Cargo.lock

@@ -1498,6 +1498,7 @@ dependencies = [
  "bson 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "byte-unit 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono-humanize 0.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1540,7 +1541,7 @@ dependencies = [
  "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "roxmltree 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustyline 5.0.3 (git+https://github.com/kkawakam/rustyline.git)",
+ "rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2077,8 +2078,8 @@ dependencies = [
 
 [[package]]
 name = "rustyline"
-version = "5.0.3"
+version = "5.0.4"
-source = "git+https://github.com/kkawakam/rustyline.git#449c811998f630102bb2d9fb0b59b890d9eabac5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3056,7 +3057,7 @@ dependencies = [
 "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
 "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
 "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-"checksum rustyline 5.0.3 (git+https://github.com/kkawakam/rustyline.git)" = "<none>"
+"checksum rustyline 5.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e9d8eb9912bc492db051324d36f5cea56984fc2afeaa5c6fa84e0b0e3cde550f"
 "checksum ryu 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "19d2271fa48eaf61e53cc88b4ad9adcbafa2d512c531e7fadb6dc11a4d3656c5"
 "checksum safemem 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d2b08423011dae9a5ca23f07cf57dac3857f5c885d352b76f6d95f4aea9434d0"
 "checksum same-file 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "585e8ddcedc187886a30fa705c47985c3fa88d06624095856b36ca0b82ff4421"

Cargo.toml

@@ -75,6 +75,7 @@ serde_urlencoded = "0.6.1"
 sublime_fuzzy = "0.5"
 trash = "1.0.0"
 regex = "1"
+cfg-if = "0.1"
 
 neso = { version = "0.5.0", optional = true }
 crossterm = { version = "0.10.2", optional = true }

src/cli.rs

@@ -14,13 +14,13 @@ use crate::git::current_branch;
 use crate::parser::registry::Signature;
 use crate::parser::{
     hir,
-    hir::syntax_shape::{expand_syntax, PipelineShape},
+    hir::syntax_shape::{expand_syntax, ExpandContext, PipelineShape},
-    hir::{expand_external_tokens::expand_external_tokens, tokens_iterator::TokensIterator},
+    hir::{expand_external_tokens::ExternalTokensShape, tokens_iterator::TokensIterator},
     TokenNode,
 };
 use crate::prelude::*;
 
-use log::{debug, trace};
+use log::{debug, log_enabled, trace};
 use rustyline::error::ReadlineError;
 use rustyline::{self, config::Configurer, config::EditMode, ColorMode, Config, Editor};
 use std::env;
@@ -506,6 +506,7 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                 Some(ClassifiedCommand::External(_)) => {}
                 _ => pipeline
                     .commands
+                    .item
                     .push(ClassifiedCommand::Internal(InternalCommand {
                         name: "autoview".to_string(),
                         name_tag: Tag::unknown(),
@@ -513,13 +514,14 @@ async fn process_line(readline: Result<String, ReadlineError>, ctx: &mut Context
                             Box::new(hir::Expression::synthetic_string("autoview")),
                             None,
                             None,
-                        ),
+                        )
+                        .spanned_unknown(),
                     })),
             }
 
             let mut input = ClassifiedInputStream::new();
 
-            let mut iter = pipeline.commands.into_iter().peekable();
+            let mut iter = pipeline.commands.item.into_iter().peekable();
             let mut is_first_command = true;
 
             // Check the config to see if we need to update the path
@@ -679,11 +681,20 @@ fn classify_pipeline(
     let mut pipeline_list = vec![pipeline.clone()];
     let mut iterator = TokensIterator::all(&mut pipeline_list, pipeline.span());
 
-    expand_syntax(
+    let result = expand_syntax(
         &PipelineShape,
         &mut iterator,
-        &context.expand_context(source, pipeline.span()),
+        &context.expand_context(source),
     )
+    .map_err(|err| err.into());
+
+    if log_enabled!(target: "nu::expand_syntax", log::Level::Debug) {
+        println!("");
+        ptree::print_tree(&iterator.expand_tracer().print(source.clone())).unwrap();
+        println!("");
+    }
+
+    result
 }
 
 // Classify this command as an external command, which doesn't give special meaning
@@ -691,21 +702,22 @@ fn classify_pipeline(
 // strings.
 pub(crate) fn external_command(
     tokens: &mut TokensIterator,
-    source: &Text,
+    context: &ExpandContext,
     name: Tagged<&str>,
-) -> Result<ClassifiedCommand, ShellError> {
+) -> Result<ClassifiedCommand, ParseError> {
-    let arg_list_strings = expand_external_tokens(tokens, source)?;
+    let Spanned { item, span } = expand_syntax(&ExternalTokensShape, tokens, context)?;
 
     Ok(ClassifiedCommand::External(ExternalCommand {
         name: name.to_string(),
         name_tag: name.tag(),
-        args: arg_list_strings
+        args: item
             .iter()
             .map(|x| Tagged {
                 tag: x.span.into(),
                 item: x.item.clone(),
             })
-            .collect(),
+            .collect::<Vec<_>>()
+            .spanned(span),
     }))
 }
 

src/commands/classified.rs

@@ -4,7 +4,9 @@ use bytes::{BufMut, BytesMut};
 use derive_new::new;
 use futures::stream::StreamExt;
 use futures_codec::{Decoder, Encoder, Framed};
+use itertools::Itertools;
 use log::{log_enabled, trace};
+use std::fmt;
 use std::io::{Error, ErrorKind};
 use subprocess::Exec;
 
@@ -72,26 +74,77 @@ impl ClassifiedInputStream {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub(crate) struct ClassifiedPipeline {
-    pub(crate) commands: Vec<ClassifiedCommand>,
+    pub(crate) commands: Spanned<Vec<ClassifiedCommand>>,
 }
 
-#[derive(Debug, Eq, PartialEq)]
+impl FormatDebug for ClassifiedPipeline {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say_str(
+            "classified pipeline",
+            self.commands.iter().map(|c| c.debug(source)).join(" | "),
+        )
+    }
+}
+
+impl HasSpan for ClassifiedPipeline {
+    fn span(&self) -> Span {
+        self.commands.span
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
 pub(crate) enum ClassifiedCommand {
     #[allow(unused)]
     Expr(TokenNode),
     Internal(InternalCommand),
     #[allow(unused)]
-    Dynamic(hir::Call),
+    Dynamic(Spanned<hir::Call>),
     External(ExternalCommand),
 }
 
-#[derive(new, Debug, Eq, PartialEq)]
+impl FormatDebug for ClassifiedCommand {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            ClassifiedCommand::Expr(expr) => expr.fmt_debug(f, source),
+            ClassifiedCommand::Internal(internal) => internal.fmt_debug(f, source),
+            ClassifiedCommand::Dynamic(dynamic) => dynamic.fmt_debug(f, source),
+            ClassifiedCommand::External(external) => external.fmt_debug(f, source),
+        }
+    }
+}
+
+impl HasSpan for ClassifiedCommand {
+    fn span(&self) -> Span {
+        match self {
+            ClassifiedCommand::Expr(node) => node.span(),
+            ClassifiedCommand::Internal(command) => command.span(),
+            ClassifiedCommand::Dynamic(call) => call.span,
+            ClassifiedCommand::External(command) => command.span(),
+        }
+    }
+}
+
+#[derive(new, Debug, Clone, Eq, PartialEq)]
 pub(crate) struct InternalCommand {
     pub(crate) name: String,
     pub(crate) name_tag: Tag,
-    pub(crate) args: hir::Call,
+    pub(crate) args: Spanned<hir::Call>,
+}
+
+impl HasSpan for InternalCommand {
+    fn span(&self) -> Span {
+        let start = self.name_tag.span;
+
+        start.until(self.args.span)
+    }
+}
+
+impl FormatDebug for InternalCommand {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say("internal", self.args.debug(source))
+    }
 }
 
 #[derive(new, Debug, Eq, PartialEq)]
@@ -122,7 +175,7 @@ impl InternalCommand {
             context.run_command(
                 command,
                 self.name_tag.clone(),
-                self.args,
+                self.args.item,
                 &source,
                 objects,
                 is_first_command,
@@ -201,12 +254,31 @@ impl InternalCommand {
     }
 }
 
-#[derive(Debug, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq)]
 pub(crate) struct ExternalCommand {
     pub(crate) name: String,
 
     pub(crate) name_tag: Tag,
-    pub(crate) args: Vec<Tagged<String>>,
+    pub(crate) args: Spanned<Vec<Tagged<String>>>,
+}
+
+impl FormatDebug for ExternalCommand {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "{}", self.name)?;
+
+        if self.args.item.len() > 0 {
+            write!(f, " ")?;
+            write!(f, "{}", self.args.iter().map(|i| i.debug(source)).join(" "))?;
+        }
+
+        Ok(())
+    }
+}
+
+impl HasSpan for ExternalCommand {
+    fn span(&self) -> Span {
+        self.name_tag.span.until(self.args.span)
+    }
 }
 
 #[derive(Debug)]
@@ -230,7 +302,7 @@ impl ExternalCommand {
         trace!(target: "nu::run::external", "inputs = {:?}", inputs);
 
         let mut arg_string = format!("{}", self.name);
-        for arg in &self.args {
+        for arg in &self.args.item {
             arg_string.push_str(&arg);
         }
 
@@ -275,7 +347,7 @@ impl ExternalCommand {
             process = Exec::shell(itertools::join(commands, " && "))
         } else {
             process = Exec::cmd(&self.name);
-            for arg in &self.args {
+            for arg in &self.args.item {
                 let arg_chars: Vec<_> = arg.chars().collect();
                 if arg_chars.len() > 1
                     && arg_chars[0] == '"'

src/commands/command.rs

@@ -19,8 +19,8 @@ pub struct UnevaluatedCallInfo {
     pub name_tag: Tag,
 }
 
-impl ToDebug for UnevaluatedCallInfo {
+impl FormatDebug for UnevaluatedCallInfo {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         self.args.fmt_debug(f, source)
     }
 }
@@ -96,8 +96,14 @@ impl RawCommandArgs {
     }
 }
 
-impl ToDebug for CommandArgs {
+impl std::fmt::Debug for CommandArgs {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.call_info.fmt(f)
+    }
+}
+
+impl FormatDebug for CommandArgs {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         self.call_info.fmt_debug(f, source)
     }
 }
@@ -377,7 +383,7 @@ impl EvaluatedCommandArgs {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum CommandAction {
     ChangePath(String),
     Exit,
@@ -389,8 +395,8 @@ pub enum CommandAction {
     LeaveShell,
 }
 
-impl ToDebug for CommandAction {
+impl FormatDebug for CommandAction {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
         match self {
             CommandAction::ChangePath(s) => write!(f, "action:change-path={}", s),
             CommandAction::Exit => write!(f, "action:exit"),
@@ -408,7 +414,7 @@ impl ToDebug for CommandAction {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum ReturnSuccess {
     Value(Tagged<Value>),
     Action(CommandAction),
@@ -416,8 +422,8 @@ pub enum ReturnSuccess {
 
 pub type ReturnValue = Result<ReturnSuccess, ShellError>;
 
-impl ToDebug for ReturnValue {
+impl FormatDebug for ReturnValue {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         match self {
             Err(err) => write!(f, "{}", err.debug(source)),
             Ok(ReturnSuccess::Value(v)) => write!(f, "{:?}", v.debug()),

src/context.rs

@@ -71,9 +71,8 @@ impl Context {
     pub(crate) fn expand_context<'context>(
         &'context self,
         source: &'context Text,
-        span: Span,
     ) -> ExpandContext<'context> {
-        ExpandContext::new(&self.registry, span, source, self.shell_manager.homedir())
+        ExpandContext::new(&self.registry, source, self.shell_manager.homedir())
     }
 
     pub(crate) fn basic() -> Result<Context, Box<dyn Error>> {

src/data/meta.rs

@@ -5,6 +5,7 @@ use derive_new::new;
 use getset::Getters;
 use serde::Deserialize;
 use serde::Serialize;
+use std::fmt;
 use std::path::{Path, PathBuf};
 
 #[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
@@ -461,3 +462,134 @@ impl language_reporting::ReportingSpan for Span {
         self.end
     }
 }
+
+pub trait HasSpan: ToDebug {
+    fn span(&self) -> Span;
+}
+
+pub trait HasFallibleSpan: ToDebug {
+    fn maybe_span(&self) -> Option<Span>;
+}
+
+impl<T: HasSpan> HasFallibleSpan for T {
+    fn maybe_span(&self) -> Option<Span> {
+        Some(HasSpan::span(self))
+    }
+}
+
+impl<T> HasSpan for Spanned<T>
+where
+    Spanned<T>: ToDebug,
+{
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl HasFallibleSpan for Option<Span> {
+    fn maybe_span(&self) -> Option<Span> {
+        *self
+    }
+}
+
+impl FormatDebug for Option<Span> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            Option::None => write!(f, "no span"),
+            Option::Some(span) => FormatDebug::fmt_debug(span, f, source),
+        }
+    }
+}
+
+impl FormatDebug for Span {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "{:?}", self.slice(source))
+    }
+}
+
+impl HasSpan for Span {
+    fn span(&self) -> Span {
+        *self
+    }
+}
+
+impl<T> FormatDebug for Option<Spanned<T>>
+where
+    Spanned<T>: ToDebug,
+{
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            Option::None => write!(f, "nothing"),
+            Option::Some(spanned) => FormatDebug::fmt_debug(spanned, f, source),
+        }
+    }
+}
+
+impl<T> HasFallibleSpan for Option<Spanned<T>>
+where
+    Spanned<T>: ToDebug,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match self {
+            None => None,
+            Some(value) => Some(value.span),
+        }
+    }
+}
+
+impl<T> FormatDebug for Option<Tagged<T>>
+where
+    Tagged<T>: ToDebug,
+{
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            Option::None => write!(f, "nothing"),
+            Option::Some(item) => FormatDebug::fmt_debug(item, f, source),
+        }
+    }
+}
+
+impl<T> HasFallibleSpan for Option<Tagged<T>>
+where
+    Tagged<T>: ToDebug,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match self {
+            None => None,
+            Some(value) => Some(value.tag.span),
+        }
+    }
+}
+
+impl<T> HasSpan for Tagged<T>
+where
+    Tagged<T>: ToDebug,
+{
+    fn span(&self) -> Span {
+        self.tag.span
+    }
+}
+
+impl<T: ToDebug> FormatDebug for Vec<T> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "[ ")?;
+        write!(
+            f,
+            "{}",
+            self.iter().map(|item| item.debug(source)).join(" ")
+        )?;
+        write!(f, " ]")
+    }
+}
+
+impl FormatDebug for String {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
+        write!(f, "{}", self)
+    }
+}
+
+impl FormatDebug for Spanned<String> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
+        write!(f, "{}", self.item)
+    }
+}

src/errors.rs

@@ -30,6 +30,82 @@ impl Description {
     }
 }
 
+#[derive(Debug, Clone)]
+pub enum ParseErrorReason {
+    Eof {
+        expected: &'static str,
+    },
+    Mismatch {
+        expected: &'static str,
+        actual: Tagged<String>,
+    },
+    ArgumentError {
+        command: String,
+        error: ArgumentError,
+        tag: Tag,
+    },
+}
+
+#[derive(Debug, Clone)]
+pub struct ParseError {
+    reason: ParseErrorReason,
+    tag: Tag,
+}
+
+impl ParseError {
+    pub fn unexpected_eof(expected: &'static str, span: Span) -> ParseError {
+        ParseError {
+            reason: ParseErrorReason::Eof { expected },
+            tag: span.into(),
+        }
+    }
+
+    pub fn mismatch(expected: &'static str, actual: Tagged<impl Into<String>>) -> ParseError {
+        let Tagged { tag, item } = actual;
+
+        ParseError {
+            reason: ParseErrorReason::Mismatch {
+                expected,
+                actual: item.into().tagged(tag.clone()),
+            },
+            tag,
+        }
+    }
+
+    pub fn argument_error(
+        command: impl Into<String>,
+        kind: ArgumentError,
+        tag: impl Into<Tag>,
+    ) -> ParseError {
+        let tag = tag.into();
+
+        ParseError {
+            reason: ParseErrorReason::ArgumentError {
+                command: command.into(),
+                error: kind,
+                tag: tag.clone(),
+            },
+            tag: tag.clone(),
+        }
+    }
+}
+
+impl From<ParseError> for ShellError {
+    fn from(error: ParseError) -> ShellError {
+        match error.reason {
+            ParseErrorReason::Eof { expected } => ShellError::unexpected_eof(expected, error.tag),
+            ParseErrorReason::Mismatch { actual, expected } => {
+                ShellError::type_error(expected, actual.clone())
+            }
+            ParseErrorReason::ArgumentError {
+                command,
+                error,
+                tag,
+            } => ShellError::argument_error(command, error, tag),
+        }
+    }
+}
+
 #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)]
 pub enum ArgumentError {
     MissingMandatoryFlag(String),
@@ -51,8 +127,8 @@ impl ShellError {
     }
 }
 
-impl ToDebug for ShellError {
+impl FormatDebug for ShellError {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         self.error.fmt_debug(f, source)
     }
 }
@@ -153,16 +229,6 @@ impl ShellError {
         .start()
     }
 
-    pub(crate) fn invalid_external_word(tag: impl Into<Tag>) -> ShellError {
-        ProximateShellError::ArgumentError {
-            command: "Invalid argument to Nu command (did you mean to call an external command?)"
-                .into(),
-            error: ArgumentError::InvalidExternalWord,
-            tag: tag.into(),
-        }
-        .start()
-    }
-
     pub(crate) fn parse_error(
         error: nom::Err<(
             nom_locate::LocatedSpanEx<&str, TracableContext>,
@@ -490,8 +556,8 @@ impl ProximateShellError {
     }
 }
 
-impl ToDebug for ProximateShellError {
+impl FormatDebug for ProximateShellError {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
         // TODO: Custom debug for inner spans
         write!(f, "{:?}", self)
     }

src/lib.rs

@@ -30,12 +30,16 @@ pub use crate::env::host::BasicHost;
 pub use crate::parser::hir::SyntaxShape;
 pub use crate::parser::parse::token_tree_builder::TokenTreeBuilder;
 pub use crate::plugin::{serve_plugin, Plugin};
+pub use crate::traits::{DebugFormatter, FormatDebug, ToDebug};
 pub use crate::utils::{did_you_mean, AbsoluteFile, AbsolutePath, RelativePath};
 pub use cli::cli;
 pub use data::base::{Primitive, Value};
 pub use data::config::{config_path, APP_INFO};
 pub use data::dict::{Dictionary, TaggedDictBuilder};
-pub use data::meta::{tag_for_tagged_list, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem};
+pub use data::meta::{
+    tag_for_tagged_list, HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Tagged,
+    TaggedItem,
+};
 pub use errors::{CoerceInto, ShellError};
 pub use num_traits::cast::ToPrimitive;
 pub use parser::parse::text::Text;

src/main.rs

@@ -19,6 +19,12 @@ fn main() -> Result<(), Box<dyn Error>> {
                 .multiple(true)
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name("debug")
+                .long("debug")
+                .multiple(true)
+                .takes_value(true),
+        )
         .get_matches();
 
     let loglevel = match matches.value_of("loglevel") {
@@ -48,6 +54,15 @@ fn main() -> Result<(), Box<dyn Error>> {
         }
     }
 
+    match matches.values_of("debug") {
+        None => {}
+        Some(values) => {
+            for item in values {
+                builder.filter_module(&format!("nu::{}", item), LevelFilter::Debug);
+            }
+        }
+    }
+
     builder.try_init()?;
 
     futures::executor::block_on(nu::cli())?;

src/parser/hir.rs

@@ -24,7 +24,6 @@ pub(crate) use self::external_command::ExternalCommand;
 pub(crate) use self::named::NamedArguments;
 pub(crate) use self::path::Path;
 pub(crate) use self::syntax_shape::ExpandContext;
-pub(crate) use self::tokens_iterator::debug::debug_tokens;
 pub(crate) use self::tokens_iterator::TokensIterator;
 
 pub use self::syntax_shape::SyntaxShape;
@@ -50,8 +49,8 @@ impl Call {
     }
 }
 
-impl ToDebug for Call {
+impl FormatDebug for Call {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "({}", self.head.debug(source))?;
 
         if let Some(positional) = &self.positional {
@@ -242,10 +241,14 @@ impl Expression {
     pub(crate) fn it_variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
         RawExpression::Variable(Variable::It(inner.into())).spanned(outer)
     }
+
+    pub(crate) fn tagged_type_name(&self) -> Tagged<&'static str> {
+        self.item.type_name().tagged(self.span)
+    }
 }
 
-impl ToDebug for Expression {
+impl FormatDebug for Spanned<RawExpression> {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         match &self.item {
             RawExpression::Literal(l) => l.spanned(self.span).fmt_debug(f, source),
             RawExpression::FilePath(p) => write!(f, "{}", p.display()),
@@ -256,7 +259,7 @@ impl ToDebug for Expression {
             RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)),
             RawExpression::Binary(b) => write!(f, "{}", b.debug(source)),
             RawExpression::ExternalCommand(c) => write!(f, "^{}", c.name().slice(source)),
-            RawExpression::Block(exprs) => {
+            RawExpression::Block(exprs) => f.say_block("block", |f| {
                 write!(f, "{{ ")?;
 
                 for expr in exprs {
@@ -264,8 +267,8 @@ impl ToDebug for Expression {
                 }
 
                 write!(f, "}}")
-            }
+            }),
-            RawExpression::List(exprs) => {
+            RawExpression::List(exprs) => f.say_block("list", |f| {
                 write!(f, "[ ")?;
 
                 for expr in exprs {
@@ -273,7 +276,7 @@ impl ToDebug for Expression {
                 }
 
                 write!(f, "]")
-            }
+            }),
             RawExpression::Path(p) => write!(f, "{}", p.debug(source)),
             RawExpression::Boolean(true) => write!(f, "$yes"),
             RawExpression::Boolean(false) => write!(f, "$no"),
@@ -321,14 +324,14 @@ impl std::fmt::Display for Tagged<&Literal> {
     }
 }
 
-impl ToDebug for Spanned<&Literal> {
+impl FormatDebug for Spanned<&Literal> {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         match self.item {
-            Literal::Number(number) => write!(f, "{:?}", number),
+            Literal::Number(..) => f.say_str("number", self.span.slice(source)),
-            Literal::Size(number, unit) => write!(f, "{:?}{:?}", *number, unit),
+            Literal::Size(..) => f.say_str("size", self.span.slice(source)),
-            Literal::String(tag) => write!(f, "{}", tag.slice(source)),
+            Literal::String(..) => f.say_str("string", self.span.slice(source)),
-            Literal::GlobPattern(_) => write!(f, "{}", self.span.slice(source)),
+            Literal::GlobPattern(..) => f.say_str("glob", self.span.slice(source)),
-            Literal::Bare => write!(f, "{}", self.span.slice(source)),
+            Literal::Bare => f.say_str("word", self.span.slice(source)),
         }
     }
 }
@@ -359,3 +362,9 @@ impl std::fmt::Display for Variable {
         }
     }
 }
+
+impl FormatDebug for Spanned<Variable> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "{}", self.span.slice(source))
+    }
+}

src/parser/hir/baseline_parse/tests.rs

@@ -6,7 +6,7 @@ use crate::parser::hir::syntax_shape::*;
 use crate::parser::hir::TokensIterator;
 use crate::parser::parse::token_tree_builder::{CurriedToken, TokenTreeBuilder as b};
 use crate::parser::TokenNode;
-use crate::{Span, SpannedItem, Tag, Text};
+use crate::{HasSpan, Span, SpannedItem, Tag, Text};
 use pretty_assertions::assert_eq;
 use std::fmt::Debug;
 
@@ -63,7 +63,9 @@ fn test_parse_command() {
         vec![b::bare("ls"), b::sp(), b::pattern("*.txt")],
         |tokens| {
             let bare = tokens[0].expect_bare();
-            let pattern = tokens[2].expect_pattern();
+            let pat = tokens[2].expect_pattern();
+
+            eprintln!("{:?} {:?} {:?}", bare, pat, bare.until(pat));
 
             ClassifiedCommand::Internal(InternalCommand::new(
                 "ls".to_string(),
@@ -73,9 +75,10 @@ fn test_parse_command() {
                 },
                 hir::Call {
                     head: Box::new(hir::RawExpression::Command(bare).spanned(bare)),
-                    positional: Some(vec![hir::Expression::pattern("*.txt", pattern)]),
+                    positional: Some(vec![hir::Expression::pattern("*.txt", pat)]),
                     named: None,
-                },
+                }
+                .spanned(bare.until(pat)),
             ))
             // hir::Expression::path(
             //     hir::Expression::variable(inner_var, outer_var),
@@ -86,7 +89,7 @@ fn test_parse_command() {
     );
 }
 
-fn parse_tokens<T: Eq + Debug>(
+fn parse_tokens<T: Eq + HasSpan + Clone + Debug + 'static>(
     shape: impl ExpandSyntax<Output = T>,
     tokens: Vec<CurriedToken>,
     expected: impl FnOnce(&[TokenNode]) -> T,
@@ -96,19 +99,19 @@ fn parse_tokens<T: Eq + Debug>(
 
     ExpandContext::with_empty(&Text::from(source), |context| {
         let tokens = tokens.expect_list();
-        let mut iterator = TokensIterator::all(tokens, *context.span());
+        let mut iterator = TokensIterator::all(tokens.item, tokens.span);
 
         let expr = expand_syntax(&shape, &mut iterator, &context);
 
         let expr = match expr {
             Ok(expr) => expr,
             Err(err) => {
-                crate::cli::print_err(err, &BasicHost, context.source().clone());
+                crate::cli::print_err(err.into(), &BasicHost, context.source().clone());
                 panic!("Parse failed");
             }
         };
 
-        assert_eq!(expr, expected(tokens));
+        assert_eq!(expr, expected(tokens.item));
     })
 }
 

src/parser/hir/binary.rs

@@ -22,8 +22,8 @@ impl fmt::Display for Binary {
     }
 }
 
-impl ToDebug for Binary {
+impl FormatDebug for Binary {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "{}", self.left.debug(source))?;
         write!(f, " {} ", self.op.debug(source))?;
         write!(f, "{}", self.right.debug(source))?;

src/parser/hir/expand_external_tokens.rs

@@ -1,35 +1,55 @@
-use crate::errors::ShellError;
+use crate::errors::ParseError;
 #[cfg(not(coloring_in_tokens))]
 use crate::parser::hir::syntax_shape::FlatShape;
 use crate::parser::{
     hir::syntax_shape::{
-        color_syntax, expand_atom, AtomicToken, ColorSyntax, ExpandContext, ExpansionRule,
+        color_syntax, expand_atom, expand_expr, expand_syntax, AtomicToken, ColorSyntax,
-        MaybeSpaceShape,
+        ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, MaybeSpaceShape,
     },
-    TokenNode, TokensIterator,
+    hir::Expression,
+    TokensIterator,
 };
-use crate::{Span, Spanned, Text};
+use crate::{DebugFormatter, FormatDebug, Span, Spanned, SpannedItem};
-
+use std::fmt;
-pub fn expand_external_tokens(
-    token_nodes: &mut TokensIterator<'_>,
-    source: &Text,
-) -> Result<Vec<Spanned<String>>, ShellError> {
-    let mut out: Vec<Spanned<String>> = vec![];
-
-    loop {
-        if let Some(span) = expand_next_expression(token_nodes)? {
-            out.push(span.spanned_string(source));
-        } else {
-            break;
-        }
-    }
-
-    Ok(out)
-}
 
 #[derive(Debug, Copy, Clone)]
 pub struct ExternalTokensShape;
 
+impl FormatDebug for Spanned<Vec<Spanned<String>>> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        FormatDebug::fmt_debug(&self.item, f, source)
+    }
+}
+
+impl ExpandSyntax for ExternalTokensShape {
+    type Output = Spanned<Vec<Spanned<String>>>;
+
+    fn name(&self) -> &'static str {
+        "external command"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let mut out: Vec<Spanned<String>> = vec![];
+
+        let start = token_nodes.span_at_cursor();
+
+        loop {
+            match expand_syntax(&ExternalExpressionShape, token_nodes, context) {
+                Err(_) | Ok(None) => break,
+                Ok(Some(span)) => out.push(span.spanned_string(context.source())),
+            }
+        }
+
+        let end = token_nodes.span_at_cursor();
+
+        Ok(out.spanned(start.until(end)))
+    }
+}
+
 #[cfg(not(coloring_in_tokens))]
 impl ColorSyntax for ExternalTokensShape {
     type Info = ();
@@ -85,109 +105,200 @@ impl ColorSyntax for ExternalTokensShape {
     }
 }
 
-pub fn expand_next_expression(
+#[derive(Debug, Copy, Clone)]
-    token_nodes: &mut TokensIterator<'_>,
+pub struct ExternalExpressionShape;
-) -> Result<Option<Span>, ShellError> {
-    let first = token_nodes.next_non_ws();
 
-    let first = match first {
+impl ExpandSyntax for ExternalExpressionShape {
-        None => return Ok(None),
+    type Output = Option<Span>;
-        Some(v) => v,
-    };
 
-    let first = triage_external_head(first)?;
+    fn name(&self) -> &'static str {
-    let mut last = first;
+        "external expression"
-
-    loop {
-        let continuation = triage_continuation(token_nodes)?;
-
-        if let Some(continuation) = continuation {
-            last = continuation;
-        } else {
-            break;
-        }
     }
 
-    Ok(Some(first.until(last)))
+    fn expand_syntax<'a, 'b>(
-}
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        expand_syntax(&MaybeSpaceShape, token_nodes, context)?;
 
-fn triage_external_head(node: &TokenNode) -> Result<Span, ShellError> {
+        let first = expand_atom(
-    Ok(match node {
+            token_nodes,
-        TokenNode::Token(token) => token.span,
+            "external command",
-        TokenNode::Call(_call) => unimplemented!("TODO: OMG"),
+            context,
-        TokenNode::Nodes(_nodes) => unimplemented!("TODO: OMG"),
+            ExpansionRule::new().allow_external_command(),
-        TokenNode::Delimited(_delimited) => unimplemented!("TODO: OMG"),
+        )?
-        TokenNode::Pipeline(_pipeline) => unimplemented!("TODO: OMG"),
+        .span;
-        TokenNode::Flag(flag) => flag.span,
+
-        TokenNode::Whitespace(_whitespace) => {
+        let mut last = first;
-            unreachable!("This function should be called after next_non_ws()")
+
+        loop {
+            let continuation = expand_expr(&ExternalContinuationShape, token_nodes, context);
+
+            if let Ok(continuation) = continuation {
+                last = continuation.span;
+            } else {
+                break;
+            }
         }
-        TokenNode::Error(_error) => unimplemented!("TODO: OMG"),
-    })
-}
 
-fn triage_continuation<'a, 'b>(
+        Ok(Some(first.until(last)))
-    nodes: &'a mut TokensIterator<'b>,
-) -> Result<Option<Span>, ShellError> {
-    let mut peeked = nodes.peek_any();
-
-    let node = match peeked.node {
-        None => return Ok(None),
-        Some(node) => node,
-    };
-
-    match &node {
-        node if node.is_whitespace() => return Ok(None),
-        TokenNode::Token(..) | TokenNode::Flag(..) => {}
-        TokenNode::Call(..) => unimplemented!("call"),
-        TokenNode::Nodes(..) => unimplemented!("nodes"),
-        TokenNode::Delimited(..) => unimplemented!("delimited"),
-        TokenNode::Pipeline(..) => unimplemented!("pipeline"),
-        TokenNode::Whitespace(..) => unimplemented!("whitespace"),
-        TokenNode::Error(..) => unimplemented!("error"),
     }
-
-    peeked.commit();
-    Ok(Some(node.span()))
-}
-
-#[must_use]
-enum ExternalExpressionResult {
-    Eof,
-    Processed,
 }
 
 #[derive(Debug, Copy, Clone)]
 struct ExternalExpression;
 
-#[cfg(not(coloring_in_tokens))]
+impl ExpandSyntax for ExternalExpression {
-impl ColorSyntax for ExternalExpression {
+    type Output = Option<Span>;
-    type Info = ExternalExpressionResult;
-    type Input = ();
 
-    fn color_syntax<'a, 'b>(
+    fn name(&self) -> &'static str {
+        "external expression"
+    }
+
+    fn expand_syntax<'a, 'b>(
         &self,
-        _input: &(),
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-        shapes: &mut Vec<Spanned<FlatShape>>,
+    ) -> Result<Self::Output, ParseError> {
-    ) -> ExternalExpressionResult {
+        expand_syntax(&MaybeSpaceShape, token_nodes, context)?;
-        let atom = match expand_atom(
-            token_nodes,
-            "external word",
-            context,
-            ExpansionRule::permissive(),
-        ) {
-            Err(_) => unreachable!("TODO: separate infallible expand_atom"),
-            Ok(Spanned {
-                item: AtomicToken::Eof { .. },
-                ..
-            }) => return ExternalExpressionResult::Eof,
-            Ok(atom) => atom,
-        };
 
-        atom.color_tokens(shapes);
+        let first = expand_syntax(&ExternalHeadShape, token_nodes, context)?.span;
-        return ExternalExpressionResult::Processed;
+        let mut last = first;
+
+        loop {
+            let continuation = expand_syntax(&ExternalContinuationShape, token_nodes, context);
+
+            if let Ok(continuation) = continuation {
+                last = continuation.span;
+            } else {
+                break;
+            }
+        }
+
+        Ok(Some(first.until(last)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct ExternalHeadShape;
+
+impl ExpandExpression for ExternalHeadShape {
+    fn name(&self) -> &'static str {
+        "external argument"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Expression, ParseError> {
+        match expand_atom(
+            token_nodes,
+            "external argument",
+            context,
+            ExpansionRule::new()
+                .allow_external_word()
+                .treat_size_as_word(),
+        )? {
+            atom => match &atom {
+                Spanned { item, span } => Ok(match item {
+                    AtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"),
+                    AtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"),
+                    AtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"),
+                    AtomicToken::Whitespace { .. } => {
+                        unreachable!("ExpansionRule doesn't allow Whitespace")
+                    }
+                    AtomicToken::ShorthandFlag { .. }
+                    | AtomicToken::LonghandFlag { .. }
+                    | AtomicToken::SquareDelimited { .. }
+                    | AtomicToken::ParenDelimited { .. }
+                    | AtomicToken::BraceDelimited { .. }
+                    | AtomicToken::Pipeline { .. } => {
+                        return Err(ParseError::mismatch(
+                            "external command name",
+                            atom.tagged_type_name(),
+                        ))
+                    }
+                    AtomicToken::ExternalCommand { command } => {
+                        Expression::external_command(*command, *span)
+                    }
+                    AtomicToken::Number { number } => {
+                        Expression::number(number.to_number(context.source()), *span)
+                    }
+                    AtomicToken::String { body } => Expression::string(*body, *span),
+                    AtomicToken::ItVariable { name } => Expression::it_variable(*name, *span),
+                    AtomicToken::Variable { name } => Expression::variable(*name, *span),
+                    AtomicToken::ExternalWord { .. }
+                    | AtomicToken::GlobPattern { .. }
+                    | AtomicToken::FilePath { .. }
+                    | AtomicToken::Word { .. }
+                    | AtomicToken::Dot { .. }
+                    | AtomicToken::Operator { .. } => Expression::external_command(*span, *span),
+                }),
+            },
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+struct ExternalContinuationShape;
+
+impl ExpandExpression for ExternalContinuationShape {
+    fn name(&self) -> &'static str {
+        "external argument"
+    }
+
+    fn expand_expr<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+    ) -> Result<Expression, ParseError> {
+        match expand_atom(
+            token_nodes,
+            "external argument",
+            context,
+            ExpansionRule::new()
+                .allow_external_word()
+                .treat_size_as_word(),
+        )? {
+            atom => match &atom {
+                Spanned { item, span } => Ok(match item {
+                    AtomicToken::Eof { .. } => unreachable!("ExpansionRule doesn't allow EOF"),
+                    AtomicToken::Error { .. } => unreachable!("ExpansionRule doesn't allow Error"),
+                    AtomicToken::Number { number } => {
+                        Expression::number(number.to_number(context.source()), *span)
+                    }
+                    AtomicToken::Size { .. } => unreachable!("ExpansionRule treats size as word"),
+                    AtomicToken::ExternalCommand { .. } => {
+                        unreachable!("ExpansionRule doesn't allow ExternalCommand")
+                    }
+                    AtomicToken::Whitespace { .. } => {
+                        unreachable!("ExpansionRule doesn't allow Whitespace")
+                    }
+                    AtomicToken::String { body } => Expression::string(*body, *span),
+                    AtomicToken::ItVariable { name } => Expression::it_variable(*name, *span),
+                    AtomicToken::Variable { name } => Expression::variable(*name, *span),
+                    AtomicToken::ExternalWord { .. }
+                    | AtomicToken::GlobPattern { .. }
+                    | AtomicToken::FilePath { .. }
+                    | AtomicToken::Word { .. }
+                    | AtomicToken::ShorthandFlag { .. }
+                    | AtomicToken::LonghandFlag { .. }
+                    | AtomicToken::Dot { .. }
+                    | AtomicToken::Operator { .. } => Expression::bare(*span),
+                    AtomicToken::SquareDelimited { .. }
+                    | AtomicToken::ParenDelimited { .. }
+                    | AtomicToken::BraceDelimited { .. }
+                    | AtomicToken::Pipeline { .. } => {
+                        return Err(ParseError::mismatch(
+                            "external argument",
+                            atom.tagged_type_name(),
+                        ))
+                    }
+                }),
+            },
+        }
     }
 }
 
@@ -224,3 +335,40 @@ impl ColorSyntax for ExternalExpression {
         return ExternalExpressionResult::Processed;
     }
 }
+
+#[must_use]
+enum ExternalExpressionResult {
+    Eof,
+    Processed,
+}
+
+#[cfg(not(coloring_in_tokens))]
+impl ColorSyntax for ExternalExpression {
+    type Info = ExternalExpressionResult;
+    type Input = ();
+
+    fn color_syntax<'a, 'b>(
+        &self,
+        _input: &(),
+        token_nodes: &'b mut TokensIterator<'a>,
+        context: &ExpandContext,
+        shapes: &mut Vec<Spanned<FlatShape>>,
+    ) -> ExternalExpressionResult {
+        let atom = match expand_atom(
+            token_nodes,
+            "external word",
+            context,
+            ExpansionRule::permissive(),
+        ) {
+            Err(_) => unreachable!("TODO: separate infallible expand_atom"),
+            Ok(Spanned {
+                item: AtomicToken::Eof { .. },
+                ..
+            }) => return ExternalExpressionResult::Eof,
+            Ok(atom) => atom,
+        };
+
+        atom.color_tokens(shapes);
+        return ExternalExpressionResult::Processed;
+    }
+}

src/parser/hir/external_command.rs

@@ -12,8 +12,8 @@ pub struct ExternalCommand {
     pub(crate) name: Span,
 }
 
-impl ToDebug for ExternalCommand {
+impl FormatDebug for ExternalCommand {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "{}", self.name.slice(source))?;
 
         Ok(())

src/parser/hir/named.rs

@@ -21,8 +21,8 @@ pub struct NamedArguments {
     pub(crate) named: IndexMap<String, NamedValue>,
 }
 
-impl ToDebug for NamedArguments {
+impl FormatDebug for NamedArguments {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         for (name, value) in &self.named {
             match value {
                 NamedValue::AbsentSwitch => continue,

src/parser/hir/path.rs

@@ -44,8 +44,8 @@ impl Path {
     }
 }
 
-impl ToDebug for Path {
+impl FormatDebug for Path {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "{}", self.head.debug(source))?;
 
         for part in &self.tail {

src/parser/hir/syntax_shape.rs

@@ -11,16 +11,12 @@ use crate::parser::hir::expand_external_tokens::ExternalTokensShape;
 use crate::parser::hir::syntax_shape::block::AnyBlockShape;
 use crate::parser::hir::tokens_iterator::Peeked;
 use crate::parser::parse_command::{parse_command_tail, CommandTailShape};
-use crate::parser::{
+use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode};
-    hir,
-    hir::{debug_tokens, TokensIterator},
-    Operator, RawToken, TokenNode,
-};
 use crate::prelude::*;
 use derive_new::new;
 use getset::Getters;
-use log::{self, trace};
 use serde::{Deserialize, Serialize};
+use std::fmt;
 use std::path::{Path, PathBuf};
 
 pub(crate) use self::expression::atom::{expand_atom, AtomicToken, ExpansionRule};
@@ -40,15 +36,16 @@ pub(crate) use self::expression::variable_path::{
 pub(crate) use self::expression::{continue_expression, AnyExpressionShape};
 pub(crate) use self::flat_shape::FlatShape;
 
+#[cfg(not(coloring_in_tokens))]
+use crate::parser::hir::tokens_iterator::debug::debug_tokens;
 #[cfg(not(coloring_in_tokens))]
 use crate::parser::parse::pipeline::Pipeline;
 #[cfg(not(coloring_in_tokens))]
-use log::log_enabled;
+use log::{log_enabled, trace};
 
 #[derive(Debug, Copy, Clone, Serialize, Deserialize)]
 pub enum SyntaxShape {
     Any,
-    List,
     String,
     Member,
     ColumnPath,
@@ -75,10 +72,6 @@ impl FallibleColorSyntax for SyntaxShape {
             SyntaxShape::Any => {
                 color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes)
             }
-            SyntaxShape::List => {
-                color_syntax(&ExpressionListShape, token_nodes, context, shapes);
-                Ok(())
-            }
             SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context, shapes),
             SyntaxShape::String => color_fallible_syntax_with(
                 &StringShape,
@@ -126,10 +119,6 @@ impl FallibleColorSyntax for SyntaxShape {
     ) -> Result<(), ShellError> {
         match self {
             SyntaxShape::Any => color_fallible_syntax(&AnyExpressionShape, token_nodes, context),
-            SyntaxShape::List => {
-                color_syntax(&ExpressionListShape, token_nodes, context);
-                Ok(())
-            }
             SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context),
             SyntaxShape::String => {
                 color_fallible_syntax_with(&StringShape, &FlatShape::String, token_nodes, context)
@@ -147,14 +136,27 @@ impl FallibleColorSyntax for SyntaxShape {
 }
 
 impl ExpandExpression for SyntaxShape {
+    fn name(&self) -> &'static str {
+        match self {
+            SyntaxShape::Any => "any",
+            SyntaxShape::Int => "integer",
+            SyntaxShape::String => "string",
+            SyntaxShape::Member => "column name",
+            SyntaxShape::ColumnPath => "column path",
+            SyntaxShape::Number => "number",
+            SyntaxShape::Path => "file path",
+            SyntaxShape::Pattern => "glob pattern",
+            SyntaxShape::Block => "block",
+        }
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         match self {
             SyntaxShape::Any => expand_expr(&AnyExpressionShape, token_nodes, context),
-            SyntaxShape::List => Err(ShellError::unimplemented("SyntaxShape:List")),
             SyntaxShape::Int => expand_expr(&IntShape, token_nodes, context),
             SyntaxShape::String => expand_expr(&StringShape, token_nodes, context),
             SyntaxShape::Member => {
@@ -162,8 +164,9 @@ impl ExpandExpression for SyntaxShape {
                 Ok(syntax.to_expr())
             }
             SyntaxShape::ColumnPath => {
-                let Tagged { item: members, tag } =
+                let column_path = expand_syntax(&ColumnPathShape, token_nodes, context)?;
-                    expand_syntax(&ColumnPathShape, token_nodes, context)?;
+
+                let Tagged { item: members, tag } = column_path.path();
 
                 Ok(hir::Expression::list(
                     members.into_iter().map(|s| s.to_expr()).collect(),
@@ -182,7 +185,6 @@ impl std::fmt::Display for SyntaxShape {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         match self {
             SyntaxShape::Any => write!(f, "Any"),
-            SyntaxShape::List => write!(f, "List"),
             SyntaxShape::String => write!(f, "String"),
             SyntaxShape::Int => write!(f, "Integer"),
             SyntaxShape::Member => write!(f, "Member"),
@@ -200,8 +202,6 @@ pub struct ExpandContext<'context> {
     #[get = "pub(crate)"]
     registry: &'context CommandRegistry,
     #[get = "pub(crate)"]
-    span: Span,
-    #[get = "pub(crate)"]
     source: &'context Text,
     homedir: Option<PathBuf>,
 }
@@ -221,7 +221,6 @@ impl<'context> ExpandContext<'context> {
 
         callback(ExpandContext {
             registry: &registry,
-            span: Span::unknown(),
             source,
             homedir: None,
         })
@@ -237,11 +236,13 @@ pub trait TestSyntax: std::fmt::Debug + Copy {
 }
 
 pub trait ExpandExpression: std::fmt::Debug + Copy {
+    fn name(&self) -> &'static str;
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError>;
+    ) -> Result<hir::Expression, ParseError>;
 }
 
 #[cfg(coloring_in_tokens)]
@@ -303,35 +304,49 @@ pub trait ColorSyntax: std::fmt::Debug + Copy {
 }
 
 pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy {
-    type Output: std::fmt::Debug;
+    type Output: HasFallibleSpan + Clone + std::fmt::Debug + 'static;
+
+    fn name(&self) -> &'static str;
 
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError>;
+    ) -> Result<Self::Output, ParseError>;
 }
 
 pub(crate) fn expand_syntax<'a, 'b, T: ExpandSyntax>(
     shape: &T,
     token_nodes: &'b mut TokensIterator<'a>,
     context: &ExpandContext,
-) -> Result<T::Output, ShellError> {
+) -> Result<T::Output, ParseError> {
-    trace!(target: "nu::expand_syntax", "before {} :: {:?}", std::any::type_name::<T>(), debug_tokens(token_nodes.state(), context.source));
+    token_nodes.expand_frame(shape.name(), |token_nodes| {
+        shape.expand_syntax(token_nodes, context)
+    })
+}
 
-    let result = shape.expand_syntax(token_nodes, context);
+pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>(
+    shape: &T,
+    token_nodes: &'b mut TokensIterator<'a>,
+    context: &ExpandContext,
+) -> Result<hir::Expression, ParseError> {
+    token_nodes.expand_expr_frame(shape.name(), |token_nodes| {
+        shape.expand_expr(token_nodes, context)
+    })
+}
 
-    match result {
+#[cfg(coloring_in_tokens)]
-        Err(err) => {
+pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
-            trace!(target: "nu::expand_syntax", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source));
+    shape: &T,
-            Err(err)
+    token_nodes: &'b mut TokensIterator<'a>,
-        }
+    context: &ExpandContext,
-
+) -> ((), U) {
-        Ok(result) => {
+    (
-            trace!(target: "nu::expand_syntax", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source));
+        (),
-            Ok(result)
+        token_nodes.color_frame(shape.name(), |token_nodes| {
-        }
+            shape.color_syntax(&(), token_nodes, context)
-    }
+        }),
+    )
 }
 
 #[cfg(not(coloring_in_tokens))]
@@ -363,20 +378,6 @@ pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
     ((), result)
 }
 
-#[cfg(coloring_in_tokens)]
-pub fn color_syntax<'a, 'b, T: ColorSyntax<Info = U, Input = ()>, U>(
-    shape: &T,
-    token_nodes: &'b mut TokensIterator<'a>,
-    context: &ExpandContext,
-) -> ((), U) {
-    (
-        (),
-        token_nodes.color_frame(shape.name(), |token_nodes| {
-            shape.color_syntax(&(), token_nodes, context)
-        }),
-    )
-}
-
 #[cfg(not(coloring_in_tokens))]
 pub fn color_fallible_syntax<'a, 'b, T: FallibleColorSyntax<Info = U, Input = ()>, U>(
     shape: &T,
@@ -492,36 +493,18 @@ pub fn color_fallible_syntax_with<'a, 'b, T: FallibleColorSyntax<Info = U, Input
     })
 }
 
-pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>(
-    shape: &T,
-    token_nodes: &'b mut TokensIterator<'a>,
-    context: &ExpandContext,
-) -> Result<hir::Expression, ShellError> {
-    trace!(target: "nu::expand_expression", "before {} :: {:?}", std::any::type_name::<T>(), debug_tokens(token_nodes.state(), context.source));
-
-    let result = shape.expand_expr(token_nodes, context);
-
-    match result {
-        Err(err) => {
-            trace!(target: "nu::expand_expression", "error :: {} :: {:?}", err, debug_tokens(token_nodes.state(), context.source));
-            Err(err)
-        }
-
-        Ok(result) => {
-            trace!(target: "nu::expand_expression", "ok :: {:?} :: {:?}", result, debug_tokens(token_nodes.state(), context.source));
-            Ok(result)
-        }
-    }
-}
-
 impl<T: ExpandExpression> ExpandSyntax for T {
     type Output = hir::Expression;
 
+    fn name(&self) -> &'static str {
+        ExpandExpression::name(self)
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
         ExpandExpression::expand_expr(self, token_nodes, context)
     }
 }
@@ -537,7 +520,7 @@ pub trait SkipSyntax: std::fmt::Debug + Copy {
 enum BarePathState {
     Initial,
     Seen(Span, Span),
-    Error(ShellError),
+    Error(ParseError),
 }
 
 impl BarePathState {
@@ -549,7 +532,7 @@ impl BarePathState {
         }
     }
 
-    pub fn end(self, peeked: Peeked, reason: impl Into<String>) -> BarePathState {
+    pub fn end(self, peeked: Peeked, reason: &'static str) -> BarePathState {
         match self {
             BarePathState::Initial => BarePathState::Error(peeked.type_error(reason)),
             BarePathState::Seen(start, end) => BarePathState::Seen(start, end),
@@ -557,7 +540,7 @@ impl BarePathState {
         }
     }
 
-    pub fn into_bare(self) -> Result<Span, ShellError> {
+    pub fn into_bare(self) -> Result<Span, ParseError> {
         match self {
             BarePathState::Initial => unreachable!("into_bare in initial state"),
             BarePathState::Seen(start, end) => Ok(start.until(end)),
@@ -570,7 +553,7 @@ pub fn expand_bare<'a, 'b>(
     token_nodes: &'b mut TokensIterator<'a>,
     _context: &ExpandContext,
     predicate: impl Fn(&TokenNode) -> bool,
-) -> Result<Span, ShellError> {
+) -> Result<Span, ParseError> {
     let mut state = BarePathState::Initial;
 
     loop {
@@ -603,11 +586,15 @@ pub struct BarePathShape;
 impl ExpandSyntax for BarePathShape {
     type Output = Span;
 
+    fn name(&self) -> &'static str {
+        "shorthand path"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Span, ShellError> {
+    ) -> Result<Span, ParseError> {
         expand_bare(token_nodes, context, |token| match token {
             TokenNode::Token(Spanned {
                 item: RawToken::Bare,
@@ -638,19 +625,21 @@ impl FallibleColorSyntax for BareShape {
         _context: &ExpandContext,
         shapes: &mut Vec<Spanned<FlatShape>>,
     ) -> Result<(), ShellError> {
-        token_nodes.peek_any_token("word", |token| match token {
+        token_nodes
-            // If it's a bare token, color it
+            .peek_any_token("word", |token| match token {
-            TokenNode::Token(Spanned {
+                // If it's a bare token, color it
-                item: RawToken::Bare,
+                TokenNode::Token(Spanned {
-                span,
+                    item: RawToken::Bare,
-            }) => {
+                    span,
-                shapes.push((*input).spanned(*span));
+                }) => {
-                Ok(())
+                    shapes.push((*input).spanned(*span));
-            }
+                    Ok(())
+                }
 
-            // otherwise, fail
+                // otherwise, fail
-            other => Err(ShellError::type_error("word", other.tagged_type_name())),
+                other => Err(ParseError::mismatch("word", other.tagged_type_name())),
-        })
+            })
+            .map_err(|err| err.into())
     }
 }
 
@@ -677,7 +666,7 @@ impl FallibleColorSyntax for BareShape {
             }) => Ok(span),
 
             // otherwise, fail
-            other => Err(ShellError::type_error("word", other.tagged_type_name())),
+            other => Err(ParseError::mismatch("word", other.tagged_type_name())),
         })?;
 
         token_nodes.color_shape((*input).spanned(*span));
@@ -689,11 +678,15 @@ impl FallibleColorSyntax for BareShape {
 impl ExpandSyntax for BareShape {
     type Output = Spanned<String>;
 
+    fn name(&self) -> &'static str {
+        "word"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
         let peeked = token_nodes.peek_any().not_eof("word")?;
 
         match peeked.node {
@@ -705,7 +698,7 @@ impl ExpandSyntax for BareShape {
                 Ok(span.spanned_string(context.source))
             }
 
-            other => Err(ShellError::type_error("word", other.tagged_type_name())),
+            other => Err(ParseError::mismatch("word", other.tagged_type_name())),
         }
     }
 }
@@ -725,7 +718,7 @@ impl TestSyntax for BareShape {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum CommandSignature {
     Internal(Spanned<Arc<Command>>),
     LiteralExternal { outer: Span, inner: Span },
@@ -733,6 +726,34 @@ pub enum CommandSignature {
     Expression(hir::Expression),
 }
 
+impl FormatDebug for CommandSignature {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            CommandSignature::Internal(internal) => {
+                f.say_str("internal", internal.span.slice(source))
+            }
+            CommandSignature::LiteralExternal { outer, .. } => {
+                f.say_str("external", outer.slice(source))
+            }
+            CommandSignature::External(external) => {
+                write!(f, "external:{}", external.slice(source))
+            }
+            CommandSignature::Expression(expr) => expr.fmt_debug(f, source),
+        }
+    }
+}
+
+impl HasSpan for CommandSignature {
+    fn span(&self) -> Span {
+        match self {
+            CommandSignature::Internal(spanned) => spanned.span,
+            CommandSignature::LiteralExternal { outer, .. } => *outer,
+            CommandSignature::External(span) => *span,
+            CommandSignature::Expression(expr) => expr.span,
+        }
+    }
+}
+
 impl CommandSignature {
     pub fn to_expression(&self) -> hir::Expression {
         match self {
@@ -833,12 +854,17 @@ impl FallibleColorSyntax for PipelineShape {
 #[cfg(coloring_in_tokens)]
 impl ExpandSyntax for PipelineShape {
     type Output = ClassifiedPipeline;
+
+    fn name(&self) -> &'static str {
+        "pipeline"
+    }
+
     fn expand_syntax<'content, 'me>(
         &self,
         iterator: &'me mut TokensIterator<'content>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
-        let source = context.source;
+        let start = iterator.span_at_cursor();
 
         let peeked = iterator.peek_any().not_eof("pipeline")?;
         let pipeline = peeked.commit().as_pipeline()?;
@@ -851,25 +877,34 @@ impl ExpandSyntax for PipelineShape {
             let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span);
 
             let classified = iterator.child(tokens, move |token_nodes| {
-                classify_command(token_nodes, context, &source)
+                expand_syntax(&ClassifiedCommandShape, token_nodes, context)
             })?;
 
             out.push(classified);
         }
 
-        Ok(ClassifiedPipeline { commands: out })
+        let end = iterator.span_at_cursor();
+
+        Ok(ClassifiedPipeline {
+            commands: out.spanned(start.until(end)),
+        })
     }
 }
 
 #[cfg(not(coloring_in_tokens))]
 impl ExpandSyntax for PipelineShape {
     type Output = ClassifiedPipeline;
+
+    fn name(&self) -> &'static str {
+        "pipeline"
+    }
+
     fn expand_syntax<'content, 'me>(
         &self,
         iterator: &'me mut TokensIterator<'content>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
-        let source = context.source;
+        let start = iterator.span_at_cursor();
 
         let peeked = iterator.peek_any().not_eof("pipeline")?;
         let pipeline = peeked.commit().as_pipeline()?;
@@ -882,13 +917,17 @@ impl ExpandSyntax for PipelineShape {
             let tokens: Spanned<&[TokenNode]> = (&part.item.tokens[..]).spanned(part.span);
 
             let classified = iterator.child(tokens, move |token_nodes| {
-                classify_command(token_nodes, context, &source)
+                expand_syntax(&ClassifiedCommandShape, token_nodes, context)
             })?;
 
             out.push(classified);
         }
 
-        Ok(ClassifiedPipeline { commands: out })
+        let end = iterator.span_at_cursor();
+
+        Ok(ClassifiedPipeline {
+            commands: out.spanned(start.until(end)),
+        })
     }
 }
 
@@ -1014,11 +1053,15 @@ impl FallibleColorSyntax for CommandHeadShape {
 impl ExpandSyntax for CommandHeadShape {
     type Output = CommandSignature;
 
+    fn name(&self) -> &'static str {
+        "command head"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<CommandSignature, ShellError> {
+    ) -> Result<CommandSignature, ParseError> {
         let node =
             parse_single_node_skipping_ws(token_nodes, "command head1", |token, token_span, _| {
                 Ok(match token {
@@ -1060,29 +1103,34 @@ pub struct ClassifiedCommandShape;
 impl ExpandSyntax for ClassifiedCommandShape {
     type Output = ClassifiedCommand;
 
+    fn name(&self) -> &'static str {
+        "classified command"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         iterator: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
+        let start = iterator.span_at_cursor();
         let head = expand_syntax(&CommandHeadShape, iterator, context)?;
 
         match &head {
-            CommandSignature::Expression(expr) => Err(ShellError::syntax_error(
+            CommandSignature::Expression(expr) => {
-                "Unexpected expression in command position".tagged(expr.span),
+                Err(ParseError::mismatch("command", expr.tagged_type_name()))
-            )),
+            }
 
             // If the command starts with `^`, treat it as an external command no matter what
             CommandSignature::External(name) => {
                 let name_str = name.slice(&context.source);
 
-                external_command(iterator, &context.source, name_str.tagged(name))
+                external_command(iterator, context, name_str.tagged(name))
             }
 
             CommandSignature::LiteralExternal { outer, inner } => {
                 let name_str = inner.slice(&context.source);
 
-                external_command(iterator, &context.source, name_str.tagged(outer))
+                external_command(iterator, context, name_str.tagged(outer))
             }
 
             CommandSignature::Internal(command) => {
@@ -1094,11 +1142,14 @@ impl ExpandSyntax for ClassifiedCommandShape {
                     Some((positional, named)) => (positional, named),
                 };
 
+                let end = iterator.span_at_cursor();
+
                 let call = hir::Call {
                     head: Box::new(head.to_expression()),
                     positional,
                     named,
-                };
+                }
+                .spanned(start.until(end));
 
                 Ok(ClassifiedCommand::Internal(InternalCommand::new(
                     command.item.name().to_string(),
@@ -1198,12 +1249,16 @@ impl FallibleColorSyntax for InternalCommandHeadShape {
 }
 
 impl ExpandExpression for InternalCommandHeadShape {
+    fn name(&self) -> &'static str {
+        "internal command head"
+    }
+
     fn expand_expr(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         _context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
-        let peeked_head = token_nodes.peek_non_ws().not_eof("command head4")?;
+        let peeked_head = token_nodes.peek_non_ws().not_eof("command head")?;
 
         let expr = match peeked_head.node {
             TokenNode::Token(
@@ -1219,8 +1274,8 @@ impl ExpandExpression for InternalCommandHeadShape {
             }) => hir::RawExpression::Literal(hir::Literal::String(*inner_span)).spanned(*span),
 
             node => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
-                    "command head5",
+                    "command head",
                     node.tagged_type_name(),
                 ))
             }
@@ -1238,16 +1293,16 @@ pub(crate) struct SingleError<'token> {
 }
 
 impl<'token> SingleError<'token> {
-    pub(crate) fn error(&self) -> ShellError {
+    pub(crate) fn error(&self) -> ParseError {
-        ShellError::type_error(self.expected, self.node.type_name().tagged(self.node.span))
+        ParseError::mismatch(self.expected, self.node.type_name().tagged(self.node.span))
     }
 }
 
 fn parse_single_node<'a, 'b, T>(
     token_nodes: &'b mut TokensIterator<'a>,
     expected: &'static str,
-    callback: impl FnOnce(RawToken, Span, SingleError) -> Result<T, ShellError>,
+    callback: impl FnOnce(RawToken, Span, SingleError) -> Result<T, ParseError>,
-) -> Result<T, ShellError> {
+) -> Result<T, ParseError> {
     token_nodes.peek_any_token(expected, |node| match node {
         TokenNode::Token(token) => callback(
             token.item,
@@ -1258,7 +1313,7 @@ fn parse_single_node<'a, 'b, T>(
             },
         ),
 
-        other => Err(ShellError::type_error(expected, other.tagged_type_name())),
+        other => Err(ParseError::mismatch(expected, other.tagged_type_name())),
     })
 }
 
@@ -1360,22 +1415,21 @@ impl FallibleColorSyntax for WhitespaceShape {
 impl ExpandSyntax for WhitespaceShape {
     type Output = Span;
 
+    fn name(&self) -> &'static str {
+        "whitespace"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         _context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
         let peeked = token_nodes.peek_any().not_eof("whitespace")?;
 
         let span = match peeked.node {
             TokenNode::Whitespace(tag) => *tag,
 
-            other => {
+            other => return Err(ParseError::mismatch("whitespace", other.tagged_type_name())),
-                return Err(ShellError::type_error(
-                    "whitespace",
-                    other.tagged_type_name(),
-                ))
-            }
         };
 
         peeked.commit();
@@ -1390,11 +1444,15 @@ pub struct SpacedExpression<T: ExpandExpression> {
 }
 
 impl<T: ExpandExpression> ExpandExpression for SpacedExpression<T> {
+    fn name(&self) -> &'static str {
+        "spaced expression"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // TODO: Make the name part of the trait
         let peeked = token_nodes.peek_any().not_eof("whitespace")?;
 
@@ -1404,10 +1462,7 @@ impl<T: ExpandExpression> ExpandExpression for SpacedExpression<T> {
                 expand_expr(&self.inner, token_nodes, context)
             }
 
-            other => Err(ShellError::type_error(
+            other => Err(ParseError::mismatch("whitespace", other.tagged_type_name())),
-                "whitespace",
-                other.tagged_type_name(),
-            )),
         }
     }
 }
@@ -1424,6 +1479,36 @@ pub struct MaybeSpacedExpression<T: ExpandExpression> {
 #[derive(Debug, Copy, Clone)]
 pub struct MaybeSpaceShape;
 
+impl ExpandSyntax for MaybeSpaceShape {
+    type Output = Option<Span>;
+
+    fn name(&self) -> &'static str {
+        "maybe space"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        _context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        let peeked = token_nodes.peek_any().not_eof("whitespace");
+
+        let span = match peeked {
+            Err(_) => None,
+            Ok(peeked) => {
+                if let TokenNode::Whitespace(..) = peeked.node {
+                    let node = peeked.commit();
+                    Some(node.span())
+                } else {
+                    None
+                }
+            }
+        };
+
+        Ok(span)
+    }
+}
+
 #[cfg(not(coloring_in_tokens))]
 impl ColorSyntax for MaybeSpaceShape {
     type Info = ();
@@ -1544,11 +1629,15 @@ impl FallibleColorSyntax for SpaceShape {
 }
 
 impl<T: ExpandExpression> ExpandExpression for MaybeSpacedExpression<T> {
+    fn name(&self) -> &'static str {
+        "maybe space"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // TODO: Make the name part of the trait
         let peeked = token_nodes.peek_any().not_eof("whitespace")?;
 
@@ -1578,58 +1667,6 @@ fn expand_variable(span: Span, token_span: Span, source: &Text) -> hir::Expressi
     }
 }
 
-fn classify_command(
-    mut iterator: &mut TokensIterator,
-    context: &ExpandContext,
-    source: &Text,
-) -> Result<ClassifiedCommand, ShellError> {
-    let head = CommandHeadShape.expand_syntax(&mut iterator, &context)?;
-
-    match &head {
-        CommandSignature::Expression(_) => Err(ShellError::syntax_error(
-            "Unexpected expression in command position".tagged(iterator.whole_span()),
-        )),
-
-        // If the command starts with `^`, treat it as an external command no matter what
-        CommandSignature::External(name) => {
-            let name_str = name.slice(source);
-
-            external_command(&mut iterator, source, name_str.tagged(name))
-        }
-
-        CommandSignature::LiteralExternal { outer, inner } => {
-            let name_str = inner.slice(source);
-
-            external_command(&mut iterator, source, name_str.tagged(outer))
-        }
-
-        CommandSignature::Internal(command) => {
-            let tail =
-                parse_command_tail(&command.signature(), &context, &mut iterator, command.span)?;
-
-            let (positional, named) = match tail {
-                None => (None, None),
-                Some((positional, named)) => (positional, named),
-            };
-
-            let call = hir::Call {
-                head: Box::new(head.to_expression()),
-                positional,
-                named,
-            };
-
-            Ok(ClassifiedCommand::Internal(InternalCommand::new(
-                command.name().to_string(),
-                Tag {
-                    span: command.span,
-                    anchor: None,
-                },
-                call,
-            )))
-        }
-    }
-}
-
 #[derive(Debug, Copy, Clone)]
 pub struct CommandShape;
 

src/parser/hir/syntax_shape/block.rs

@@ -6,7 +6,8 @@ use crate::parser::{
     hir::syntax_shape::{
         color_fallible_syntax, color_syntax_with, continue_expression, expand_expr, expand_syntax,
         DelimitedShape, ExpandContext, ExpandExpression, ExpressionContinuationShape,
-        ExpressionListShape, FallibleColorSyntax, MemberShape, PathTailShape, VariablePathShape,
+        ExpressionListShape, FallibleColorSyntax, MemberShape, ParseError, PathTailShape,
+        VariablePathShape,
     },
     hir::tokens_iterator::TokensIterator,
     parse::token_tree::Delimiter,
@@ -42,7 +43,7 @@ impl FallibleColorSyntax for AnyBlockShape {
         match block {
             // If so, color it as a block
             Some((children, spans)) => {
-                let mut token_nodes = TokensIterator::new(children.item, context.span, false);
+                let mut token_nodes = TokensIterator::new(children.item, children.span, false);
                 color_syntax_with(
                     &DelimitedShape,
                     &(Delimiter::Brace, spans.0, spans.1),
@@ -109,11 +110,15 @@ impl FallibleColorSyntax for AnyBlockShape {
 }
 
 impl ExpandExpression for AnyBlockShape {
+    fn name(&self) -> &'static str {
+        "any block"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         let block = token_nodes.peek_non_ws().not_eof("block")?;
 
         // is it just a block?
@@ -121,11 +126,11 @@ impl ExpandExpression for AnyBlockShape {
 
         match block {
             Some((block, _tags)) => {
-                let mut iterator = TokensIterator::new(&block.item, context.span, false);
+                let mut iterator = TokensIterator::new(&block.item, block.span, false);
 
                 let exprs = expand_syntax(&ExpressionListShape, &mut iterator, context)?;
 
-                return Ok(hir::RawExpression::Block(exprs).spanned(block.span));
+                return Ok(hir::RawExpression::Block(exprs.item).spanned(block.span));
             }
             _ => {}
         }
@@ -204,14 +209,18 @@ impl FallibleColorSyntax for ShorthandBlock {
 }
 
 impl ExpandExpression for ShorthandBlock {
+    fn name(&self) -> &'static str {
+        "shorthand block"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         let path = expand_expr(&ShorthandPath, token_nodes, context)?;
         let start = path.span;
-        let expr = continue_expression(path, token_nodes, context)?;
+        let expr = continue_expression(path, token_nodes, context);
         let end = expr.span;
         let block = hir::RawExpression::Block(vec![expr]).spanned(start.until(end));
 
@@ -317,11 +326,15 @@ impl FallibleColorSyntax for ShorthandPath {
 }
 
 impl ExpandExpression for ShorthandPath {
+    fn name(&self) -> &'static str {
+        "shorthand path"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // if it's a variable path, that's the head part
         let path = expand_expr(&VariablePathShape, token_nodes, context);
 
@@ -339,7 +352,7 @@ impl ExpandExpression for ShorthandPath {
 
         match tail {
             Err(_) => return Ok(head),
-            Ok((tail, _)) => {
+            Ok(Spanned { item: tail, .. }) => {
                 // For each member that `PathTailShape` expanded, join it onto the existing expression
                 // to form a new path
                 for member in tail {
@@ -446,11 +459,15 @@ impl FallibleColorSyntax for ShorthandHeadShape {
 }
 
 impl ExpandExpression for ShorthandHeadShape {
+    fn name(&self) -> &'static str {
+        "shorthand head"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // A shorthand path must not be at EOF
         let peeked = token_nodes.peek_non_ws().not_eof("shorthand path")?;
 
@@ -495,7 +512,7 @@ impl ExpandExpression for ShorthandHeadShape {
 
             // Any other token is not a valid bare head
             other => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
                     "shorthand path",
                     other.tagged_type_name(),
                 ))

src/parser/hir/syntax_shape/expression.rs

@@ -12,7 +12,7 @@ use crate::parser::hir::syntax_shape::{
     color_delimited_square, color_fallible_syntax, color_fallible_syntax_with, expand_atom,
     expand_delimited_square, expand_expr, expand_syntax, AtomicToken, BareShape, ColorableDotShape,
     DotShape, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, ExpressionContinuation,
-    ExpressionContinuationShape, FallibleColorSyntax, FlatShape,
+    ExpressionContinuationShape, FallibleColorSyntax, FlatShape, ParseError,
 };
 use crate::parser::{
     hir,
@@ -25,15 +25,19 @@ use std::path::PathBuf;
 pub struct AnyExpressionShape;
 
 impl ExpandExpression for AnyExpressionShape {
+    fn name(&self) -> &'static str {
+        "any expression"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // Look for an expression at the cursor
         let head = expand_expr(&AnyExpressionStartShape, token_nodes, context)?;
 
-        continue_expression(head, token_nodes, context)
+        Ok(continue_expression(head, token_nodes, context))
     }
 }
 
@@ -98,14 +102,14 @@ pub(crate) fn continue_expression(
     mut head: hir::Expression,
     token_nodes: &mut TokensIterator<'_>,
     context: &ExpandContext,
-) -> Result<hir::Expression, ShellError> {
+) -> hir::Expression {
     loop {
         // Check to see whether there's any continuation after the head expression
         let continuation = expand_syntax(&ExpressionContinuationShape, token_nodes, context);
 
         match continuation {
             // If there's no continuation, return the head
-            Err(_) => return Ok(head),
+            Err(_) => return head,
             // Otherwise, form a new expression by combining the head with the continuation
             Ok(continuation) => match continuation {
                 // If the continuation is a `.member`, form a path with the new member
@@ -174,11 +178,15 @@ pub(crate) fn continue_coloring_expression(
 pub struct AnyExpressionStartShape;
 
 impl ExpandExpression for AnyExpressionStartShape {
+    fn name(&self) -> &'static str {
+        "any expression start"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         let atom = expand_atom(token_nodes, "expression", context, ExpansionRule::new())?;
 
         match atom.item {
@@ -445,13 +453,17 @@ impl FallibleColorSyntax for BareTailShape {
 }
 
 impl ExpandSyntax for BareTailShape {
+    fn name(&self) -> &'static str {
+        "word continuation"
+    }
+
     type Output = Option<Span>;
 
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Option<Span>, ShellError> {
+    ) -> Result<Option<Span>, ParseError> {
         let mut end: Option<Span> = None;
 
         loop {

src/parser/hir/syntax_shape/expression/atom.rs

@@ -90,40 +90,40 @@ impl<'tokens> SpannedAtomicToken<'tokens> {
         &self,
         context: &ExpandContext,
         expected: &'static str,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         Ok(match &self.item {
             AtomicToken::Eof { .. } => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
                     expected,
                     "eof atomic token".tagged(self.span),
                 ))
             }
             AtomicToken::Error { .. } => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
                     expected,
                     "eof atomic token".tagged(self.span),
                 ))
             }
             AtomicToken::Operator { .. } => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(expected, "operator".tagged(self.span)))
-                    expected,
-                    "operator".tagged(self.span),
-                ))
             }
             AtomicToken::ShorthandFlag { .. } => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
                     expected,
                     "shorthand flag".tagged(self.span),
                 ))
             }
             AtomicToken::LonghandFlag { .. } => {
-                return Err(ShellError::type_error(expected, "flag".tagged(self.span)))
+                return Err(ParseError::mismatch(expected, "flag".tagged(self.span)))
             }
             AtomicToken::Whitespace { .. } => {
-                return Err(ShellError::unimplemented("whitespace in AtomicToken"))
+                return Err(ParseError::mismatch(
+                    expected,
+                    "whitespace".tagged(self.span),
+                ))
             }
             AtomicToken::Dot { .. } => {
-                return Err(ShellError::type_error(expected, "dot".tagged(self.span)))
+                return Err(ParseError::mismatch(expected, "dot".tagged(self.span)))
             }
             AtomicToken::Number { number } => {
                 Expression::number(number.to_number(context.source), self.span)
@@ -381,7 +381,7 @@ pub fn expand_atom<'me, 'content>(
     expected: &'static str,
     context: &ExpandContext,
     rule: ExpansionRule,
-) -> Result<SpannedAtomicToken<'content>, ShellError> {
+) -> Result<SpannedAtomicToken<'content>, ParseError> {
     if token_nodes.at_end() {
         match rule.allow_eof {
             true => {
@@ -390,7 +390,7 @@ pub fn expand_atom<'me, 'content>(
                 }
                 .spanned(Span::unknown()))
             }
-            false => return Err(ShellError::unexpected_eof("anything", Tag::unknown())),
+            false => return Err(ParseError::unexpected_eof("anything", Span::unknown())),
         }
     }
 
@@ -515,12 +515,13 @@ pub fn expand_atom<'me, 'content>(
 
             // if whitespace is disallowed, return an error
             WhitespaceHandling::RejectWhitespace => {
-                return Err(ShellError::syntax_error("Unexpected whitespace".tagged(
+                return Err(ParseError::mismatch(
-                    Tag {
+                    expected,
+                    "whitespace".tagged(Tag {
                         span: *span,
                         anchor: None,
-                    },
+                    }),
-                )))
+                ))
             }
         },
 
@@ -544,7 +545,7 @@ pub fn expand_atom<'me, 'content>(
             RawToken::Operator(_) if !rule.allow_operator => return Err(err.error()),
             // rule.allow_external_command
             RawToken::ExternalCommand(_) if !rule.allow_external_command => {
-                return Err(ShellError::type_error(
+                return Err(ParseError::mismatch(
                     expected,
                     token.type_name().tagged(Tag {
                         span: token_span,
@@ -554,10 +555,13 @@ pub fn expand_atom<'me, 'content>(
             }
             // rule.allow_external_word
             RawToken::ExternalWord if !rule.allow_external_word => {
-                return Err(ShellError::invalid_external_word(Tag {
+                return Err(ParseError::mismatch(
-                    span: token_span,
+                    expected,
-                    anchor: None,
+                    "external word".tagged(Tag {
-                }))
+                        span: token_span,
+                        anchor: None,
+                    }),
+                ))
             }
 
             RawToken::Number(number) => AtomicToken::Number { number }.spanned(token_span),

src/parser/hir/syntax_shape/expression/delimited.rs

@@ -8,12 +8,15 @@ pub fn expand_delimited_square(
     children: &Vec<TokenNode>,
     span: Span,
     context: &ExpandContext,
-) -> Result<hir::Expression, ShellError> {
+) -> Result<hir::Expression, ParseError> {
     let mut tokens = TokensIterator::new(&children, span, false);
 
     let list = expand_syntax(&ExpressionListShape, &mut tokens, context);
 
-    Ok(hir::Expression::list(list?, Tag { span, anchor: None }))
+    Ok(hir::Expression::list(
+        list?.item,
+        Tag { span, anchor: None },
+    ))
 }
 
 #[cfg(not(coloring_in_tokens))]

src/parser/hir/syntax_shape/expression/file_path.rs

@@ -1,6 +1,7 @@
 use crate::parser::hir::syntax_shape::expression::atom::{expand_atom, AtomicToken, ExpansionRule};
 use crate::parser::hir::syntax_shape::{
     expression::expand_file_path, ExpandContext, ExpandExpression, FallibleColorSyntax, FlatShape,
+    ParseError,
 };
 use crate::parser::{hir, hir::TokensIterator};
 use crate::prelude::*;
@@ -90,11 +91,15 @@ impl FallibleColorSyntax for FilePathShape {
 }
 
 impl ExpandExpression for FilePathShape {
+    fn name(&self) -> &'static str {
+        "file path"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         let atom = expand_atom(token_nodes, "file path", context, ExpansionRule::new())?;
 
         match atom.item {

src/parser/hir/syntax_shape/expression/list.rs

@@ -1,4 +1,4 @@
-use crate::errors::ShellError;
+use crate::errors::ParseError;
 #[cfg(not(coloring_in_tokens))]
 use crate::parser::hir::syntax_shape::FlatShape;
 use crate::parser::{
@@ -10,24 +10,36 @@ use crate::parser::{
     },
     hir::TokensIterator,
 };
-#[cfg(not(coloring_in_tokens))]
+use crate::{DebugFormatter, FormatDebug, Spanned, SpannedItem};
-use crate::Spanned;
+use std::fmt;
 
 #[derive(Debug, Copy, Clone)]
 pub struct ExpressionListShape;
 
+impl FormatDebug for Spanned<Vec<hir::Expression>> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        FormatDebug::fmt_debug(&self.item, f, source)
+    }
+}
+
 impl ExpandSyntax for ExpressionListShape {
-    type Output = Vec<hir::Expression>;
+    type Output = Spanned<Vec<hir::Expression>>;
+
+    fn name(&self) -> &'static str {
+        "expression list"
+    }
 
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<Vec<hir::Expression>, ShellError> {
+    ) -> Result<Spanned<Vec<hir::Expression>>, ParseError> {
         let mut exprs = vec![];
 
+        let start = token_nodes.span_at_cursor();
+
         if token_nodes.at_end_possible_ws() {
-            return Ok(exprs);
+            return Ok(exprs.spanned(start));
         }
 
         let expr = expand_expr(&maybe_spaced(AnyExpressionShape), token_nodes, context)?;
@@ -36,7 +48,8 @@ impl ExpandSyntax for ExpressionListShape {
 
         loop {
             if token_nodes.at_end_possible_ws() {
-                return Ok(exprs);
+                let end = token_nodes.span_at_cursor();
+                return Ok(exprs.spanned(start.until(end)));
             }
 
             let expr = expand_expr(&spaced(AnyExpressionShape), token_nodes, context)?;

src/parser/hir/syntax_shape/expression/number.rs

@@ -1,6 +1,6 @@
 use crate::parser::hir::syntax_shape::{
     expand_atom, parse_single_node, ExpandContext, ExpandExpression, ExpansionRule,
-    FallibleColorSyntax, FlatShape,
+    FallibleColorSyntax, FlatShape, ParseError,
 };
 use crate::parser::{
     hir,
@@ -13,11 +13,15 @@ use crate::prelude::*;
 pub struct NumberShape;
 
 impl ExpandExpression for NumberShape {
+    fn name(&self) -> &'static str {
+        "number"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         parse_single_node(token_nodes, "Number", |token, token_span, err| {
             Ok(match token {
                 RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()),
@@ -28,10 +32,13 @@ impl ExpandExpression for NumberShape {
                     hir::Expression::external_command(tag, token_span)
                 }
                 RawToken::ExternalWord => {
-                    return Err(ShellError::invalid_external_word(Tag {
+                    return Err(ParseError::mismatch(
-                        span: token_span,
+                        "number",
-                        anchor: None,
+                        "syntax error".tagged(Tag {
-                    }))
+                            span: token_span,
+                            anchor: None,
+                        }),
+                    ))
                 }
                 RawToken::Variable(tag) => hir::Expression::variable(tag, token_span),
                 RawToken::Number(number) => {
@@ -111,16 +118,19 @@ impl FallibleColorSyntax for NumberShape {
 pub struct IntShape;
 
 impl ExpandExpression for IntShape {
+    fn name(&self) -> &'static str {
+        "integer"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         parse_single_node(token_nodes, "Integer", |token, token_span, err| {
             Ok(match token {
-                RawToken::GlobPattern | RawToken::Operator(..) => return Err(err.error()),
+                RawToken::GlobPattern | RawToken::Operator(..) | RawToken::ExternalWord => {
-                RawToken::ExternalWord => {
+                    return Err(err.error())
-                    return Err(ShellError::invalid_external_word(token_span))
                 }
                 RawToken::Variable(span) if span.slice(context.source) == "it" => {
                     hir::Expression::it_variable(span, token_span)

src/parser/hir/syntax_shape/expression/pattern.rs

@@ -1,6 +1,6 @@
 use crate::parser::hir::syntax_shape::{
     expand_atom, expand_bare, expression::expand_file_path, AtomicToken, ExpandContext,
-    ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape,
+    ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError,
 };
 use crate::parser::{hir, hir::TokensIterator, Operator, RawToken, TokenNode};
 use crate::prelude::*;
@@ -66,11 +66,15 @@ impl FallibleColorSyntax for PatternShape {
 }
 
 impl ExpandExpression for PatternShape {
+    fn name(&self) -> &'static str {
+        "glob pattern"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::new())?;
 
         match atom.item {
@@ -91,11 +95,15 @@ pub struct BarePatternShape;
 impl ExpandSyntax for BarePatternShape {
     type Output = Span;
 
+    fn name(&self) -> &'static str {
+        "bare pattern"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Span, ShellError> {
+    ) -> Result<Span, ParseError> {
         expand_bare(token_nodes, context, |token| match token {
             TokenNode::Token(Spanned {
                 item: RawToken::Bare,

src/parser/hir/syntax_shape/expression/string.rs

@@ -1,6 +1,6 @@
 use crate::parser::hir::syntax_shape::{
     expand_atom, expand_variable, parse_single_node, AtomicToken, ExpandContext, ExpandExpression,
-    ExpansionRule, FallibleColorSyntax, FlatShape, TestSyntax,
+    ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, TestSyntax,
 };
 use crate::parser::hir::tokens_iterator::Peeked;
 use crate::parser::{hir, hir::TokensIterator, RawToken};
@@ -75,32 +75,24 @@ impl FallibleColorSyntax for StringShape {
 }
 
 impl ExpandExpression for StringShape {
+    fn name(&self) -> &'static str {
+        "string"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
-        parse_single_node(token_nodes, "String", |token, token_span, _| {
+        parse_single_node(token_nodes, "String", |token, token_span, err| {
             Ok(match token {
-                RawToken::GlobPattern => {
+                RawToken::GlobPattern | RawToken::Operator(..) | RawToken::ExternalWord => {
-                    return Err(ShellError::type_error(
+                    return Err(err.error())
-                        "String",
-                        "glob pattern".tagged(token_span),
-                    ))
-                }
-                RawToken::Operator(..) => {
-                    return Err(ShellError::type_error(
-                        "String",
-                        "operator".tagged(token_span),
-                    ))
                 }
                 RawToken::Variable(span) => expand_variable(span, token_span, &context.source),
                 RawToken::ExternalCommand(span) => {
                     hir::Expression::external_command(span, token_span)
                 }
-                RawToken::ExternalWord => {
-                    return Err(ShellError::invalid_external_word(token_span))
-                }
                 RawToken::Number(_) => hir::Expression::bare(token_span),
                 RawToken::Bare => hir::Expression::bare(token_span),
                 RawToken::String(span) => hir::Expression::string(span, token_span),

src/parser/hir/syntax_shape/expression/unit.rs

@@ -1,5 +1,5 @@
 use crate::data::meta::Span;
-use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax};
+use crate::parser::hir::syntax_shape::{ExpandContext, ExpandSyntax, ParseError};
 use crate::parser::parse::tokens::RawNumber;
 use crate::parser::parse::unit::Unit;
 use crate::parser::{hir::TokensIterator, RawToken, TokenNode};
@@ -9,18 +9,34 @@ use nom::bytes::complete::tag;
 use nom::character::complete::digit1;
 use nom::combinator::{all_consuming, opt, value};
 use nom::IResult;
+use std::fmt;
 
 #[derive(Debug, Copy, Clone)]
 pub struct UnitShape;
 
+impl FormatDebug for Spanned<(Spanned<RawNumber>, Spanned<Unit>)> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        let dict = indexmap::indexmap! {
+            "number" => format!("{}", self.item.0.item.debug(source)),
+            "unit" => format!("{}", self.item.1.debug(source)),
+        };
+
+        f.say_dict("unit", dict)
+    }
+}
+
 impl ExpandSyntax for UnitShape {
     type Output = Spanned<(Spanned<RawNumber>, Spanned<Unit>)>;
 
+    fn name(&self) -> &'static str {
+        "unit"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Spanned<(Spanned<RawNumber>, Spanned<Unit>)>, ShellError> {
+    ) -> Result<Spanned<(Spanned<RawNumber>, Spanned<Unit>)>, ParseError> {
         let peeked = token_nodes.peek_any().not_eof("unit")?;
 
         let span = match peeked.node {
@@ -34,12 +50,7 @@ impl ExpandSyntax for UnitShape {
         let unit = unit_size(span.slice(context.source), *span);
 
         let (_, (number, unit)) = match unit {
-            Err(_) => {
+            Err(_) => return Err(ParseError::mismatch("unit", "word".tagged(Tag::unknown()))),
-                return Err(ShellError::type_error(
-                    "unit",
-                    "word".tagged(Tag::unknown()),
-                ))
-            }
             Ok((number, unit)) => (number, unit),
         };
 

src/parser/hir/syntax_shape/expression/variable_path.rs

@@ -1,21 +1,28 @@
 use crate::parser::hir::syntax_shape::{
     color_fallible_syntax, color_fallible_syntax_with, expand_atom, expand_expr, expand_syntax,
     parse_single_node, AnyExpressionShape, AtomicToken, BareShape, ExpandContext, ExpandExpression,
-    ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, Peeked, SkipSyntax, StringShape,
+    ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, ParseError, Peeked, SkipSyntax,
-    TestSyntax, WhitespaceShape,
+    StringShape, TestSyntax, WhitespaceShape,
 };
 use crate::parser::{hir, hir::Expression, hir::TokensIterator, Operator, RawToken};
 use crate::prelude::*;
+use derive_new::new;
+use getset::Getters;
+use std::fmt;
 
 #[derive(Debug, Copy, Clone)]
 pub struct VariablePathShape;
 
 impl ExpandExpression for VariablePathShape {
+    fn name(&self) -> &'static str {
+        "variable path"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
         // 1. let the head be the first token, expecting a variable
         // 2. let the tail be an empty list of members
         // 2. while the next token (excluding ws) is a dot:
@@ -200,12 +207,17 @@ impl FallibleColorSyntax for PathTailShape {
 }
 
 impl ExpandSyntax for PathTailShape {
-    type Output = (Vec<Spanned<String>>, Span);
+    type Output = Spanned<Vec<Spanned<String>>>;
+
+    fn name(&self) -> &'static str {
+        "path continuation"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
         let mut end: Option<Span> = None;
         let mut tail = vec![];
 
@@ -223,7 +235,7 @@ impl ExpandSyntax for PathTailShape {
 
         match end {
             None => {
-                return Err(ShellError::type_error("path tail", {
+                return Err(ParseError::mismatch("path tail", {
                     let typed_span = token_nodes.typed_span_at_cursor();
 
                     Tagged {
@@ -233,17 +245,41 @@ impl ExpandSyntax for PathTailShape {
                 }))
             }
 
-            Some(end) => Ok((tail, end)),
+            Some(end) => Ok(tail.spanned(end)),
         }
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum ExpressionContinuation {
     DotSuffix(Span, Spanned<String>),
     InfixSuffix(Spanned<Operator>, Expression),
 }
 
+impl FormatDebug for ExpressionContinuation {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            ExpressionContinuation::DotSuffix(dot, rest) => {
+                f.say_str("dot suffix", dot.until(rest.span).slice(source))
+            }
+            ExpressionContinuation::InfixSuffix(operator, expr) => {
+                f.say_str("infix suffix", operator.span.until(expr.span).slice(source))
+            }
+        }
+    }
+}
+
+impl HasSpan for ExpressionContinuation {
+    fn span(&self) -> Span {
+        match self {
+            ExpressionContinuation::DotSuffix(dot, column) => dot.until(column.span),
+            ExpressionContinuation::InfixSuffix(operator, expression) => {
+                operator.span.until(expression.span)
+            }
+        }
+    }
+}
+
 /// An expression continuation
 #[derive(Debug, Copy, Clone)]
 pub struct ExpressionContinuationShape;
@@ -251,11 +287,15 @@ pub struct ExpressionContinuationShape;
 impl ExpandSyntax for ExpressionContinuationShape {
     type Output = ExpressionContinuation;
 
+    fn name(&self) -> &'static str {
+        "expression continuation"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<ExpressionContinuation, ShellError> {
+    ) -> Result<ExpressionContinuation, ParseError> {
         // Try to expand a `.`
         let dot = expand_syntax(&DotShape, token_nodes, context);
 
@@ -270,7 +310,7 @@ impl ExpandSyntax for ExpressionContinuationShape {
 
             // Otherwise, we expect an infix operator and an expression next
             Err(_) => {
-                let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?;
+                let (_, op, _) = expand_syntax(&InfixShape, token_nodes, context)?.item;
                 let next = expand_expr(&AnyExpressionShape, token_nodes, context)?;
 
                 Ok(ExpressionContinuation::InfixSuffix(op, next))
@@ -390,12 +430,16 @@ impl FallibleColorSyntax for ExpressionContinuationShape {
 pub struct VariableShape;
 
 impl ExpandExpression for VariableShape {
+    fn name(&self) -> &'static str {
+        "variable"
+    }
+
     fn expand_expr<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<hir::Expression, ShellError> {
+    ) -> Result<hir::Expression, ParseError> {
-        parse_single_node(token_nodes, "variable", |token, token_tag, _| {
+        parse_single_node(token_nodes, "variable", |token, token_tag, err| {
             Ok(match token {
                 RawToken::Variable(tag) => {
                     if tag.slice(context.source) == "it" {
@@ -404,12 +448,7 @@ impl ExpandExpression for VariableShape {
                         hir::Expression::variable(tag, token_tag)
                     }
                 }
-                _ => {
+                _ => return Err(err.error()),
-                    return Err(ShellError::type_error(
-                        "variable",
-                        token.type_name().tagged(token_tag),
-                    ))
-                }
             })
         })
     }
@@ -435,7 +474,7 @@ impl FallibleColorSyntax for VariableShape {
         );
 
         let atom = match atom {
-            Err(err) => return Err(err),
+            Err(err) => return Err(err.into()),
             Ok(atom) => atom,
         };
 
@@ -476,7 +515,7 @@ impl FallibleColorSyntax for VariableShape {
         );
 
         let atom = match atom {
-            Err(err) => return Err(err),
+            Err(err) => return Err(err.into()),
             Ok(atom) => atom,
         };
 
@@ -489,7 +528,7 @@ impl FallibleColorSyntax for VariableShape {
                 token_nodes.color_shape(FlatShape::ItVariable.spanned(atom.span));
                 Ok(())
             }
-            _ => Err(ShellError::type_error("variable", atom.tagged_type_name())),
+            _ => Err(ParseError::mismatch("variable", atom.tagged_type_name()).into()),
         }
     }
 }
@@ -500,6 +539,24 @@ pub enum Member {
     Bare(Span),
 }
 
+impl FormatDebug for Member {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            Member::String(outer, _) => write!(f, "member ({})", outer.slice(source)),
+            Member::Bare(bare) => write!(f, "member ({})", bare.slice(source)),
+        }
+    }
+}
+
+impl HasSpan for Member {
+    fn span(&self) -> Span {
+        match self {
+            Member::String(outer, ..) => *outer,
+            Member::Bare(name) => *name,
+        }
+    }
+}
+
 impl Member {
     pub(crate) fn to_expr(&self) -> hir::Expression {
         match self {
@@ -538,7 +595,7 @@ enum ColumnPathState {
     LeadingDot(Span),
     Dot(Span, Vec<Member>, Span),
     Member(Span, Vec<Member>),
-    Error(ShellError),
+    Error(ParseError),
 }
 
 impl ColumnPathState {
@@ -546,10 +603,10 @@ impl ColumnPathState {
         match self {
             ColumnPathState::Initial => ColumnPathState::LeadingDot(dot),
             ColumnPathState::LeadingDot(_) => {
-                ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot)))
+                ColumnPathState::Error(ParseError::mismatch("column", "dot".tagged(dot)))
             }
             ColumnPathState::Dot(..) => {
-                ColumnPathState::Error(ShellError::type_error("column", "dot".tagged(dot)))
+                ColumnPathState::Error(ParseError::mismatch("column", "dot".tagged(dot)))
             }
             ColumnPathState::Member(tag, members) => ColumnPathState::Dot(tag, members, dot),
             ColumnPathState::Error(err) => ColumnPathState::Error(err),
@@ -570,20 +627,20 @@ impl ColumnPathState {
                 })
             }
             ColumnPathState::Member(..) => {
-                ColumnPathState::Error(ShellError::type_error("column", member.tagged_type_name()))
+                ColumnPathState::Error(ParseError::mismatch("column", member.tagged_type_name()))
             }
             ColumnPathState::Error(err) => ColumnPathState::Error(err),
         }
     }
 
-    pub fn into_path(self, next: Peeked) -> Result<Tagged<Vec<Member>>, ShellError> {
+    pub fn into_path(self, next: Peeked) -> Result<Tagged<Vec<Member>>, ParseError> {
         match self {
             ColumnPathState::Initial => Err(next.type_error("column path")),
             ColumnPathState::LeadingDot(dot) => {
-                Err(ShellError::type_error("column", "dot".tagged(dot)))
+                Err(ParseError::mismatch("column", "dot".tagged(dot)))
             }
             ColumnPathState::Dot(_tag, _members, dot) => {
-                Err(ShellError::type_error("column", "dot".tagged(dot)))
+                Err(ParseError::mismatch("column", "dot".tagged(dot)))
             }
             ColumnPathState::Member(tag, tags) => Ok(tags.tagged(tag)),
             ColumnPathState::Error(err) => Err(err),
@@ -594,7 +651,7 @@ impl ColumnPathState {
 pub fn expand_column_path<'a, 'b>(
     token_nodes: &'b mut TokensIterator<'a>,
     context: &ExpandContext,
-) -> Result<Tagged<Vec<Member>>, ShellError> {
+) -> Result<Tagged<Vec<Member>>, ParseError> {
     let mut state = ColumnPathState::Initial;
 
     loop {
@@ -720,15 +777,43 @@ impl FallibleColorSyntax for ColumnPathShape {
     }
 }
 
+impl FormatDebug for Tagged<Vec<Member>> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        self.item.fmt_debug(f, source)
+    }
+}
+
+#[derive(Debug, Clone, Getters, new)]
+pub struct ColumnPath {
+    #[get = "pub"]
+    path: Tagged<Vec<Member>>,
+}
+
+impl HasSpan for ColumnPath {
+    fn span(&self) -> Span {
+        self.path.tag.span
+    }
+}
+
+impl FormatDebug for ColumnPath {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say("column path", self.path.item.debug(source))
+    }
+}
+
 impl ExpandSyntax for ColumnPathShape {
-    type Output = Tagged<Vec<Member>>;
+    type Output = ColumnPath;
+
+    fn name(&self) -> &'static str {
+        "column path"
+    }
 
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
-        expand_column_path(token_nodes, context)
+        Ok(ColumnPath::new(expand_column_path(token_nodes, context)?))
     }
 }
 
@@ -806,11 +891,15 @@ impl FallibleColorSyntax for MemberShape {
 impl ExpandSyntax for MemberShape {
     type Output = Member;
 
+    fn name(&self) -> &'static str {
+        "column"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &mut TokensIterator<'_>,
         context: &ExpandContext,
-    ) -> Result<Member, ShellError> {
+    ) -> Result<Member, ParseError> {
         let bare = BareShape.test(token_nodes, context);
         if let Some(peeked) = bare {
             let node = peeked.not_eof("column")?.commit();
@@ -906,16 +995,20 @@ impl SkipSyntax for DotShape {
 impl ExpandSyntax for DotShape {
     type Output = Span;
 
+    fn name(&self) -> &'static str {
+        "dot"
+    }
+
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         _context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
         parse_single_node(token_nodes, "dot", |token, token_span, _| {
             Ok(match token {
                 RawToken::Operator(Operator::Dot) => token_span,
                 _ => {
-                    return Err(ShellError::type_error(
+                    return Err(ParseError::mismatch(
                         "dot",
                         token.type_name().tagged(token_span),
                     ))
@@ -950,7 +1043,7 @@ impl FallibleColorSyntax for InfixShape {
         parse_single_node(
             checkpoint.iterator,
             "infix operator",
-            |token, token_span, _| {
+            |token, token_span, err| {
                 match token {
                     // If it's an operator (and not `.`), it's a match
                     RawToken::Operator(operator) if operator != Operator::Dot => {
@@ -959,10 +1052,7 @@ impl FallibleColorSyntax for InfixShape {
                     }
 
                     // Otherwise, it's not a match
-                    _ => Err(ShellError::type_error(
+                    _ => Err(err.error()),
-                        "infix operator",
-                        token.type_name().tagged(token_span),
-                    )),
                 }
             },
         )?;
@@ -1006,7 +1096,7 @@ impl FallibleColorSyntax for InfixShape {
                     RawToken::Operator(operator) if operator != Operator::Dot => Ok(token_span),
 
                     // Otherwise, it's not a match
-                    _ => Err(ShellError::type_error(
+                    _ => Err(ParseError::mismatch(
                         "infix operator",
                         token.type_name().tagged(token_span),
                     )),
@@ -1026,46 +1116,72 @@ impl FallibleColorSyntax for InfixShape {
     }
 }
 
+impl FormatDebug for Spanned<(Span, Spanned<Operator>, Span)> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say_str("operator", self.item.1.span.slice(source))
+    }
+}
+
 impl ExpandSyntax for InfixShape {
-    type Output = (Span, Spanned<Operator>, Span);
+    type Output = Spanned<(Span, Spanned<Operator>, Span)>;
+
+    fn name(&self) -> &'static str {
+        "infix operator"
+    }
 
     fn expand_syntax<'a, 'b>(
         &self,
         token_nodes: &'b mut TokensIterator<'a>,
         context: &ExpandContext,
-    ) -> Result<Self::Output, ShellError> {
+    ) -> Result<Self::Output, ParseError> {
-        let checkpoint = token_nodes.checkpoint();
+        let mut checkpoint = token_nodes.checkpoint();
 
         // An infix operator must be prefixed by whitespace
         let start = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?;
 
         // Parse the next TokenNode after the whitespace
-        let operator = parse_single_node(
+        let operator = expand_syntax(&InfixInnerShape, &mut checkpoint.iterator, context)?;
-            checkpoint.iterator,
-            "infix operator",
-            |token, token_span, _| {
-                Ok(match token {
-                    // If it's an operator (and not `.`), it's a match
-                    RawToken::Operator(operator) if operator != Operator::Dot => {
-                        operator.spanned(token_span)
-                    }
-
-                    // Otherwise, it's not a match
-                    _ => {
-                        return Err(ShellError::type_error(
-                            "infix operator",
-                            token.type_name().tagged(token_span),
-                        ))
-                    }
-                })
-            },
-        )?;
 
         // An infix operator must be followed by whitespace
         let end = expand_syntax(&WhitespaceShape, checkpoint.iterator, context)?;
 
         checkpoint.commit();
 
-        Ok((start, operator, end))
+        Ok((start, operator, end).spanned(start.until(end)))
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct InfixInnerShape;
+
+impl FormatDebug for Spanned<Operator> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say_str("operator", self.span.slice(source))
+    }
+}
+
+impl ExpandSyntax for InfixInnerShape {
+    type Output = Spanned<Operator>;
+
+    fn name(&self) -> &'static str {
+        "infix inner"
+    }
+
+    fn expand_syntax<'a, 'b>(
+        &self,
+        token_nodes: &'b mut TokensIterator<'a>,
+        _context: &ExpandContext,
+    ) -> Result<Self::Output, ParseError> {
+        parse_single_node(token_nodes, "infix operator", |token, token_span, err| {
+            Ok(match token {
+                // If it's an operator (and not `.`), it's a match
+                RawToken::Operator(operator) if operator != Operator::Dot => {
+                    operator.spanned(token_span)
+                }
+
+                // Otherwise, it's not a match
+                _ => return Err(err.error()),
+            })
+        })
     }
 }

src/parser/hir/syntax_shape/flat_shape.rs

@@ -1,5 +1,5 @@
 use crate::parser::{Delimiter, Flag, FlagKind, Operator, RawNumber, RawToken, TokenNode};
-use crate::{Span, Spanned, SpannedItem, Text};
+use crate::{HasSpan, Span, Spanned, SpannedItem, Text};
 
 #[derive(Debug, Copy, Clone)]
 pub enum FlatShape {

src/parser/hir/tokens_iterator.rs

@@ -1,25 +1,38 @@
 pub(crate) mod debug;
 
-use self::debug::Tracer;
+use self::debug::{ColorTracer, ExpandTracer};
 use crate::errors::ShellError;
 #[cfg(coloring_in_tokens)]
 use crate::parser::hir::syntax_shape::FlatShape;
+use crate::parser::hir::Expression;
 use crate::parser::TokenNode;
 use crate::prelude::*;
 use crate::{Span, Spanned, SpannedItem};
 #[allow(unused)]
 use getset::{Getters, MutGetters};
 
-#[derive(Getters, Debug)]
+cfg_if::cfg_if! {
-pub struct TokensIteratorState<'content> {
+    if #[cfg(coloring_in_tokens)] {
-    tokens: &'content [TokenNode],
+        #[derive(Getters, Debug)]
-    span: Span,
+        pub struct TokensIteratorState<'content> {
-    skip_ws: bool,
+            tokens: &'content [TokenNode],
-    index: usize,
+            span: Span,
-    seen: indexmap::IndexSet<usize>,
+            skip_ws: bool,
-    #[cfg(coloring_in_tokens)]
+            index: usize,
-    #[cfg_attr(coloring_in_tokens, get = "pub")]
+            seen: indexmap::IndexSet<usize>,
-    shapes: Vec<Spanned<FlatShape>>,
+            #[get = "pub"]
+            shapes: Vec<Spanned<FlatShape>>,
+        }
+    } else {
+        #[derive(Getters, Debug)]
+        pub struct TokensIteratorState<'content> {
+            tokens: &'content [TokenNode],
+            span: Span,
+            skip_ws: bool,
+            index: usize,
+            seen: indexmap::IndexSet<usize>,
+        }
+    }
 }
 
 #[derive(Getters, MutGetters, Debug)]
@@ -29,7 +42,10 @@ pub struct TokensIterator<'content> {
     state: TokensIteratorState<'content>,
     #[get = "pub"]
     #[get_mut = "pub"]
-    tracer: Tracer,
+    color_tracer: ColorTracer,
+    #[get = "pub"]
+    #[get_mut = "pub"]
+    expand_tracer: ExpandTracer,
 }
 
 #[derive(Debug)]
@@ -83,12 +99,9 @@ impl<'content, 'me> Peeked<'content, 'me> {
         Some(node)
     }
 
-    pub fn not_eof(
+    pub fn not_eof(self, expected: &'static str) -> Result<PeekedNode<'content, 'me>, ParseError> {
-        self,
-        expected: impl Into<String>,
-    ) -> Result<PeekedNode<'content, 'me>, ShellError> {
         match self.node {
-            None => Err(ShellError::unexpected_eof(
+            None => Err(ParseError::unexpected_eof(
                 expected,
                 self.iterator.eof_span(),
             )),
@@ -101,7 +114,7 @@ impl<'content, 'me> Peeked<'content, 'me> {
         }
     }
 
-    pub fn type_error(&self, expected: impl Into<String>) -> ShellError {
+    pub fn type_error(&self, expected: &'static str) -> ParseError {
         peek_error(&self.node, self.iterator.eof_span(), expected)
     }
 }
@@ -129,19 +142,15 @@ impl<'content, 'me> PeekedNode<'content, 'me> {
 
     pub fn rollback(self) {}
 
-    pub fn type_error(&self, expected: impl Into<String>) -> ShellError {
+    pub fn type_error(&self, expected: &'static str) -> ParseError {
         peek_error(&Some(self.node), self.iterator.eof_span(), expected)
     }
 }
 
-pub fn peek_error(
+pub fn peek_error(node: &Option<&TokenNode>, eof_span: Span, expected: &'static str) -> ParseError {
-    node: &Option<&TokenNode>,
-    eof_span: Span,
-    expected: impl Into<String>,
-) -> ShellError {
     match node {
-        None => ShellError::unexpected_eof(expected, eof_span),
+        None => ParseError::unexpected_eof(expected, eof_span),
-        Some(node) => ShellError::type_error(expected, node.tagged_type_name()),
+        Some(node) => ParseError::mismatch(expected, node.tagged_type_name()),
     }
 }
 
@@ -161,7 +170,8 @@ impl<'content> TokensIterator<'content> {
                 #[cfg(coloring_in_tokens)]
                 shapes: vec![],
             },
-            tracer: Tracer::new(),
+            color_tracer: ColorTracer::new(),
+            expand_tracer: ExpandTracer::new(),
         }
     }
 
@@ -188,7 +198,7 @@ impl<'content> TokensIterator<'content> {
 
     #[cfg(coloring_in_tokens)]
     pub fn color_shape(&mut self, shape: Spanned<FlatShape>) {
-        self.with_tracer(|_, tracer| tracer.add_shape(shape));
+        self.with_color_tracer(|_, tracer| tracer.add_shape(shape));
         self.state.shapes.push(shape);
     }
 
@@ -201,7 +211,7 @@ impl<'content> TokensIterator<'content> {
             (len..(shapes.len())).map(|i| shapes[i]).collect()
         };
 
-        self.with_tracer(|_, tracer| {
+        self.with_color_tracer(|_, tracer| {
             for shape in new_shapes {
                 tracer.add_shape(shape)
             }
@@ -233,8 +243,11 @@ impl<'content> TokensIterator<'content> {
         let mut shapes = vec![];
         std::mem::swap(&mut shapes, &mut self.state.shapes);
 
-        let mut tracer = Tracer::new();
+        let mut color_tracer = ColorTracer::new();
-        std::mem::swap(&mut tracer, &mut self.tracer);
+        std::mem::swap(&mut color_tracer, &mut self.color_tracer);
+
+        let mut expand_tracer = ExpandTracer::new();
+        std::mem::swap(&mut expand_tracer, &mut self.expand_tracer);
 
         let mut iterator = TokensIterator {
             state: TokensIteratorState {
@@ -245,13 +258,15 @@ impl<'content> TokensIterator<'content> {
                 seen: indexmap::IndexSet::new(),
                 shapes,
             },
-            tracer,
+            color_tracer,
+            expand_tracer,
         };
 
         let result = block(&mut iterator);
 
         std::mem::swap(&mut iterator.state.shapes, &mut self.state.shapes);
-        std::mem::swap(&mut iterator.tracer, &mut self.tracer);
+        std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer);
+        std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer);
 
         result
     }
@@ -262,8 +277,11 @@ impl<'content> TokensIterator<'content> {
         tokens: Spanned<&'me [TokenNode]>,
         block: impl FnOnce(&mut TokensIterator<'me>) -> T,
     ) -> T {
-        let mut tracer = Tracer::new();
+        let mut color_tracer = ColorTracer::new();
-        std::mem::swap(&mut tracer, &mut self.tracer);
+        std::mem::swap(&mut color_tracer, &mut self.color_tracer);
+
+        let mut expand_tracer = ExpandTracer::new();
+        std::mem::swap(&mut expand_tracer, &mut self.expand_tracer);
 
         let mut iterator = TokensIterator {
             state: TokensIteratorState {
@@ -273,19 +291,34 @@ impl<'content> TokensIterator<'content> {
                 index: 0,
                 seen: indexmap::IndexSet::new(),
             },
-            tracer,
+            color_tracer,
+            expand_tracer,
         };
 
         let result = block(&mut iterator);
 
-        std::mem::swap(&mut iterator.tracer, &mut self.tracer);
+        std::mem::swap(&mut iterator.color_tracer, &mut self.color_tracer);
+        std::mem::swap(&mut iterator.expand_tracer, &mut self.expand_tracer);
 
         result
     }
 
-    pub fn with_tracer(&mut self, block: impl FnOnce(&mut TokensIteratorState, &mut Tracer)) {
+    pub fn with_color_tracer(
+        &mut self,
+        block: impl FnOnce(&mut TokensIteratorState, &mut ColorTracer),
+    ) {
         let state = &mut self.state;
-        let tracer = &mut self.tracer;
+        let color_tracer = &mut self.color_tracer;
+
+        block(state, color_tracer)
+    }
+
+    pub fn with_expand_tracer(
+        &mut self,
+        block: impl FnOnce(&mut TokensIteratorState, &mut ExpandTracer),
+    ) {
+        let state = &mut self.state;
+        let tracer = &mut self.expand_tracer;
 
         block(state, tracer)
     }
@@ -296,32 +329,77 @@ impl<'content> TokensIterator<'content> {
         desc: &'static str,
         block: impl FnOnce(&mut TokensIterator) -> T,
     ) -> T {
-        self.with_tracer(|_, tracer| tracer.start(desc));
+        self.with_color_tracer(|_, tracer| tracer.start(desc));
 
         let result = block(self);
 
-        self.with_tracer(|_, tracer| {
+        self.with_color_tracer(|_, tracer| {
             tracer.success();
         });
 
         result
     }
 
+    pub fn expand_frame<T>(
+        &mut self,
+        desc: &'static str,
+        block: impl FnOnce(&mut TokensIterator) -> Result<T, ParseError>,
+    ) -> Result<T, ParseError>
+    where
+        T: std::fmt::Debug + FormatDebug + Clone + HasFallibleSpan + 'static,
+    {
+        self.with_expand_tracer(|_, tracer| tracer.start(desc));
+
+        let result = block(self);
+
+        self.with_expand_tracer(|_, tracer| match &result {
+            Ok(result) => {
+                tracer.add_result(Box::new(result.clone()));
+                tracer.success();
+            }
+
+            Err(err) => tracer.failed(err),
+        });
+
+        result
+    }
+
+    pub fn expand_expr_frame(
+        &mut self,
+        desc: &'static str,
+        block: impl FnOnce(&mut TokensIterator) -> Result<Expression, ParseError>,
+    ) -> Result<Expression, ParseError> {
+        self.with_expand_tracer(|_, tracer| tracer.start(desc));
+
+        let result = block(self);
+
+        self.with_expand_tracer(|_, tracer| match &result {
+            Ok(expr) => {
+                tracer.add_expr(expr.clone());
+                tracer.success()
+            }
+
+            Err(err) => tracer.failed(err),
+        });
+
+        result
+    }
+
     pub fn color_fallible_frame<T>(
         &mut self,
         desc: &'static str,
         block: impl FnOnce(&mut TokensIterator) -> Result<T, ShellError>,
     ) -> Result<T, ShellError> {
-        self.with_tracer(|_, tracer| tracer.start(desc));
+        self.with_color_tracer(|_, tracer| tracer.start(desc));
 
         if self.at_end() {
-            self.with_tracer(|_, tracer| tracer.eof_frame());
+            self.with_color_tracer(|_, tracer| tracer.eof_frame());
             return Err(ShellError::unexpected_eof("coloring", Tag::unknown()));
         }
 
         let result = block(self);
 
-        self.with_tracer(|_, tracer| match &result {
+        self.with_color_tracer(|_, tracer| match &result {
             Ok(_) => {
                 tracer.success();
             }
@@ -431,10 +509,6 @@ impl<'content> TokensIterator<'content> {
         }
     }
 
-    pub fn whole_span(&self) -> Span {
-        self.state.span
-    }
-
     pub fn span_at_cursor(&mut self) -> Span {
         let next = self.peek_any();
 
@@ -491,27 +565,22 @@ impl<'content> TokensIterator<'content> {
         self.state.index = 0;
     }
 
-    pub fn clone(&self) -> TokensIterator<'content> {
+    // pub fn clone(&self) -> TokensIterator<'content> {
-        let state = &self.state;
+    //     let state = &self.state;
-        TokensIterator {
+    //     TokensIterator {
-            state: TokensIteratorState {
+    //         state: TokensIteratorState {
-                tokens: state.tokens,
+    //             tokens: state.tokens,
-                span: state.span,
+    //             span: state.span,
-                index: state.index,
+    //             index: state.index,
-                seen: state.seen.clone(),
+    //             seen: state.seen.clone(),
-                skip_ws: state.skip_ws,
+    //             skip_ws: state.skip_ws,
-                #[cfg(coloring_in_tokens)]
+    //             #[cfg(coloring_in_tokens)]
-                shapes: state.shapes.clone(),
+    //             shapes: state.shapes.clone(),
-            },
+    //         },
-            tracer: self.tracer.clone(),
+    //         color_tracer: self.color_tracer.clone(),
-        }
+    //         expand_tracer: self.expand_tracer.clone(),
-    }
+    //     }
-
+    // }
-    // Get the next token, not including whitespace
-    pub fn next_non_ws(&mut self) -> Option<&TokenNode> {
-        let mut peeked = start_next(self, true);
-        peeked.commit()
-    }
 
     // Peek the next token, not including whitespace
     pub fn peek_non_ws<'me>(&'me mut self) -> Peeked<'content, 'me> {
@@ -527,8 +596,8 @@ impl<'content> TokensIterator<'content> {
     pub fn peek_any_token<'me, T>(
         &'me mut self,
         expected: &'static str,
-        block: impl FnOnce(&'content TokenNode) -> Result<T, ShellError>,
+        block: impl FnOnce(&'content TokenNode) -> Result<T, ParseError>,
-    ) -> Result<T, ShellError> {
+    ) -> Result<T, ParseError> {
         let peeked = start_next(self, false);
         let peeked = peeked.not_eof(expected);
 
@@ -557,9 +626,11 @@ impl<'content> TokensIterator<'content> {
     }
 
     pub fn debug_remaining(&self) -> Vec<TokenNode> {
-        let mut tokens = self.clone();
+        // TODO: TODO: TODO: Clean up
-        tokens.restart();
+        vec![]
-        tokens.cloned().collect()
+        // let mut tokens = self.clone();
+        // tokens.restart();
+        // tokens.cloned().collect()
     }
 }
 

src/parser/hir/tokens_iterator/debug.rs

@@ -1,13 +1,13 @@
-use crate::errors::ShellError;
+#![allow(unused)]
-use crate::parser::hir::syntax_shape::FlatShape;
+
+pub(crate) mod color_trace;
+pub(crate) mod expand_trace;
+
+pub(crate) use self::color_trace::*;
+pub(crate) use self::expand_trace::*;
+
 use crate::parser::hir::tokens_iterator::TokensIteratorState;
-use crate::prelude::*;
 use crate::traits::ToDebug;
-use ansi_term::Color;
-use log::trace;
-use ptree::*;
-use std::borrow::Cow;
-use std::io;
 
 #[derive(Debug)]
 pub(crate) enum DebugIteratorToken {
@@ -36,344 +36,3 @@ pub(crate) fn debug_tokens(state: &TokensIteratorState, source: &str) -> Vec<Deb
 
     out
 }
-
-#[derive(Debug, Clone)]
-pub enum FrameChild {
-    #[allow(unused)]
-    Shape(Spanned<FlatShape>),
-    Frame(ColorFrame),
-}
-
-impl FrameChild {
-    fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
-        match self {
-            FrameChild::Shape(shape) => write!(
-                f,
-                "{} {:?}",
-                Color::White
-                    .bold()
-                    .on(Color::Green)
-                    .paint(format!("{:?}", shape.item)),
-                shape.span.slice(text)
-            ),
-
-            FrameChild::Frame(frame) => frame.colored_leaf_description(f),
-        }
-    }
-
-    fn into_tree_child(self, text: &Text) -> TreeChild {
-        match self {
-            FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()),
-            FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct ColorFrame {
-    description: &'static str,
-    children: Vec<FrameChild>,
-    error: Option<ShellError>,
-}
-
-impl ColorFrame {
-    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
-        if self.has_only_error_descendents() {
-            if self.children.len() == 0 {
-                write!(
-                    f,
-                    "{}",
-                    Color::White.bold().on(Color::Red).paint(self.description)
-                )
-            } else {
-                write!(f, "{}", Color::Red.normal().paint(self.description))
-            }
-        } else if self.has_descendent_shapes() {
-            write!(f, "{}", Color::Green.normal().paint(self.description))
-        } else {
-            write!(f, "{}", Color::Yellow.bold().paint(self.description))
-        }
-    }
-
-    fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
-        if self.children.len() == 1 {
-            let child = &self.children[0];
-
-            self.colored_leaf_description(f)?;
-            write!(f, " -> ")?;
-            child.colored_leaf_description(text, f)
-        } else {
-            self.colored_leaf_description(f)
-        }
-    }
-
-    fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
-        if self.children.len() == 1 {
-            let child = &self.children[0];
-
-            match child {
-                FrameChild::Shape(_) => vec![],
-                FrameChild::Frame(frame) => frame.tree_children(text),
-            }
-        } else {
-            self.tree_children(text)
-        }
-    }
-
-    fn tree_children(&self, text: &Text) -> Vec<TreeChild> {
-        self.children
-            .clone()
-            .into_iter()
-            .map(|c| c.into_tree_child(text))
-            .collect()
-    }
-
-    #[allow(unused)]
-    fn add_shape(&mut self, shape: Spanned<FlatShape>) {
-        self.children.push(FrameChild::Shape(shape))
-    }
-
-    fn has_child_shapes(&self) -> bool {
-        self.any_child_shape(|_| true)
-    }
-
-    fn any_child_shape(&self, predicate: impl Fn(Spanned<FlatShape>) -> bool) -> bool {
-        for item in &self.children {
-            match item {
-                FrameChild::Shape(shape) => {
-                    if predicate(*shape) {
-                        return true;
-                    }
-                }
-
-                _ => {}
-            }
-        }
-
-        false
-    }
-
-    fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool {
-        for item in &self.children {
-            match item {
-                FrameChild::Frame(frame) => {
-                    if predicate(frame) {
-                        return true;
-                    }
-                }
-
-                _ => {}
-            }
-        }
-
-        false
-    }
-
-    fn has_descendent_shapes(&self) -> bool {
-        if self.has_child_shapes() {
-            true
-        } else {
-            self.any_child_frame(|frame| frame.has_descendent_shapes())
-        }
-    }
-
-    fn has_only_error_descendents(&self) -> bool {
-        if self.children.len() == 0 {
-            // if this frame has no children at all, it has only error descendents if this frame
-            // is an error
-            self.error.is_some()
-        } else {
-            // otherwise, it has only error descendents if all of its children terminate in an
-            // error (transitively)
-
-            let mut seen_error = false;
-
-            for child in &self.children {
-                match child {
-                    // if this frame has at least one child shape, this frame has non-error descendents
-                    FrameChild::Shape(_) => return false,
-                    FrameChild::Frame(frame) => {
-                        // if the chi
-                        if frame.has_only_error_descendents() {
-                            seen_error = true;
-                        } else {
-                            return false;
-                        }
-                    }
-                }
-            }
-
-            seen_error
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub enum TreeChild {
-    Shape(Spanned<FlatShape>, Text),
-    Frame(ColorFrame, Text),
-}
-
-impl TreeChild {
-    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
-        match self {
-            TreeChild::Shape(shape, text) => write!(
-                f,
-                "{} {:?}",
-                Color::White
-                    .bold()
-                    .on(Color::Green)
-                    .paint(format!("{:?}", shape.item)),
-                shape.span.slice(text)
-            ),
-
-            TreeChild::Frame(frame, _) => frame.colored_leaf_description(f),
-        }
-    }
-}
-
-impl TreeItem for TreeChild {
-    type Child = TreeChild;
-
-    fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
-        match self {
-            shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f),
-
-            TreeChild::Frame(frame, text) => frame.colored_description(text, f),
-        }
-    }
-
-    fn children(&self) -> Cow<[Self::Child]> {
-        match self {
-            TreeChild::Shape(..) => Cow::Borrowed(&[]),
-            TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct Tracer {
-    frame_stack: Vec<ColorFrame>,
-}
-
-impl Tracer {
-    pub fn print(self, source: Text) -> PrintTracer {
-        PrintTracer {
-            tracer: self,
-            source,
-        }
-    }
-
-    pub fn new() -> Tracer {
-        let root = ColorFrame {
-            description: "Trace",
-            children: vec![],
-            error: None,
-        };
-
-        Tracer {
-            frame_stack: vec![root],
-        }
-    }
-
-    fn current_frame(&mut self) -> &mut ColorFrame {
-        let frames = &mut self.frame_stack;
-        let last = frames.len() - 1;
-        &mut frames[last]
-    }
-
-    fn pop_frame(&mut self) -> ColorFrame {
-        let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
-
-        if self.frame_stack.len() == 0 {
-            panic!("Can't pop root tracer frame");
-        }
-
-        self.debug();
-
-        result
-    }
-
-    pub fn start(&mut self, description: &'static str) {
-        let frame = ColorFrame {
-            description,
-            children: vec![],
-            error: None,
-        };
-
-        self.frame_stack.push(frame);
-        self.debug();
-    }
-
-    pub fn eof_frame(&mut self) {
-        let current = self.pop_frame();
-        self.current_frame()
-            .children
-            .push(FrameChild::Frame(current));
-    }
-
-    #[allow(unused)]
-    pub fn finish(&mut self) {
-        loop {
-            if self.frame_stack.len() == 1 {
-                break;
-            }
-
-            let frame = self.pop_frame();
-            self.current_frame().children.push(FrameChild::Frame(frame));
-        }
-    }
-
-    #[allow(unused)]
-    pub fn add_shape(&mut self, shape: Spanned<FlatShape>) {
-        self.current_frame().add_shape(shape);
-    }
-
-    pub fn success(&mut self) {
-        let current = self.pop_frame();
-        self.current_frame()
-            .children
-            .push(FrameChild::Frame(current));
-    }
-
-    pub fn failed(&mut self, error: &ShellError) {
-        let mut current = self.pop_frame();
-        current.error = Some(error.clone());
-        self.current_frame()
-            .children
-            .push(FrameChild::Frame(current));
-    }
-
-    fn debug(&self) {
-        trace!(target: "nu::color_syntax",
-            "frames = {:?}",
-            self.frame_stack
-                .iter()
-                .map(|f| f.description)
-                .collect::<Vec<_>>()
-        );
-
-        trace!(target: "nu::color_syntax", "{:#?}", self);
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct PrintTracer {
-    tracer: Tracer,
-    source: Text,
-}
-
-impl TreeItem for PrintTracer {
-    type Child = TreeChild;
-
-    fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
-        write!(f, "{}", style.paint("Color Trace"))
-    }
-
-    fn children(&self) -> Cow<[Self::Child]> {
-        Cow::Owned(vec![TreeChild::Frame(
-            self.tracer.frame_stack[0].clone(),
-            self.source.clone(),
-        )])
-    }
-}

src/parser/hir/tokens_iterator/debug/color_trace.rs

@@ -0,0 +1,351 @@
+use crate::errors::ShellError;
+use crate::parser::hir::syntax_shape::FlatShape;
+use crate::prelude::*;
+use ansi_term::Color;
+use log::trace;
+use ptree::*;
+use std::borrow::Cow;
+use std::io;
+
+#[derive(Debug, Clone)]
+pub enum FrameChild {
+    #[allow(unused)]
+    Shape(Spanned<FlatShape>),
+    Frame(ColorFrame),
+}
+
+impl FrameChild {
+    fn colored_leaf_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            FrameChild::Shape(shape) => write!(
+                f,
+                "{} {:?}",
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{:?}", shape.item)),
+                shape.span.slice(text)
+            ),
+
+            FrameChild::Frame(frame) => frame.colored_leaf_description(f),
+        }
+    }
+
+    fn into_tree_child(self, text: &Text) -> TreeChild {
+        match self {
+            FrameChild::Shape(shape) => TreeChild::Shape(shape, text.clone()),
+            FrameChild::Frame(frame) => TreeChild::Frame(frame, text.clone()),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ColorFrame {
+    description: &'static str,
+    children: Vec<FrameChild>,
+    error: Option<ShellError>,
+}
+
+impl ColorFrame {
+    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        if self.has_only_error_descendents() {
+            if self.children.len() == 0 {
+                write!(
+                    f,
+                    "{}",
+                    Color::White.bold().on(Color::Red).paint(self.description)
+                )
+            } else {
+                write!(f, "{}", Color::Red.normal().paint(self.description))
+            }
+        } else if self.has_descendent_shapes() {
+            write!(f, "{}", Color::Green.normal().paint(self.description))
+        } else {
+            write!(f, "{}", Color::Yellow.bold().paint(self.description))
+        }
+    }
+
+    fn colored_description(&self, text: &Text, f: &mut impl io::Write) -> io::Result<()> {
+        if self.children.len() == 1 {
+            let child = &self.children[0];
+
+            self.colored_leaf_description(f)?;
+            write!(f, " -> ")?;
+            child.colored_leaf_description(text, f)
+        } else {
+            self.colored_leaf_description(f)
+        }
+    }
+
+    fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
+        if self.children.len() == 1 {
+            let child = &self.children[0];
+
+            match child {
+                FrameChild::Shape(_) => vec![],
+                FrameChild::Frame(frame) => frame.tree_children(text),
+            }
+        } else {
+            self.tree_children(text)
+        }
+    }
+
+    fn tree_children(&self, text: &Text) -> Vec<TreeChild> {
+        self.children
+            .clone()
+            .into_iter()
+            .map(|c| c.into_tree_child(text))
+            .collect()
+    }
+
+    #[allow(unused)]
+    fn add_shape(&mut self, shape: Spanned<FlatShape>) {
+        self.children.push(FrameChild::Shape(shape))
+    }
+
+    fn has_child_shapes(&self) -> bool {
+        self.any_child_shape(|_| true)
+    }
+
+    fn any_child_shape(&self, predicate: impl Fn(Spanned<FlatShape>) -> bool) -> bool {
+        for item in &self.children {
+            match item {
+                FrameChild::Shape(shape) => {
+                    if predicate(*shape) {
+                        return true;
+                    }
+                }
+
+                _ => {}
+            }
+        }
+
+        false
+    }
+
+    fn any_child_frame(&self, predicate: impl Fn(&ColorFrame) -> bool) -> bool {
+        for item in &self.children {
+            match item {
+                FrameChild::Frame(frame) => {
+                    if predicate(frame) {
+                        return true;
+                    }
+                }
+
+                _ => {}
+            }
+        }
+
+        false
+    }
+
+    fn has_descendent_shapes(&self) -> bool {
+        if self.has_child_shapes() {
+            true
+        } else {
+            self.any_child_frame(|frame| frame.has_descendent_shapes())
+        }
+    }
+
+    fn has_only_error_descendents(&self) -> bool {
+        if self.children.len() == 0 {
+            // if this frame has no children at all, it has only error descendents if this frame
+            // is an error
+            self.error.is_some()
+        } else {
+            // otherwise, it has only error descendents if all of its children terminate in an
+            // error (transitively)
+
+            let mut seen_error = false;
+
+            for child in &self.children {
+                match child {
+                    // if this frame has at least one child shape, this frame has non-error descendents
+                    FrameChild::Shape(_) => return false,
+                    FrameChild::Frame(frame) => {
+                        // if the chi
+                        if frame.has_only_error_descendents() {
+                            seen_error = true;
+                        } else {
+                            return false;
+                        }
+                    }
+                }
+            }
+
+            seen_error
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum TreeChild {
+    Shape(Spanned<FlatShape>, Text),
+    Frame(ColorFrame, Text),
+}
+
+impl TreeChild {
+    fn colored_leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            TreeChild::Shape(shape, text) => write!(
+                f,
+                "{} {:?}",
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{:?}", shape.item)),
+                shape.span.slice(text)
+            ),
+
+            TreeChild::Frame(frame, _) => frame.colored_leaf_description(f),
+        }
+    }
+}
+
+impl TreeItem for TreeChild {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
+        match self {
+            shape @ TreeChild::Shape(..) => shape.colored_leaf_description(f),
+
+            TreeChild::Frame(frame, text) => frame.colored_description(text, f),
+        }
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        match self {
+            TreeChild::Shape(..) => Cow::Borrowed(&[]),
+            TreeChild::Frame(frame, text) => Cow::Owned(frame.children_for_formatting(text)),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ColorTracer {
+    frame_stack: Vec<ColorFrame>,
+}
+
+impl ColorTracer {
+    pub fn print(self, source: Text) -> PrintTracer {
+        PrintTracer {
+            tracer: self,
+            source,
+        }
+    }
+
+    pub fn new() -> ColorTracer {
+        let root = ColorFrame {
+            description: "Trace",
+            children: vec![],
+            error: None,
+        };
+
+        ColorTracer {
+            frame_stack: vec![root],
+        }
+    }
+
+    fn current_frame(&mut self) -> &mut ColorFrame {
+        let frames = &mut self.frame_stack;
+        let last = frames.len() - 1;
+        &mut frames[last]
+    }
+
+    fn pop_frame(&mut self) -> ColorFrame {
+        trace!(target: "nu::color_syntax", "Popping {:#?}", self);
+
+        let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
+
+        if self.frame_stack.len() == 0 {
+            panic!("Can't pop root tracer frame {:#?}", self);
+        }
+
+        self.debug();
+
+        result
+    }
+
+    pub fn start(&mut self, description: &'static str) {
+        let frame = ColorFrame {
+            description,
+            children: vec![],
+            error: None,
+        };
+
+        self.frame_stack.push(frame);
+        self.debug();
+    }
+
+    pub fn eof_frame(&mut self) {
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    #[allow(unused)]
+    pub fn finish(&mut self) {
+        loop {
+            if self.frame_stack.len() == 1 {
+                break;
+            }
+
+            let frame = self.pop_frame();
+            self.current_frame().children.push(FrameChild::Frame(frame));
+        }
+    }
+
+    #[allow(unused)]
+    pub fn add_shape(&mut self, shape: Spanned<FlatShape>) {
+        self.current_frame().add_shape(shape);
+    }
+
+    pub fn success(&mut self) {
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    pub fn failed(&mut self, error: &ShellError) {
+        let mut current = self.pop_frame();
+        current.error = Some(error.clone());
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    fn debug(&self) {
+        trace!(target: "nu::color_syntax",
+            "frames = {:?}",
+            self.frame_stack
+                .iter()
+                .map(|f| f.description)
+                .collect::<Vec<_>>()
+        );
+
+        trace!(target: "nu::color_syntax", "{:#?}", self);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PrintTracer {
+    tracer: ColorTracer,
+    source: Text,
+}
+
+impl TreeItem for PrintTracer {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
+        write!(f, "{}", style.paint("Color Trace"))
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        Cow::Owned(vec![TreeChild::Frame(
+            self.tracer.frame_stack[0].clone(),
+            self.source.clone(),
+        )])
+    }
+}

src/parser/hir/tokens_iterator/debug/expand_trace.rs

@@ -0,0 +1,365 @@
+use crate::parser::hir::Expression;
+use crate::prelude::*;
+use ansi_term::Color;
+use log::trace;
+use ptree::*;
+use std::borrow::Cow;
+use std::io;
+
+#[derive(Debug)]
+pub enum FrameChild {
+    Expr(Expression),
+    Frame(ExprFrame),
+    Result(Box<dyn FormatDebug>),
+}
+
+impl FrameChild {
+    fn get_error_leaf(&self) -> Option<&'static str> {
+        match self {
+            FrameChild::Frame(frame) if frame.error.is_some() => {
+                if frame.children.len() == 0 {
+                    Some(frame.description)
+                } else {
+                    None
+                }
+            }
+            _ => None,
+        }
+    }
+
+    fn to_tree_child(&self, text: &Text) -> TreeChild {
+        match self {
+            FrameChild::Expr(expr) => TreeChild::OkExpr(expr.clone(), text.clone()),
+            FrameChild::Result(result) => {
+                let result = format!("{}", result.debug(text));
+                TreeChild::OkNonExpr(result)
+            }
+            FrameChild::Frame(frame) => {
+                if frame.error.is_some() {
+                    if frame.children.len() == 0 {
+                        TreeChild::ErrorLeaf(vec![frame.description])
+                    } else {
+                        TreeChild::ErrorFrame(frame.to_tree_frame(text), text.clone())
+                    }
+                } else {
+                    TreeChild::OkFrame(frame.to_tree_frame(text), text.clone())
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ExprFrame {
+    description: &'static str,
+    children: Vec<FrameChild>,
+    error: Option<ParseError>,
+}
+
+impl ExprFrame {
+    fn to_tree_frame(&self, text: &Text) -> TreeFrame {
+        let mut children = vec![];
+        let mut errors = vec![];
+
+        for child in &self.children {
+            if let Some(error_leaf) = child.get_error_leaf() {
+                errors.push(error_leaf);
+                continue;
+            } else if errors.len() > 0 {
+                children.push(TreeChild::ErrorLeaf(errors));
+                errors = vec![];
+            }
+
+            children.push(child.to_tree_child(text));
+        }
+
+        if errors.len() > 0 {
+            children.push(TreeChild::ErrorLeaf(errors));
+        }
+
+        TreeFrame {
+            description: self.description,
+            children,
+            error: self.error.clone(),
+        }
+    }
+
+    fn add_expr(&mut self, expr: Expression) {
+        self.children.push(FrameChild::Expr(expr))
+    }
+
+    fn add_result(&mut self, result: Box<dyn FormatDebug>) {
+        self.children.push(FrameChild::Result(result))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TreeFrame {
+    description: &'static str,
+    children: Vec<TreeChild>,
+    error: Option<ParseError>,
+}
+
+impl TreeFrame {
+    fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        if self.children.len() == 1 {
+            if self.error.is_some() {
+                write!(f, "{}", Color::Red.normal().paint(self.description))?;
+            } else if self.has_descendent_green() {
+                write!(f, "{}", Color::Green.normal().paint(self.description))?;
+            } else {
+                write!(f, "{}", Color::Yellow.bold().paint(self.description))?;
+            }
+
+            write!(f, " -> ")?;
+            self.children[0].leaf_description(f)
+        } else {
+            if self.error.is_some() {
+                if self.children.len() == 0 {
+                    write!(
+                        f,
+                        "{}",
+                        Color::White.bold().on(Color::Red).paint(self.description)
+                    )
+                } else {
+                    write!(f, "{}", Color::Red.normal().paint(self.description))
+                }
+            } else if self.has_descendent_green() {
+                write!(f, "{}", Color::Green.normal().paint(self.description))
+            } else {
+                write!(f, "{}", Color::Yellow.bold().paint(self.description))
+            }
+        }
+    }
+
+    fn has_child_green(&self) -> bool {
+        self.children.iter().any(|item| match item {
+            TreeChild::OkFrame(..) | TreeChild::ErrorFrame(..) | TreeChild::ErrorLeaf(..) => false,
+            TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) => true,
+        })
+    }
+
+    fn any_child_frame(&self, predicate: impl Fn(&TreeFrame) -> bool) -> bool {
+        for item in &self.children {
+            match item {
+                TreeChild::OkFrame(frame, ..) => {
+                    if predicate(frame) {
+                        return true;
+                    }
+                }
+
+                _ => {}
+            }
+        }
+
+        false
+    }
+
+    fn has_descendent_green(&self) -> bool {
+        if self.has_child_green() {
+            true
+        } else {
+            self.any_child_frame(|frame| frame.has_child_green())
+        }
+    }
+
+    fn children_for_formatting(&self, text: &Text) -> Vec<TreeChild> {
+        if self.children.len() == 1 {
+            let child: &TreeChild = &self.children[0];
+            match child {
+                TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => {
+                    vec![]
+                }
+                TreeChild::OkFrame(frame, _) | TreeChild::ErrorFrame(frame, _) => {
+                    frame.children_for_formatting(text)
+                }
+            }
+        } else {
+            self.children.clone()
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum TreeChild {
+    OkNonExpr(String),
+    OkExpr(Expression, Text),
+    OkFrame(TreeFrame, Text),
+    ErrorFrame(TreeFrame, Text),
+    ErrorLeaf(Vec<&'static str>),
+}
+
+impl TreeChild {
+    fn leaf_description(&self, f: &mut impl io::Write) -> io::Result<()> {
+        match self {
+            TreeChild::OkExpr(expr, text) => write!(
+                f,
+                "{} {} {}",
+                Color::Cyan.normal().paint("returns"),
+                Color::White.bold().on(Color::Green).paint(expr.type_name()),
+                expr.span.slice(text)
+            ),
+
+            TreeChild::OkNonExpr(result) => write!(
+                f,
+                "{} {}",
+                Color::Cyan.normal().paint("returns"),
+                Color::White
+                    .bold()
+                    .on(Color::Green)
+                    .paint(format!("{}", result))
+            ),
+
+            TreeChild::ErrorLeaf(desc) => {
+                let last = desc.len() - 1;
+
+                for (i, item) in desc.iter().enumerate() {
+                    write!(f, "{}", Color::White.bold().on(Color::Red).paint(*item))?;
+
+                    if i != last {
+                        write!(f, "{}", Color::White.normal().paint(", "))?;
+                    }
+                }
+
+                Ok(())
+            }
+
+            TreeChild::ErrorFrame(frame, _) | TreeChild::OkFrame(frame, _) => {
+                frame.leaf_description(f)
+            }
+        }
+    }
+}
+
+impl TreeItem for TreeChild {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, _style: &Style) -> io::Result<()> {
+        self.leaf_description(f)
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        match self {
+            TreeChild::OkExpr(..) | TreeChild::OkNonExpr(..) | TreeChild::ErrorLeaf(..) => {
+                Cow::Borrowed(&[])
+            }
+            TreeChild::OkFrame(frame, text) | TreeChild::ErrorFrame(frame, text) => {
+                Cow::Owned(frame.children_for_formatting(text))
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ExpandTracer {
+    frame_stack: Vec<ExprFrame>,
+}
+
+impl ExpandTracer {
+    pub fn print(&self, source: Text) -> PrintTracer {
+        let root = self
+            .frame_stack
+            .iter()
+            .nth(0)
+            .unwrap()
+            .to_tree_frame(&source);
+
+        PrintTracer { root, source }
+    }
+
+    pub fn new() -> ExpandTracer {
+        let root = ExprFrame {
+            description: "Trace",
+            children: vec![],
+            error: None,
+        };
+
+        ExpandTracer {
+            frame_stack: vec![root],
+        }
+    }
+
+    fn current_frame(&mut self) -> &mut ExprFrame {
+        let frames = &mut self.frame_stack;
+        let last = frames.len() - 1;
+        &mut frames[last]
+    }
+
+    fn pop_frame(&mut self) -> ExprFrame {
+        let result = self.frame_stack.pop().expect("Can't pop root tracer frame");
+
+        if self.frame_stack.len() == 0 {
+            panic!("Can't pop root tracer frame");
+        }
+
+        self.debug();
+
+        result
+    }
+
+    pub fn start(&mut self, description: &'static str) {
+        let frame = ExprFrame {
+            description,
+            children: vec![],
+            error: None,
+        };
+
+        self.frame_stack.push(frame);
+        self.debug();
+    }
+
+    pub fn add_expr(&mut self, shape: Expression) {
+        self.current_frame().add_expr(shape);
+    }
+
+    pub fn add_result(&mut self, result: Box<dyn FormatDebug>) {
+        self.current_frame().add_result(result);
+    }
+
+    pub fn success(&mut self) {
+        trace!(target: "parser::expand_syntax", "success {:#?}", self);
+
+        let current = self.pop_frame();
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    pub fn failed(&mut self, error: &ParseError) {
+        let mut current = self.pop_frame();
+        current.error = Some(error.clone());
+        self.current_frame()
+            .children
+            .push(FrameChild::Frame(current));
+    }
+
+    fn debug(&self) {
+        trace!(target: "nu::parser::expand",
+            "frames = {:?}",
+            self.frame_stack
+                .iter()
+                .map(|f| f.description)
+                .collect::<Vec<_>>()
+        );
+
+        trace!(target: "nu::parser::expand", "{:#?}", self);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct PrintTracer {
+    root: TreeFrame,
+    source: Text,
+}
+
+impl TreeItem for PrintTracer {
+    type Child = TreeChild;
+
+    fn write_self<W: io::Write>(&self, f: &mut W, style: &Style) -> io::Result<()> {
+        write!(f, "{}", style.paint("Expansion Trace"))
+    }
+
+    fn children(&self) -> Cow<[Self::Child]> {
+        Cow::Borrowed(&self.root.children)
+    }
+}

src/parser/parse/call_node.rs

@@ -1,7 +1,7 @@
 use crate::parser::TokenNode;
-use crate::traits::ToDebug;
+use crate::traits::{DebugFormatter, FormatDebug, ToDebug};
 use getset::Getters;
-use std::fmt;
+use std::fmt::{self, Write};
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
 pub struct CallNode {
@@ -27,8 +27,8 @@ impl CallNode {
     }
 }
 
-impl ToDebug for CallNode {
+impl FormatDebug for CallNode {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "{}", self.head.debug(source))?;
 
         if let Some(children) = &self.children {

src/parser/parse/operator.rs

@@ -14,8 +14,8 @@ pub enum Operator {
     Dot,
 }
 
-impl ToDebug for Operator {
+impl FormatDebug for Operator {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
         write!(f, "{}", self.as_str())
     }
 }

src/parser/parse/pipeline.rs

@@ -1,24 +1,22 @@
 use crate::parser::TokenNode;
-use crate::traits::ToDebug;
+use crate::{DebugFormatter, FormatDebug, Span, Spanned, ToDebug};
-use crate::{Span, Spanned};
 use derive_new::new;
 use getset::Getters;
-use std::fmt;
+use itertools::Itertools;
+use std::fmt::{self, Write};
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Getters, new)]
 pub struct Pipeline {
     #[get = "pub"]
     pub(crate) parts: Vec<Spanned<PipelineElement>>,
-    // pub(crate) post_ws: Option<Tag>,
 }
 
-impl ToDebug for Pipeline {
+impl FormatDebug for Pipeline {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
-        for part in self.parts.iter() {
+        f.say_str(
-            write!(f, "{}", part.debug(source))?;
+            "pipeline",
-        }
+            self.parts.iter().map(|p| p.debug(source)).join(" "),
-
+        )
-        Ok(())
     }
 }
 
@@ -29,8 +27,8 @@ pub struct PipelineElement {
     pub tokens: Spanned<Vec<TokenNode>>,
 }
 
-impl ToDebug for PipelineElement {
+impl FormatDebug for PipelineElement {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         if let Some(pipe) = self.pipe {
             write!(f, "{}", pipe.slice(source))?;
         }

src/parser/parse/token_tree.rs

@@ -1,4 +1,4 @@
-use crate::errors::ShellError;
+use crate::errors::{ParseError, ShellError};
 use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
 use crate::prelude::*;
 use crate::traits::ToDebug;
@@ -21,8 +21,14 @@ pub enum TokenNode {
     Error(Spanned<ShellError>),
 }
 
-impl ToDebug for TokenNode {
+impl HasSpan for TokenNode {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+    fn span(&self) -> Span {
+        self.get_span()
+    }
+}
+
+impl FormatDebug for TokenNode {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         write!(f, "{:?}", self.old_debug(&Text::from(source)))
     }
 }
@@ -84,12 +90,12 @@ impl fmt::Debug for DebugTokenNode<'_> {
 
 impl From<&TokenNode> for Span {
     fn from(token: &TokenNode) -> Span {
-        token.span()
+        token.get_span()
     }
 }
 
 impl TokenNode {
-    pub fn span(&self) -> Span {
+    pub fn get_span(&self) -> Span {
         match self {
             TokenNode::Token(t) => t.span,
             TokenNode::Nodes(t) => t.span,
@@ -231,10 +237,10 @@ impl TokenNode {
         }
     }
 
-    pub fn as_pipeline(&self) -> Result<Pipeline, ShellError> {
+    pub fn as_pipeline(&self) -> Result<Pipeline, ParseError> {
         match self {
             TokenNode::Pipeline(Spanned { item, .. }) => Ok(item.clone()),
-            _ => Err(ShellError::type_error("pipeline", self.tagged_type_name())),
+            other => Err(ParseError::mismatch("pipeline", other.tagged_type_name())),
         }
     }
 
@@ -321,9 +327,9 @@ impl TokenNode {
         }
     }
 
-    pub fn expect_list(&self) -> &[TokenNode] {
+    pub fn expect_list(&self) -> Spanned<&[TokenNode]> {
         match self {
-            TokenNode::Nodes(token_nodes) => &token_nodes[..],
+            TokenNode::Nodes(token_nodes) => token_nodes[..].spanned(token_nodes.span),
             other => panic!("Expected list, found {:?}", other),
         }
     }

src/parser/parse/tokens.rs

@@ -23,8 +23,8 @@ impl RawToken {
             RawToken::Operator(..) => "operator",
             RawToken::String(_) => "string",
             RawToken::Variable(_) => "variable",
-            RawToken::ExternalCommand(_) => "external command",
+            RawToken::ExternalCommand(_) => "syntax error",
-            RawToken::ExternalWord => "external word",
+            RawToken::ExternalWord => "syntax error",
             RawToken::GlobPattern => "glob pattern",
             RawToken::Bare => "string",
         }
@@ -37,6 +37,15 @@ pub enum RawNumber {
     Decimal(Span),
 }
 
+impl FormatDebug for RawNumber {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            RawNumber::Int(span) => f.say_str("int", span.slice(source)),
+            RawNumber::Decimal(span) => f.say_str("decimal", span.slice(source)),
+        }
+    }
+}
+
 impl RawNumber {
     pub fn int(span: impl Into<Span>) -> Spanned<RawNumber> {
         let span = span.into();

src/parser/parse/unit.rs

@@ -1,6 +1,7 @@
 use crate::data::base::Value;
 use crate::prelude::*;
 use serde::{Deserialize, Serialize};
+use std::fmt;
 use std::str::FromStr;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)]
@@ -13,6 +14,12 @@ pub enum Unit {
     PB,
 }
 
+impl FormatDebug for Spanned<Unit> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "{}", self.span.slice(source))
+    }
+}
+
 impl Unit {
     pub fn as_str(&self) -> &str {
         match *self {

src/parser/parse_command.rs

@@ -1,4 +1,4 @@
-use crate::errors::{ArgumentError, ShellError};
+use crate::errors::{ArgumentError, ParseError};
 use crate::parser::hir::syntax_shape::{
     color_fallible_syntax, color_syntax, expand_expr, flat_shape::FlatShape, spaced,
     BackoffColoringMode, ColorSyntax, MaybeSpaceShape,
@@ -18,9 +18,9 @@ pub fn parse_command_tail(
     context: &ExpandContext,
     tail: &mut TokensIterator,
     command_span: Span,
-) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ShellError> {
+) -> Result<Option<(Option<Vec<hir::Expression>>, Option<NamedArguments>)>, ParseError> {
     let mut named = NamedArguments::new();
-    trace_remaining("nodes", tail.clone(), context.source());
+    trace_remaining("nodes", &tail, context.source());
 
     for (name, kind) in &config.named {
         trace!(target: "nu::parse", "looking for {} : {:?}", name, kind);
@@ -38,7 +38,7 @@ pub fn parse_command_tail(
                         tail.move_to(pos);
 
                         if tail.at_end() {
-                            return Err(ShellError::argument_error(
+                            return Err(ParseError::argument_error(
                                 config.name.clone(),
                                 ArgumentError::MissingValueForName(name.to_string()),
                                 flag.span,
@@ -59,7 +59,7 @@ pub fn parse_command_tail(
                         tail.move_to(pos);
 
                         if tail.at_end() {
-                            return Err(ShellError::argument_error(
+                            return Err(ParseError::argument_error(
                                 config.name.clone(),
                                 ArgumentError::MissingValueForName(name.to_string()),
                                 flag.span,
@@ -85,7 +85,7 @@ pub fn parse_command_tail(
         };
     }
 
-    trace_remaining("after named", tail.clone(), context.source());
+    trace_remaining("after named", &tail, context.source());
 
     let mut positional = vec![];
 
@@ -95,7 +95,7 @@ pub fn parse_command_tail(
         match &arg.0 {
             PositionalType::Mandatory(..) => {
                 if tail.at_end_possible_ws() {
-                    return Err(ShellError::argument_error(
+                    return Err(ParseError::argument_error(
                         config.name.clone(),
                         ArgumentError::MissingMandatoryPositional(arg.0.name().to_string()),
                         Tag {
@@ -118,7 +118,7 @@ pub fn parse_command_tail(
         positional.push(result);
     }
 
-    trace_remaining("after positional", tail.clone(), context.source());
+    trace_remaining("after positional", &tail, context.source());
 
     if let Some((syntax_type, _)) = config.rest_positional {
         let mut out = vec![];
@@ -136,7 +136,7 @@ pub fn parse_command_tail(
         positional.extend(out);
     }
 
-    trace_remaining("after rest", tail.clone(), context.source());
+    trace_remaining("after rest", &tail, context.source());
 
     trace!(target: "nu::parse", "Constructed positional={:?} named={:?}", positional, named);
 
@@ -202,8 +202,6 @@ impl ColorSyntax for CommandTailShape {
         shapes: &mut Vec<Spanned<FlatShape>>,
     ) -> Self::Info {
         let mut args = ColoringArgs::new(token_nodes.len());
-        trace_remaining("nodes", token_nodes.clone(), context.source());
-
         for (name, kind) in &signature.named {
             trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind);
 
@@ -295,8 +293,6 @@ impl ColorSyntax for CommandTailShape {
             };
         }
 
-        trace_remaining("after named", token_nodes.clone(), context.source());
-
         for arg in &signature.positional {
             trace!("Processing positional {:?}", arg);
 
@@ -341,8 +337,6 @@ impl ColorSyntax for CommandTailShape {
             }
         }
 
-        trace_remaining("after positional", token_nodes.clone(), context.source());
-
         if let Some((syntax_type, _)) = signature.rest_positional {
             loop {
                 if token_nodes.at_end_possible_ws() {
@@ -402,7 +396,7 @@ impl ColorSyntax for CommandTailShape {
         context: &ExpandContext,
     ) -> Self::Info {
         let mut args = ColoringArgs::new(token_nodes.len());
-        trace_remaining("nodes", token_nodes.clone(), context.source());
+        trace_remaining("nodes", &token_nodes, context.source());
 
         for (name, kind) in &signature.named {
             trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind);
@@ -497,7 +491,7 @@ impl ColorSyntax for CommandTailShape {
             };
         }
 
-        trace_remaining("after named", token_nodes.clone(), context.source());
+        trace_remaining("after named", &token_nodes, context.source());
 
         for arg in &signature.positional {
             trace!("Processing positional {:?}", arg);
@@ -537,7 +531,7 @@ impl ColorSyntax for CommandTailShape {
             }
         }
 
-        trace_remaining("after positional", token_nodes.clone(), context.source());
+        trace_remaining("after positional", &token_nodes, context.source());
 
         if let Some((syntax_type, _)) = signature.rest_positional {
             loop {
@@ -594,11 +588,11 @@ fn extract_mandatory(
     tokens: &mut hir::TokensIterator<'_>,
     source: &Text,
     span: Span,
-) -> Result<(usize, Spanned<Flag>), ShellError> {
+) -> Result<(usize, Spanned<Flag>), ParseError> {
     let flag = tokens.extract(|t| t.as_flag(name, source));
 
     match flag {
-        None => Err(ShellError::argument_error(
+        None => Err(ParseError::argument_error(
             config.name.clone(),
             ArgumentError::MissingMandatoryFlag(name.to_string()),
             span,
@@ -615,7 +609,7 @@ fn extract_optional(
     name: &str,
     tokens: &mut hir::TokensIterator<'_>,
     source: &Text,
-) -> Result<(Option<(usize, Spanned<Flag>)>), ShellError> {
+) -> Result<(Option<(usize, Spanned<Flag>)>), ParseError> {
     let flag = tokens.extract(|t| t.as_flag(name, source));
 
     match flag {
@@ -627,7 +621,7 @@ fn extract_optional(
     }
 }
 
-pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source: &Text) {
+pub fn trace_remaining(desc: &'static str, tail: &hir::TokensIterator<'_>, source: &Text) {
     trace!(
         target: "nu::parse",
         "{} = {:?}",

src/prelude.rs

@@ -67,13 +67,14 @@ pub(crate) use crate::context::CommandRegistry;
 pub(crate) use crate::context::{AnchorLocation, Context};
 pub(crate) use crate::data::base as value;
 pub(crate) use crate::data::meta::{
-    tag_for_tagged_list, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem,
+    tag_for_tagged_list, HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Tagged,
+    TaggedItem,
 };
 pub(crate) use crate::data::types::ExtractType;
 pub(crate) use crate::data::{Primitive, Value};
 pub(crate) use crate::env::host::handle_unexpected;
 pub(crate) use crate::env::Host;
-pub(crate) use crate::errors::{CoerceInto, ShellError};
+pub(crate) use crate::errors::{CoerceInto, ParseError, ShellError};
 pub(crate) use crate::parser::hir::SyntaxShape;
 pub(crate) use crate::parser::parse::parser::Number;
 pub(crate) use crate::parser::registry::Signature;
@@ -82,7 +83,7 @@ pub(crate) use crate::shell::help_shell::HelpShell;
 pub(crate) use crate::shell::shell_manager::ShellManager;
 pub(crate) use crate::shell::value_shell::ValueShell;
 pub(crate) use crate::stream::{InputStream, OutputStream};
-pub(crate) use crate::traits::{HasTag, ToDebug};
+pub(crate) use crate::traits::{DebugFormatter, FormatDebug, HasTag, ToDebug};
 pub(crate) use crate::Text;
 pub(crate) use async_stream::stream as async_stream;
 pub(crate) use bigdecimal::BigDecimal;
@@ -93,9 +94,12 @@ pub(crate) use num_traits::cast::{FromPrimitive, ToPrimitive};
 pub(crate) use num_traits::identities::Zero;
 pub(crate) use serde::Deserialize;
 pub(crate) use std::collections::VecDeque;
+pub(crate) use std::fmt::Write;
 pub(crate) use std::future::Future;
 pub(crate) use std::sync::{Arc, Mutex};
 
+pub(crate) use itertools::Itertools;
+
 pub trait FromInputStream {
     fn from_input_stream(self) -> OutputStream;
 }

src/shell/helper.rs

@@ -3,7 +3,7 @@ use crate::parser::hir::syntax_shape::{color_fallible_syntax, FlatShape, Pipelin
 use crate::parser::hir::TokensIterator;
 use crate::parser::nom_input;
 use crate::parser::parse::token_tree::TokenNode;
-use crate::{Span, Spanned, SpannedItem, Tag, Tagged, Text};
+use crate::{HasSpan, Spanned, SpannedItem, Tag, Tagged, Text};
 use ansi_term::Color;
 use log::{log_enabled, trace};
 use rustyline::completion::Completer;
@@ -65,9 +65,7 @@ impl Highlighter for Helper {
                 let mut tokens = TokensIterator::all(&tokens[..], v.span());
 
                 let text = Text::from(line);
-                let expand_context = self
+                let expand_context = self.context.expand_context(&text);
-                    .context
-                    .expand_context(&text, Span::new(0, line.len() - 1));
 
                 #[cfg(not(coloring_in_tokens))]
                 let shapes = {
@@ -86,16 +84,17 @@ impl Highlighter for Helper {
                 let shapes = {
                     // We just constructed a token list that only contains a pipeline, so it can't fail
                     color_fallible_syntax(&PipelineShape, &mut tokens, &expand_context).unwrap();
-                    tokens.with_tracer(|_, tracer| tracer.finish());
+                    tokens.with_color_tracer(|_, tracer| tracer.finish());
 
                     tokens.state().shapes()
                 };
 
-                trace!(target: "nu::color_syntax", "{:#?}", tokens.tracer());
+                trace!(target: "nu::color_syntax", "{:#?}", tokens.color_tracer());
 
-                if log_enabled!(target: "nu::color_syntax", log::Level::Trace) {
+                if log_enabled!(target: "nu::color_syntax", log::Level::Debug) {
                     println!("");
-                    ptree::print_tree(&tokens.tracer().clone().print(Text::from(line))).unwrap();
+                    ptree::print_tree(&tokens.color_tracer().clone().print(Text::from(line)))
+                        .unwrap();
                     println!("");
                 }
 

src/traits.rs

@@ -1,14 +1,28 @@
 use crate::prelude::*;
-use std::fmt;
+use derive_new::new;
+use std::fmt::{self, Write};
 
-pub struct Debuggable<'a, T: ToDebug> {
+pub struct Debuggable<'a, T: FormatDebug> {
     inner: &'a T,
     source: &'a str,
 }
 
+impl FormatDebug for str {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
+        write!(f, "{}", self)
+    }
+}
+
 impl<T: ToDebug> fmt::Display for Debuggable<'_, T> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.inner.fmt_debug(f, self.source)
+        self.inner.fmt_debug(
+            &mut DebugFormatter::new(
+                f,
+                ansi_term::Color::White.bold(),
+                ansi_term::Color::Black.bold(),
+            ),
+            self.source,
+        )
     }
 }
 
@@ -16,13 +30,109 @@ pub trait HasTag {
     fn tag(&self) -> Tag;
 }
 
-pub trait ToDebug: Sized {
+#[derive(new)]
+pub struct DebugFormatter<'me, 'args> {
+    formatter: &'me mut std::fmt::Formatter<'args>,
+    style: ansi_term::Style,
+    default_style: ansi_term::Style,
+}
+
+impl<'me, 'args> DebugFormatter<'me, 'args> {
+    pub fn say<'debuggable>(
+        &mut self,
+        kind: &str,
+        debuggable: Debuggable<'debuggable, impl FormatDebug>,
+    ) -> std::fmt::Result {
+        write!(self, "{}", self.style.paint(kind))?;
+        write!(self, "{}", self.default_style.paint(" "))?;
+        write!(
+            self,
+            "{}",
+            self.default_style.paint(format!("{}", debuggable))
+        )
+    }
+
+    pub fn say_str<'debuggable>(
+        &mut self,
+        kind: &str,
+        string: impl AsRef<str>,
+    ) -> std::fmt::Result {
+        write!(self, "{}", self.style.paint(kind))?;
+        write!(self, "{}", self.default_style.paint(" "))?;
+        write!(self, "{}", self.default_style.paint(string.as_ref()))
+    }
+
+    pub fn say_block(
+        &mut self,
+        kind: &str,
+        block: impl FnOnce(&mut Self) -> std::fmt::Result,
+    ) -> std::fmt::Result {
+        write!(self, "{}", self.style.paint(kind))?;
+        write!(self, "{}", self.default_style.paint(" "))?;
+        block(self)
+    }
+
+    pub fn say_dict<'debuggable>(
+        &mut self,
+        kind: &str,
+        dict: indexmap::IndexMap<&str, String>,
+    ) -> std::fmt::Result {
+        write!(self, "{}", self.style.paint(kind))?;
+        write!(self, "{}", self.default_style.paint(" "))?;
+
+        let last = dict.len() - 1;
+
+        for (i, (key, value)) in dict.into_iter().enumerate() {
+            write!(self, "{}", self.default_style.paint(key))?;
+            write!(self, "{}", self.default_style.paint("=["))?;
+            write!(self, "{}", self.style.paint(value))?;
+            write!(self, "{}", self.default_style.paint("]"))?;
+
+            if i != last {
+                write!(self, "{}", self.default_style.paint(" "))?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl<'a, 'b> std::fmt::Write for DebugFormatter<'a, 'b> {
+    fn write_str(&mut self, s: &str) -> std::fmt::Result {
+        self.formatter.write_str(s)
+    }
+
+    fn write_char(&mut self, c: char) -> std::fmt::Result {
+        self.formatter.write_char(c)
+    }
+
+    fn write_fmt(self: &mut Self, args: std::fmt::Arguments<'_>) -> std::fmt::Result {
+        self.formatter.write_fmt(args)
+    }
+}
+
+pub trait FormatDebug: std::fmt::Debug {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result;
+}
+
+pub trait ToDebug: Sized + FormatDebug {
+    fn debug<'a>(&'a self, source: &'a str) -> Debuggable<'a, Self>;
+}
+
+impl FormatDebug for Box<dyn FormatDebug> {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        (&**self).fmt_debug(f, source)
+    }
+}
+
+impl<T> ToDebug for T
+where
+    T: FormatDebug + Sized,
+{
     fn debug<'a>(&'a self, source: &'a str) -> Debuggable<'a, Self> {
         Debuggable {
             inner: self,
             source,
         }
     }
-
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result;
 }