Modernize external parse and improve trace

The original purpose of this PR was to modernize the external parser to
use the new Shape system.

This commit does include some of that change, but a more important
aspect of this change is an improvement to the expansion trace.

Previous commit 6a7c00ea adding trace infrastructure to the syntax coloring
feature. This commit adds tracing to the expander.

The bulk of that work, in addition to the tree builder logic, was an
overhaul of the formatter traits to make them more general purpose, and
more structured.

Some highlights:

- `ToDebug` was split into two traits (`ToDebug` and `DebugFormat`)
  because implementations needed to become objects, but a convenience
  method on `ToDebug` didn't qualify
- `DebugFormat`'s `fmt_debug` method now takes a `DebugFormatter` rather
  than a standard formatter, and `DebugFormatter` has a new (but still
  limited) facility for structured formatting.
- Implementations of `ExpandSyntax` need to produce output that
  implements `DebugFormat`.

Unlike the highlighter changes, these changes are fairly focused in the
trace output, so these changes aren't behind a flag.

src/commands/classified.rs

@@ -4,7 +4,9 @@ use bytes::{BufMut, BytesMut};
 use derive_new::new;
 use futures::stream::StreamExt;
 use futures_codec::{Decoder, Encoder, Framed};
+use itertools::Itertools;
 use log::{log_enabled, trace};
+use std::fmt;
 use std::io::{Error, ErrorKind};
 use subprocess::Exec;
 
@@ -72,26 +74,77 @@ impl ClassifiedInputStream {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub(crate) struct ClassifiedPipeline {
-    pub(crate) commands: Vec<ClassifiedCommand>,
+    pub(crate) commands: Spanned<Vec<ClassifiedCommand>>,
 }
 
-#[derive(Debug, Eq, PartialEq)]
+impl FormatDebug for ClassifiedPipeline {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say_str(
+            "classified pipeline",
+            self.commands.iter().map(|c| c.debug(source)).join(" | "),
+        )
+    }
+}
+
+impl HasSpan for ClassifiedPipeline {
+    fn span(&self) -> Span {
+        self.commands.span
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
 pub(crate) enum ClassifiedCommand {
     #[allow(unused)]
     Expr(TokenNode),
     Internal(InternalCommand),
     #[allow(unused)]
-    Dynamic(hir::Call),
+    Dynamic(Spanned<hir::Call>),
     External(ExternalCommand),
 }
 
-#[derive(new, Debug, Eq, PartialEq)]
+impl FormatDebug for ClassifiedCommand {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        match self {
+            ClassifiedCommand::Expr(expr) => expr.fmt_debug(f, source),
+            ClassifiedCommand::Internal(internal) => internal.fmt_debug(f, source),
+            ClassifiedCommand::Dynamic(dynamic) => dynamic.fmt_debug(f, source),
+            ClassifiedCommand::External(external) => external.fmt_debug(f, source),
+        }
+    }
+}
+
+impl HasSpan for ClassifiedCommand {
+    fn span(&self) -> Span {
+        match self {
+            ClassifiedCommand::Expr(node) => node.span(),
+            ClassifiedCommand::Internal(command) => command.span(),
+            ClassifiedCommand::Dynamic(call) => call.span,
+            ClassifiedCommand::External(command) => command.span(),
+        }
+    }
+}
+
+#[derive(new, Debug, Clone, Eq, PartialEq)]
 pub(crate) struct InternalCommand {
     pub(crate) name: String,
     pub(crate) name_tag: Tag,
-    pub(crate) args: hir::Call,
+    pub(crate) args: Spanned<hir::Call>,
+}
+
+impl HasSpan for InternalCommand {
+    fn span(&self) -> Span {
+        let start = self.name_tag.span;
+
+        start.until(self.args.span)
+    }
+}
+
+impl FormatDebug for InternalCommand {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        f.say("internal", self.args.debug(source))
+    }
 }
 
 #[derive(new, Debug, Eq, PartialEq)]
@@ -122,7 +175,7 @@ impl InternalCommand {
             context.run_command(
                 command,
                 self.name_tag.clone(),
-                self.args,
+                self.args.item,
                 &source,
                 objects,
                 is_first_command,
@@ -201,12 +254,31 @@ impl InternalCommand {
     }
 }
 
-#[derive(Debug, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq)]
 pub(crate) struct ExternalCommand {
     pub(crate) name: String,
 
     pub(crate) name_tag: Tag,
-    pub(crate) args: Vec<Tagged<String>>,
+    pub(crate) args: Spanned<Vec<Tagged<String>>>,
+}
+
+impl FormatDebug for ExternalCommand {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
+        write!(f, "{}", self.name)?;
+
+        if self.args.item.len() > 0 {
+            write!(f, " ")?;
+            write!(f, "{}", self.args.iter().map(|i| i.debug(source)).join(" "))?;
+        }
+
+        Ok(())
+    }
+}
+
+impl HasSpan for ExternalCommand {
+    fn span(&self) -> Span {
+        self.name_tag.span.until(self.args.span)
+    }
 }
 
 #[derive(Debug)]
@@ -230,7 +302,7 @@ impl ExternalCommand {
         trace!(target: "nu::run::external", "inputs = {:?}", inputs);
 
         let mut arg_string = format!("{}", self.name);
-        for arg in &self.args {
+        for arg in &self.args.item {
             arg_string.push_str(&arg);
         }
 
@@ -275,7 +347,7 @@ impl ExternalCommand {
             process = Exec::shell(itertools::join(commands, " && "))
         } else {
             process = Exec::cmd(&self.name);
-            for arg in &self.args {
+            for arg in &self.args.item {
                 let arg_chars: Vec<_> = arg.chars().collect();
                 if arg_chars.len() > 1
                     && arg_chars[0] == '"'

src/commands/command.rs

@@ -19,8 +19,8 @@ pub struct UnevaluatedCallInfo {
     pub name_tag: Tag,
 }
 
-impl ToDebug for UnevaluatedCallInfo {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+impl FormatDebug for UnevaluatedCallInfo {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         self.args.fmt_debug(f, source)
     }
 }
@@ -96,8 +96,14 @@ impl RawCommandArgs {
     }
 }
 
-impl ToDebug for CommandArgs {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+impl std::fmt::Debug for CommandArgs {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.call_info.fmt(f)
+    }
+}
+
+impl FormatDebug for CommandArgs {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         self.call_info.fmt_debug(f, source)
     }
 }
@@ -377,7 +383,7 @@ impl EvaluatedCommandArgs {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum CommandAction {
     ChangePath(String),
     Exit,
@@ -389,8 +395,8 @@ pub enum CommandAction {
     LeaveShell,
 }
 
-impl ToDebug for CommandAction {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
+impl FormatDebug for CommandAction {
+    fn fmt_debug(&self, f: &mut DebugFormatter, _source: &str) -> fmt::Result {
         match self {
             CommandAction::ChangePath(s) => write!(f, "action:change-path={}", s),
             CommandAction::Exit => write!(f, "action:exit"),
@@ -408,7 +414,7 @@ impl ToDebug for CommandAction {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum ReturnSuccess {
     Value(Tagged<Value>),
     Action(CommandAction),
@@ -416,8 +422,8 @@ pub enum ReturnSuccess {
 
 pub type ReturnValue = Result<ReturnSuccess, ShellError>;
 
-impl ToDebug for ReturnValue {
-    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+impl FormatDebug for ReturnValue {
+    fn fmt_debug(&self, f: &mut DebugFormatter, source: &str) -> fmt::Result {
         match self {
             Err(err) => write!(f, "{}", err.debug(source)),
             Ok(ReturnSuccess::Value(v)) => write!(f, "{:?}", v.debug()),