Restructure and streamline token expansion (#1123)

Restructure and streamline token expansion

The purpose of this commit is to streamline the token expansion code, by
removing aspects of the code that are no longer relevant, removing
pointless duplication, and eliminating the need to pass the same
arguments to `expand_syntax`.

The first big-picture change in this commit is that instead of a handful
of `expand_` functions, which take a TokensIterator and ExpandContext, a
smaller number of methods on the `TokensIterator` do the same job.

The second big-picture change in this commit is fully eliminating the
coloring traits, making coloring a responsibility of the base expansion
implementations. This also means that the coloring tracer is merged into
the expansion tracer, so you can follow a single expansion and see how
the expansion process produced colored tokens.

One side effect of this change is that the expander itself is marginally
more error-correcting. The error correction works by switching from
structured expansion to `BackoffColoringMode` when an unexpected token
is found, which guarantees that all spans of the source are colored, but
may not be the most optimal error recovery strategy.

That said, because `BackoffColoringMode` only extends as far as a
closing delimiter (`)`, `]`, `}`) or pipe (`|`), it does result in
fairly granular correction strategy.

The current code still produces an `Err` (plus a complete list of
colored shapes) from the parsing process if any errors are encountered,
but this could easily be addressed now that the underlying expansion is
error-correcting.

This commit also colors any spans that are syntax errors in red, and
causes the parser to include some additional information about what
tokens were expected at any given point where an error was encountered,
so that completions and hinting could be more robust in the future.

Co-authored-by: Jonathan Turner <jonathandturner@users.noreply.github.com>
Co-authored-by: Andrés N. Robalino <andres@androbtech.com>

crates/nu-source/src/lib.rs

@@ -6,10 +6,11 @@ mod tracable;
 
 pub use self::meta::{
     span_for_spanned_list, tag_for_tagged_list, AnchorLocation, HasFallibleSpan, HasSpan, HasTag,
-    Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem,
+    IntoSpanned, Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem,
 };
 pub use self::pretty::{
-    b, DebugDoc, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource, ShellAnnotation,
+    b, DebugDoc, DebugDocBuilder, PrettyDebug, PrettyDebugRefineKind, PrettyDebugWithSource,
+    ShellAnnotation,
 };
 pub use self::term_colored::TermColored;
 pub use self::text::Text;

crates/nu-source/src/meta.rs

@@ -490,6 +490,10 @@ impl Span {
         }
     }
 
+    pub fn contains(&self, pos: usize) -> bool {
+        self.start <= pos && self.end >= pos
+    }
+
     pub fn since(&self, other: impl Into<Span>) -> Span {
         let other = other.into();
 
@@ -568,29 +572,66 @@ impl language_reporting::ReportingSpan for Span {
     }
 }
 
-pub trait HasSpan: PrettyDebugWithSource {
-    fn span(&self) -> Span;
+pub trait IntoSpanned {
+    type Output: HasFallibleSpan;
+
+    fn into_spanned(self, span: impl Into<Span>) -> Self::Output;
 }
 
-pub trait HasFallibleSpan: PrettyDebugWithSource {
-    fn maybe_span(&self) -> Option<Span>;
-}
-
-impl<T: HasSpan> HasFallibleSpan for T {
-    fn maybe_span(&self) -> Option<Span> {
-        Some(HasSpan::span(self))
+impl<T: HasFallibleSpan> IntoSpanned for T {
+    type Output = T;
+    fn into_spanned(self, _span: impl Into<Span>) -> Self::Output {
+        self
     }
 }
 
-impl<T> HasSpan for Spanned<T>
+pub trait HasSpan {
+    fn span(&self) -> Span;
+}
+
+impl<T, E> HasSpan for Result<T, E>
 where
-    Spanned<T>: PrettyDebugWithSource,
+    T: HasSpan,
 {
+    fn span(&self) -> Span {
+        match self {
+            Result::Ok(val) => val.span(),
+            Result::Err(_) => Span::unknown(),
+        }
+    }
+}
+
+impl<T> HasSpan for Spanned<T> {
     fn span(&self) -> Span {
         self.span
     }
 }
 
+pub trait HasFallibleSpan {
+    fn maybe_span(&self) -> Option<Span>;
+}
+
+impl HasFallibleSpan for bool {
+    fn maybe_span(&self) -> Option<Span> {
+        None
+    }
+}
+
+impl HasFallibleSpan for () {
+    fn maybe_span(&self) -> Option<Span> {
+        None
+    }
+}
+
+impl<T> HasFallibleSpan for T
+where
+    T: HasSpan,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        Some(HasSpan::span(self))
+    }
+}
+
 impl PrettyDebugWithSource for Option<Span> {
     fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
         match self {
@@ -609,8 +650,8 @@ impl HasFallibleSpan for Option<Span> {
 impl PrettyDebugWithSource for Span {
     fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
         b::typed(
-            "spanned",
-            b::keyword("for") + b::space() + b::description(format!("{:?}", source)),
+            "span",
+            b::keyword("for") + b::space() + b::description(format!("{:?}", self.slice(source))),
         )
     }
 }
@@ -628,15 +669,12 @@ where
     fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
         match self {
             None => b::description("nothing"),
-            Some(v) => v.pretty_debug(source),
+            Some(v) => v.pretty_debug(v.span.slice(source)),
         }
     }
 }
 
-impl<T> HasFallibleSpan for Option<Spanned<T>>
-where
-    Spanned<T>: PrettyDebugWithSource,
-{
+impl<T> HasFallibleSpan for Option<Spanned<T>> {
     fn maybe_span(&self) -> Option<Span> {
         match self {
             None => None,
@@ -657,10 +695,7 @@ where
     }
 }
 
-impl<T> HasFallibleSpan for Option<Tagged<T>>
-where
-    Tagged<T>: PrettyDebugWithSource,
-{
+impl<T> HasFallibleSpan for Option<Tagged<T>> {
     fn maybe_span(&self) -> Option<Span> {
         match self {
             None => None,
@@ -669,10 +704,7 @@ where
     }
 }
 
-impl<T> HasSpan for Tagged<T>
-where
-    Tagged<T>: PrettyDebugWithSource,
-{
+impl<T> HasSpan for Tagged<T> {
     fn span(&self) -> Span {
         self.tag.span
     }

crates/nu-source/src/pretty.rs

@@ -1,3 +1,4 @@
+use crate::meta::Spanned;
 use crate::term_colored::TermColored;
 use crate::text::Text;
 use derive_new::new;
@@ -98,6 +99,21 @@ pub struct DebugDocBuilder {
     pub inner: PrettyDebugDocBuilder,
 }
 
+impl PrettyDebug for bool {
+    fn pretty(&self) -> DebugDocBuilder {
+        match self {
+            true => b::primitive("true"),
+            false => b::primitive("false"),
+        }
+    }
+}
+
+impl PrettyDebug for () {
+    fn pretty(&self) -> DebugDocBuilder {
+        b::primitive("nothing")
+    }
+}
+
 impl PrettyDebug for DebugDocBuilder {
     fn pretty(&self) -> DebugDocBuilder {
         self.clone()
@@ -156,7 +172,7 @@ impl DebugDocBuilder {
     }
 
     pub fn typed(kind: &str, value: DebugDocBuilder) -> DebugDocBuilder {
-        b::delimit("(", b::kind(kind) + b::space() + value.group(), ")").group()
+        b::kind(kind) + b::delimit("[", value.group(), "]")
     }
 
     pub fn subtyped(
@@ -340,9 +356,23 @@ pub struct DebugDoc {
     pub inner: PrettyDebugDoc,
 }
 
+#[derive(Debug, Copy, Clone)]
+pub enum PrettyDebugRefineKind {
+    ContextFree,
+    WithContext,
+}
+
 pub trait PrettyDebugWithSource: Sized {
     fn pretty_debug(&self, source: &str) -> DebugDocBuilder;
 
+    fn refined_pretty_debug(
+        &self,
+        _refine: PrettyDebugRefineKind,
+        source: &str,
+    ) -> DebugDocBuilder {
+        self.pretty_debug(source)
+    }
+
     // This is a transitional convenience method
     fn debug(&self, source: impl Into<Text>) -> String
     where
@@ -359,12 +389,27 @@ pub trait PrettyDebugWithSource: Sized {
     }
 }
 
+impl<T: PrettyDebug> PrettyDebug for Spanned<T> {
+    fn pretty(&self) -> DebugDocBuilder {
+        self.item.pretty()
+    }
+}
+
 impl<T: PrettyDebug> PrettyDebugWithSource for T {
     fn pretty_debug(&self, _source: &str) -> DebugDocBuilder {
         self.pretty()
     }
 }
 
+impl<T: PrettyDebugWithSource, E> PrettyDebugWithSource for Result<T, E> {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            Err(_) => b::error("error"),
+            Ok(val) => val.pretty_debug(source),
+        }
+    }
+}
+
 pub struct DebuggableWithSource<T: PrettyDebugWithSource> {
     inner: T,
     source: Text,