Extract nu_source into a crate

This commit extracts Tag, Span, Text, as well as source-related debug
facilities into a new crate called nu_source.

This change is much bigger than one might have expected because the
previous code relied heavily on implementing inherent methods on
`Tagged<T>` and `Spanned<T>`, which is no longer possible.

As a result, this change creates more concrete types instead of using
`Tagged<T>`. One notable example: Tagged<Value> became Value, and Value
became UntaggedValue.

This change clarifies the intent of the code in many places, but it does
make it a big change.

crates/nu-source/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "nu-source"
+version = "0.1.0"
+authors = ["Yehuda Katz <wycats@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+
+serde = { version = "1.0.102", features = ["derive"] }
+derive-new = "0.5.8"
+getset = "0.0.9"
+nom_locate = "1.0.0"
+nom-tracable = "0.4.1"
+language-reporting = "0.4.0"
+termcolor = "1.0.5"
+pretty = "0.5.2"

crates/nu-source/src/lib.rs

@@ -0,0 +1,15 @@
+mod meta;
+mod pretty;
+mod term_colored;
+mod text;
+mod tracable;
+
+pub use self::meta::{
+    span_for_spanned_list, tag_for_tagged_list, AnchorLocation, HasFallibleSpan, HasSpan, HasTag,
+    Span, Spanned, SpannedItem, Tag, Tagged, TaggedItem,
+};
+pub use self::pretty::{
+    b, DebugDoc, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource, ShellAnnotation,
+};
+pub use self::text::Text;
+pub use self::tracable::{nom_input, NomSpan, TracableContext};

crates/nu-source/src/meta.rs

@@ -0,0 +1,655 @@
+use crate::pretty::{b, DebugDocBuilder, PrettyDebugWithSource};
+use crate::text::Text;
+use crate::tracable::TracableContext;
+
+use derive_new::new;
+use getset::Getters;
+use serde::Deserialize;
+use serde::Serialize;
+use std::path::{Path, PathBuf};
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum AnchorLocation {
+    Url(String),
+    File(String),
+    Source(Text),
+}
+
+pub trait HasTag {
+    fn tag(&self) -> Tag;
+}
+
+#[derive(new, Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+pub struct Spanned<T> {
+    pub span: Span,
+    pub item: T,
+}
+
+impl<T> Spanned<T> {
+    pub fn map<U>(self, input: impl FnOnce(T) -> U) -> Spanned<U> {
+        let span = self.span;
+
+        let mapped = input(self.item);
+        mapped.spanned(span)
+    }
+}
+
+impl Spanned<String> {
+    pub fn items<'a, U>(
+        items: impl Iterator<Item = &'a Spanned<String>>,
+    ) -> impl Iterator<Item = &'a str> {
+        items.into_iter().map(|item| &item.item[..])
+    }
+}
+
+impl Spanned<String> {
+    pub fn borrow_spanned(&self) -> Spanned<&str> {
+        let span = self.span;
+        self.item[..].spanned(span)
+    }
+}
+
+pub trait SpannedItem: Sized {
+    fn spanned(self, span: impl Into<Span>) -> Spanned<Self> {
+        Spanned {
+            item: self,
+            span: span.into(),
+        }
+    }
+
+    fn spanned_unknown(self) -> Spanned<Self> {
+        Spanned {
+            item: self,
+            span: Span::unknown(),
+        }
+    }
+}
+impl<T> SpannedItem for T {}
+
+impl<T> std::ops::Deref for Spanned<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &self.item
+    }
+}
+
+#[derive(new, Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+pub struct Tagged<T> {
+    pub tag: Tag,
+    pub item: T,
+}
+
+impl Tagged<String> {
+    pub fn borrow_spanned(&self) -> Spanned<&str> {
+        let span = self.tag.span;
+        self.item[..].spanned(span)
+    }
+
+    pub fn borrow_tagged(&self) -> Tagged<&str> {
+        self.item[..].tagged(self.tag.clone())
+    }
+}
+
+impl<T> Tagged<Vec<T>> {
+    pub fn items(&self) -> impl Iterator<Item = &T> {
+        self.item.iter()
+    }
+}
+
+impl<T> HasTag for Tagged<T> {
+    fn tag(&self) -> Tag {
+        self.tag.clone()
+    }
+}
+
+impl AsRef<Path> for Tagged<PathBuf> {
+    fn as_ref(&self) -> &Path {
+        self.item.as_ref()
+    }
+}
+
+pub trait TaggedItem: Sized {
+    fn tagged(self, tag: impl Into<Tag>) -> Tagged<Self> {
+        Tagged {
+            item: self,
+            tag: tag.into(),
+        }
+    }
+
+    // For now, this is a temporary facility. In many cases, there are other useful spans that we
+    // could be using, such as the original source spans of JSON or Toml files, but we don't yet
+    // have the infrastructure to make that work.
+    fn tagged_unknown(self) -> Tagged<Self> {
+        Tagged {
+            item: self,
+            tag: Tag {
+                span: Span::unknown(),
+                anchor: None,
+            },
+        }
+    }
+}
+
+impl<T> TaggedItem for T {}
+
+impl<T> std::ops::Deref for Tagged<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &self.item
+    }
+}
+
+impl<T> Tagged<T> {
+    pub fn map<U>(self, input: impl FnOnce(T) -> U) -> Tagged<U> {
+        let tag = self.tag();
+
+        let mapped = input(self.item);
+        mapped.tagged(tag)
+    }
+
+    pub fn map_anchored(self, anchor: &Option<AnchorLocation>) -> Tagged<T> {
+        let mut tag = self.tag;
+
+        tag.anchor = anchor.clone();
+
+        Tagged {
+            item: self.item,
+            tag: tag,
+        }
+    }
+
+    pub fn transpose(&self) -> Tagged<&T> {
+        Tagged {
+            item: &self.item,
+            tag: self.tag.clone(),
+        }
+    }
+
+    pub fn tag(&self) -> Tag {
+        self.tag.clone()
+    }
+
+    pub fn span(&self) -> Span {
+        self.tag.span
+    }
+
+    pub fn anchor(&self) -> Option<AnchorLocation> {
+        self.tag.anchor.clone()
+    }
+
+    pub fn anchor_name(&self) -> Option<String> {
+        match self.tag.anchor {
+            Some(AnchorLocation::File(ref file)) => Some(file.clone()),
+            Some(AnchorLocation::Url(ref url)) => Some(url.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn item(&self) -> &T {
+        &self.item
+    }
+
+    pub fn into_parts(self) -> (T, Tag) {
+        (self.item, self.tag)
+    }
+}
+
+impl From<&Tag> for Tag {
+    fn from(input: &Tag) -> Tag {
+        input.clone()
+    }
+}
+
+impl<T> From<nom_locate::LocatedSpanEx<&str, T>> for Span {
+    fn from(input: nom_locate::LocatedSpanEx<&str, T>) -> Span {
+        Span::new(input.offset, input.offset + input.fragment.len())
+    }
+}
+
+impl<T>
+    From<(
+        nom_locate::LocatedSpanEx<T, u64>,
+        nom_locate::LocatedSpanEx<T, u64>,
+    )> for Span
+{
+    fn from(
+        input: (
+            nom_locate::LocatedSpanEx<T, u64>,
+            nom_locate::LocatedSpanEx<T, u64>,
+        ),
+    ) -> Span {
+        Span {
+            start: input.0.offset,
+            end: input.1.offset,
+        }
+    }
+}
+
+impl From<(usize, usize)> for Span {
+    fn from(input: (usize, usize)) -> Span {
+        Span::new(input.0, input.1)
+    }
+}
+
+impl From<&std::ops::Range<usize>> for Span {
+    fn from(input: &std::ops::Range<usize>) -> Span {
+        Span {
+            start: input.start,
+            end: input.end,
+        }
+    }
+}
+
+#[derive(
+    Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash, Getters, new,
+)]
+pub struct Tag {
+    pub anchor: Option<AnchorLocation>,
+    pub span: Span,
+}
+
+impl From<Span> for Tag {
+    fn from(span: Span) -> Self {
+        Tag { anchor: None, span }
+    }
+}
+
+impl From<&Span> for Tag {
+    fn from(span: &Span) -> Self {
+        Tag {
+            anchor: None,
+            span: *span,
+        }
+    }
+}
+
+impl From<(usize, usize, TracableContext)> for Tag {
+    fn from((start, end, _context): (usize, usize, TracableContext)) -> Self {
+        Tag {
+            anchor: None,
+            span: Span::new(start, end),
+        }
+    }
+}
+
+impl From<(usize, usize, AnchorLocation)> for Tag {
+    fn from((start, end, anchor): (usize, usize, AnchorLocation)) -> Self {
+        Tag {
+            anchor: Some(anchor),
+            span: Span::new(start, end),
+        }
+    }
+}
+
+impl From<(usize, usize, Option<AnchorLocation>)> for Tag {
+    fn from((start, end, anchor): (usize, usize, Option<AnchorLocation>)) -> Self {
+        Tag {
+            anchor,
+            span: Span::new(start, end),
+        }
+    }
+}
+
+impl From<nom_locate::LocatedSpanEx<&str, TracableContext>> for Tag {
+    fn from(input: nom_locate::LocatedSpanEx<&str, TracableContext>) -> Tag {
+        Tag {
+            anchor: None,
+            span: Span::new(input.offset, input.offset + input.fragment.len()),
+        }
+    }
+}
+
+impl From<Tag> for Span {
+    fn from(tag: Tag) -> Self {
+        tag.span
+    }
+}
+
+impl From<&Tag> for Span {
+    fn from(tag: &Tag) -> Self {
+        tag.span
+    }
+}
+
+impl Tag {
+    pub fn unknown_anchor(span: Span) -> Tag {
+        Tag { anchor: None, span }
+    }
+
+    pub fn for_char(pos: usize, anchor: AnchorLocation) -> Tag {
+        Tag {
+            anchor: Some(anchor),
+            span: Span {
+                start: pos,
+                end: pos + 1,
+            },
+        }
+    }
+
+    pub fn unknown_span(anchor: AnchorLocation) -> Tag {
+        Tag {
+            anchor: Some(anchor),
+            span: Span::unknown(),
+        }
+    }
+
+    pub fn unknown() -> Tag {
+        Tag {
+            anchor: None,
+            span: Span::unknown(),
+        }
+    }
+
+    pub fn anchor(&self) -> Option<AnchorLocation> {
+        self.anchor.clone()
+    }
+
+    pub fn until(&self, other: impl Into<Tag>) -> Tag {
+        let other = other.into();
+        debug_assert!(
+            self.anchor == other.anchor,
+            "Can only merge two tags with the same anchor"
+        );
+
+        Tag {
+            span: Span::new(self.span.start, other.span.end),
+            anchor: self.anchor.clone(),
+        }
+    }
+
+    pub fn until_option(&self, other: Option<impl Into<Tag>>) -> Tag {
+        match other {
+            Some(other) => {
+                let other = other.into();
+                debug_assert!(
+                    self.anchor == other.anchor,
+                    "Can only merge two tags with the same anchor"
+                );
+
+                Tag {
+                    span: Span::new(self.span.start, other.span.end),
+                    anchor: self.anchor.clone(),
+                }
+            }
+            None => self.clone(),
+        }
+    }
+
+    pub fn slice<'a>(&self, source: &'a str) -> &'a str {
+        self.span.slice(source)
+    }
+
+    pub fn string<'a>(&self, source: &'a str) -> String {
+        self.span.slice(source).to_string()
+    }
+
+    pub fn tagged_slice<'a>(&self, source: &'a str) -> Tagged<&'a str> {
+        self.span.slice(source).tagged(self)
+    }
+
+    pub fn tagged_string<'a>(&self, source: &'a str) -> Tagged<String> {
+        self.span.slice(source).to_string().tagged(self)
+    }
+
+    pub fn anchor_name(&self) -> Option<String> {
+        match self.anchor {
+            Some(AnchorLocation::File(ref file)) => Some(file.clone()),
+            Some(AnchorLocation::Url(ref url)) => Some(url.clone()),
+            _ => None,
+        }
+    }
+}
+
+#[allow(unused)]
+pub fn tag_for_tagged_list(mut iter: impl Iterator<Item = Tag>) -> Tag {
+    let first = iter.next();
+
+    let first = match first {
+        None => return Tag::unknown(),
+        Some(first) => first,
+    };
+
+    let last = iter.last();
+
+    match last {
+        None => first,
+        Some(last) => first.until(last),
+    }
+}
+
+#[allow(unused)]
+pub fn span_for_spanned_list(mut iter: impl Iterator<Item = Span>) -> Span {
+    let first = iter.next();
+
+    let first = match first {
+        None => return Span::unknown(),
+        Some(first) => first,
+    };
+
+    let last = iter.last();
+
+    match last {
+        None => first,
+        Some(last) => first.until(last),
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
+pub struct Span {
+    start: usize,
+    end: usize,
+}
+
+impl From<&Span> for Span {
+    fn from(span: &Span) -> Span {
+        *span
+    }
+}
+
+impl From<Option<Span>> for Span {
+    fn from(input: Option<Span>) -> Span {
+        match input {
+            None => Span::new(0, 0),
+            Some(span) => span,
+        }
+    }
+}
+
+impl Span {
+    pub fn unknown() -> Span {
+        Span::new(0, 0)
+    }
+
+    pub fn new(start: usize, end: usize) -> Span {
+        assert!(
+            end >= start,
+            "Can't create a Span whose end < start, start={}, end={}",
+            start,
+            end
+        );
+
+        Span { start, end }
+    }
+
+    pub fn for_char(pos: usize) -> Span {
+        Span {
+            start: pos,
+            end: pos + 1,
+        }
+    }
+
+    pub fn until(&self, other: impl Into<Span>) -> Span {
+        let other = other.into();
+
+        Span::new(self.start, other.end)
+    }
+
+    pub fn until_option(&self, other: Option<impl Into<Span>>) -> Span {
+        match other {
+            Some(other) => {
+                let other = other.into();
+
+                Span::new(self.start, other.end)
+            }
+            None => *self,
+        }
+    }
+
+    pub fn string<'a>(&self, source: &'a str) -> String {
+        self.slice(source).to_string()
+    }
+
+    pub fn spanned_slice<'a>(&self, source: &'a str) -> Spanned<&'a str> {
+        self.slice(source).spanned(*self)
+    }
+
+    pub fn spanned_string<'a>(&self, source: &'a str) -> Spanned<String> {
+        self.slice(source).to_string().spanned(*self)
+    }
+
+    pub fn start(&self) -> usize {
+        self.start
+    }
+
+    pub fn end(&self) -> usize {
+        self.end
+    }
+
+    pub fn is_unknown(&self) -> bool {
+        self.start == 0 && self.end == 0
+    }
+
+    pub fn slice<'a>(&self, source: &'a str) -> &'a str {
+        &source[self.start..self.end]
+    }
+}
+
+impl language_reporting::ReportingSpan for Span {
+    fn with_start(&self, start: usize) -> Self {
+        Span::new(start, self.end)
+    }
+
+    fn with_end(&self, end: usize) -> Self {
+        Span::new(self.start, end)
+    }
+
+    fn start(&self) -> usize {
+        self.start
+    }
+
+    fn end(&self) -> usize {
+        self.end
+    }
+}
+
+pub trait HasSpan: PrettyDebugWithSource {
+    fn span(&self) -> Span;
+}
+
+pub trait HasFallibleSpan: PrettyDebugWithSource {
+    fn maybe_span(&self) -> Option<Span>;
+}
+
+impl<T: HasSpan> HasFallibleSpan for T {
+    fn maybe_span(&self) -> Option<Span> {
+        Some(HasSpan::span(self))
+    }
+}
+
+impl<T> HasSpan for Spanned<T>
+where
+    Spanned<T>: PrettyDebugWithSource,
+{
+    fn span(&self) -> Span {
+        self.span
+    }
+}
+
+impl PrettyDebugWithSource for Option<Span> {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            None => b::description("no span"),
+            Some(span) => span.pretty_debug(source),
+        }
+    }
+}
+
+impl HasFallibleSpan for Option<Span> {
+    fn maybe_span(&self) -> Option<Span> {
+        *self
+    }
+}
+
+impl PrettyDebugWithSource for Span {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "spanned",
+            b::keyword("for") + b::space() + b::description(format!("{:?}", source)),
+        )
+    }
+}
+
+impl HasSpan for Span {
+    fn span(&self) -> Span {
+        *self
+    }
+}
+
+impl<T> PrettyDebugWithSource for Option<Spanned<T>>
+where
+    Spanned<T>: PrettyDebugWithSource,
+{
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            None => b::description("nothing"),
+            Some(v) => v.pretty_debug(source),
+        }
+    }
+}
+
+impl<T> HasFallibleSpan for Option<Spanned<T>>
+where
+    Spanned<T>: PrettyDebugWithSource,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match self {
+            None => None,
+            Some(value) => Some(value.span),
+        }
+    }
+}
+
+impl<T> PrettyDebugWithSource for Option<Tagged<T>>
+where
+    Tagged<T>: PrettyDebugWithSource,
+{
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        match self {
+            None => b::description("nothing"),
+            Some(d) => d.pretty_debug(source),
+        }
+    }
+}
+
+impl<T> HasFallibleSpan for Option<Tagged<T>>
+where
+    Tagged<T>: PrettyDebugWithSource,
+{
+    fn maybe_span(&self) -> Option<Span> {
+        match self {
+            None => None,
+            Some(value) => Some(value.tag.span),
+        }
+    }
+}
+
+impl<T> HasSpan for Tagged<T>
+where
+    Tagged<T>: PrettyDebugWithSource,
+{
+    fn span(&self) -> Span {
+        self.tag.span
+    }
+}

crates/nu-source/src/pretty.rs

@@ -0,0 +1,495 @@
+use crate::term_colored::TermColored;
+use crate::text::Text;
+use derive_new::new;
+use pretty::{BoxAllocator, DocAllocator};
+use std::hash::Hash;
+use termcolor::{Color, ColorSpec};
+
+#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Hash)]
+pub enum ShellStyle {
+    Delimiter,
+    Key,
+    Value,
+    Equals,
+    Kind,
+    Keyword,
+    Operator,
+    Variable,
+    Primitive,
+    Opaque,
+    Description,
+    Error,
+}
+
+impl From<ShellAnnotation> for ColorSpec {
+    fn from(ann: ShellAnnotation) -> ColorSpec {
+        match ann.style {
+            ShellStyle::Delimiter => ColorSpec::new()
+                .set_fg(Some(Color::White))
+                .set_intense(false)
+                .clone(),
+            ShellStyle::Key => ColorSpec::new()
+                .set_fg(Some(Color::Black))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Value => ColorSpec::new()
+                .set_fg(Some(Color::White))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Equals => ColorSpec::new()
+                .set_fg(Some(Color::Black))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Kind => ColorSpec::new().set_fg(Some(Color::Cyan)).clone(),
+            ShellStyle::Variable => ColorSpec::new()
+                .set_fg(Some(Color::Green))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Keyword => ColorSpec::new().set_fg(Some(Color::Magenta)).clone(),
+            ShellStyle::Operator => ColorSpec::new().set_fg(Some(Color::Yellow)).clone(),
+            ShellStyle::Primitive => ColorSpec::new()
+                .set_fg(Some(Color::Green))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Opaque => ColorSpec::new()
+                .set_fg(Some(Color::Yellow))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Description => ColorSpec::new()
+                .set_fg(Some(Color::Black))
+                .set_intense(true)
+                .clone(),
+            ShellStyle::Error => ColorSpec::new()
+                .set_fg(Some(Color::Red))
+                .set_intense(true)
+                .clone(),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash, new)]
+pub struct ShellAnnotation {
+    style: ShellStyle,
+}
+
+impl std::fmt::Debug for ShellAnnotation {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self.style)
+    }
+}
+
+impl ShellAnnotation {
+    pub fn style(style: impl Into<ShellStyle>) -> ShellAnnotation {
+        ShellAnnotation {
+            style: style.into(),
+        }
+    }
+}
+
+pub type PrettyDebugDoc =
+    pretty::Doc<'static, pretty::BoxDoc<'static, ShellAnnotation>, ShellAnnotation>;
+
+pub type PrettyDebugDocBuilder = pretty::DocBuilder<'static, pretty::BoxAllocator, ShellAnnotation>;
+
+pub use self::DebugDocBuilder as b;
+
+#[derive(Clone, new)]
+pub struct DebugDocBuilder {
+    pub inner: PrettyDebugDocBuilder,
+}
+
+impl PrettyDebug for DebugDocBuilder {
+    fn pretty(&self) -> DebugDocBuilder {
+        self.clone()
+    }
+}
+
+impl std::ops::Add for DebugDocBuilder {
+    type Output = DebugDocBuilder;
+
+    fn add(self, rhs: DebugDocBuilder) -> DebugDocBuilder {
+        DebugDocBuilder::new(self.inner.append(rhs.inner))
+    }
+}
+
+impl DebugDocBuilder {
+    pub fn from_doc(doc: DebugDoc) -> DebugDocBuilder {
+        DebugDocBuilder {
+            inner: BoxAllocator.nil().append(doc),
+        }
+    }
+
+    pub fn blank() -> DebugDocBuilder {
+        BoxAllocator.nil().into()
+    }
+
+    pub fn delimiter(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Delimiter)
+    }
+
+    pub fn key(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Key)
+    }
+
+    pub fn value(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Value)
+    }
+
+    pub fn as_value(self) -> DebugDocBuilder {
+        self.inner
+            .annotate(ShellAnnotation::style(ShellStyle::Value))
+            .into()
+    }
+
+    pub fn equals() -> DebugDocBuilder {
+        DebugDocBuilder::styled("=", ShellStyle::Equals)
+    }
+
+    pub fn kind(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Kind)
+    }
+
+    pub fn as_kind(self) -> DebugDocBuilder {
+        self.inner
+            .annotate(ShellAnnotation::style(ShellStyle::Kind))
+            .into()
+    }
+
+    pub fn typed(kind: &str, value: DebugDocBuilder) -> DebugDocBuilder {
+        b::delimit("(", b::kind(kind) + b::space() + value.group(), ")").group()
+    }
+
+    pub fn subtyped(
+        kind: &str,
+        subkind: impl std::fmt::Display,
+        value: DebugDocBuilder,
+    ) -> DebugDocBuilder {
+        b::delimit(
+            "(",
+            (b::kind(kind) + b::delimit("[", b::kind(format!("{}", subkind)), "]")).group()
+                + b::space()
+                + value.group(),
+            ")",
+        )
+        .group()
+    }
+
+    pub fn keyword(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Keyword)
+    }
+
+    pub fn var(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Variable)
+    }
+
+    pub fn operator(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Operator)
+    }
+
+    pub fn primitive(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(format!("{}", string), ShellStyle::Primitive)
+    }
+
+    pub fn opaque(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Opaque)
+    }
+
+    pub fn description(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Description)
+    }
+
+    pub fn error(string: impl std::fmt::Display) -> DebugDocBuilder {
+        DebugDocBuilder::styled(string, ShellStyle::Error)
+    }
+
+    pub fn delimit(start: &str, doc: DebugDocBuilder, end: &str) -> DebugDocBuilder {
+        DebugDocBuilder::delimiter(start) + doc + DebugDocBuilder::delimiter(end)
+    }
+
+    pub fn preceded(before: DebugDocBuilder, body: DebugDocBuilder) -> DebugDocBuilder {
+        if body.is_empty() {
+            body
+        } else {
+            before + body
+        }
+    }
+
+    pub fn surrounded_option(
+        before: Option<DebugDocBuilder>,
+        builder: Option<DebugDocBuilder>,
+        after: Option<DebugDocBuilder>,
+    ) -> DebugDocBuilder {
+        match builder {
+            None => DebugDocBuilder::blank(),
+            Some(b) => b::option(before) + b + b::option(after),
+        }
+    }
+
+    pub fn preceded_option(
+        before: Option<DebugDocBuilder>,
+        builder: Option<DebugDocBuilder>,
+    ) -> DebugDocBuilder {
+        DebugDocBuilder::surrounded_option(before, builder, None)
+    }
+
+    pub fn option(builder: Option<DebugDocBuilder>) -> DebugDocBuilder {
+        match builder {
+            None => DebugDocBuilder::blank(),
+            Some(b) => b,
+        }
+    }
+
+    pub fn space() -> DebugDocBuilder {
+        BoxAllocator.space().into()
+    }
+
+    pub fn newline() -> DebugDocBuilder {
+        BoxAllocator.newline().into()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        match &self.inner.1 {
+            pretty::Doc::Nil => true,
+            _ => false,
+        }
+    }
+
+    pub fn or(self, doc: DebugDocBuilder) -> DebugDocBuilder {
+        if self.is_empty() {
+            doc
+        } else {
+            self
+        }
+    }
+
+    pub fn group(self) -> DebugDocBuilder {
+        self.inner.group().into()
+    }
+
+    pub fn nest(self) -> DebugDocBuilder {
+        self.inner.nest(1).group().into()
+    }
+
+    pub fn intersperse_with_source<'a, T: PrettyDebugWithSource + 'a>(
+        list: impl IntoIterator<Item = &'a T>,
+        separator: DebugDocBuilder,
+        source: &str,
+    ) -> DebugDocBuilder {
+        BoxAllocator
+            .intersperse(
+                list.into_iter().filter_map(|item| {
+                    let item = item.pretty_debug(source);
+                    if item.is_empty() {
+                        None
+                    } else {
+                        Some(item)
+                    }
+                }),
+                separator,
+            )
+            .into()
+    }
+
+    pub fn intersperse<T: PrettyDebug>(
+        list: impl IntoIterator<Item = T>,
+        separator: DebugDocBuilder,
+    ) -> DebugDocBuilder {
+        BoxAllocator
+            .intersperse(
+                list.into_iter().filter_map(|item| {
+                    let item = item.pretty();
+                    if item.is_empty() {
+                        None
+                    } else {
+                        Some(item)
+                    }
+                }),
+                separator,
+            )
+            .into()
+    }
+
+    pub fn list(list: impl IntoIterator<Item = DebugDocBuilder>) -> DebugDocBuilder {
+        let mut result: DebugDocBuilder = BoxAllocator.nil().into();
+
+        for item in list {
+            result = result + item;
+        }
+
+        result.into()
+    }
+
+    fn styled(string: impl std::fmt::Display, style: ShellStyle) -> DebugDocBuilder {
+        BoxAllocator
+            .text(string.to_string())
+            .annotate(ShellAnnotation::style(style))
+            .into()
+    }
+}
+
+impl std::ops::Deref for DebugDocBuilder {
+    type Target = PrettyDebugDocBuilder;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, new)]
+pub struct DebugDoc {
+    pub inner: PrettyDebugDoc,
+}
+
+pub trait PrettyDebugWithSource: Sized {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder;
+
+    // This is a transitional convenience method
+    fn debug(&self, source: impl Into<Text>) -> String
+    where
+        Self: Clone,
+    {
+        self.clone().debuggable(source).display()
+    }
+
+    fn debuggable(self, source: impl Into<Text>) -> DebuggableWithSource<Self> {
+        DebuggableWithSource {
+            inner: self,
+            source: source.into(),
+        }
+    }
+}
+
+impl<T: PrettyDebug> PrettyDebugWithSource for T {
+    fn pretty_debug(&self, _source: &str) -> DebugDocBuilder {
+        self.pretty()
+    }
+}
+
+pub struct DebuggableWithSource<T: PrettyDebugWithSource> {
+    inner: T,
+    source: Text,
+}
+
+impl<T> PrettyDebug for DebuggableWithSource<T>
+where
+    T: PrettyDebugWithSource,
+{
+    fn pretty(&self) -> DebugDocBuilder {
+        self.inner.pretty_debug(&self.source)
+    }
+}
+
+impl PrettyDebug for DebugDoc {
+    fn pretty(&self) -> DebugDocBuilder {
+        DebugDocBuilder::new(BoxAllocator.nil().append(self.inner.clone()))
+    }
+}
+
+pub trait PrettyDebug {
+    fn pretty(&self) -> DebugDocBuilder;
+
+    fn to_doc(&self) -> DebugDoc {
+        DebugDoc::new(self.pretty().into())
+    }
+
+    fn pretty_doc(&self) -> PrettyDebugDoc {
+        let builder = self.pretty();
+        builder.inner.into()
+    }
+
+    fn pretty_builder(&self) -> PrettyDebugDocBuilder {
+        let doc = self.pretty();
+        doc.inner
+    }
+
+    /// A convenience method that prints out the document without colors in
+    /// 70 columns. Generally, you should use plain_string or colored_string
+    /// if possible, but display() can be useful for trace lines and things
+    /// like that, where you don't have control over the terminal.
+    fn display(&self) -> String {
+        self.plain_string(70)
+    }
+
+    fn plain_string(&self, width: usize) -> String {
+        let doc = self.pretty_doc();
+        let mut buffer = termcolor::Buffer::no_color();
+
+        doc.render_raw(width, &mut TermColored::new(&mut buffer))
+            .unwrap();
+
+        String::from_utf8_lossy(buffer.as_slice()).to_string()
+    }
+
+    fn colored_string(&self, width: usize) -> String {
+        let doc = self.pretty_doc();
+        let mut buffer = termcolor::Buffer::ansi();
+
+        doc.render_raw(width, &mut TermColored::new(&mut buffer))
+            .unwrap();
+
+        String::from_utf8_lossy(buffer.as_slice()).to_string()
+    }
+}
+
+impl Into<DebugDocBuilder> for PrettyDebugDocBuilder {
+    fn into(self) -> DebugDocBuilder {
+        DebugDocBuilder { inner: self }
+    }
+}
+
+impl std::ops::Deref for DebugDoc {
+    type Target = PrettyDebugDoc;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl From<DebugDoc> for PrettyDebugDoc {
+    fn from(input: DebugDoc) -> PrettyDebugDoc {
+        input.inner
+    }
+}
+
+impl Into<PrettyDebugDoc> for DebugDocBuilder {
+    fn into(self) -> PrettyDebugDoc {
+        self.inner.into()
+    }
+}
+
+fn hash_doc<H: std::hash::Hasher>(doc: &PrettyDebugDoc, state: &mut H) {
+    match doc {
+        pretty::Doc::Nil => 0u8.hash(state),
+        pretty::Doc::Append(a, b) => {
+            1u8.hash(state);
+            hash_doc(&*a, state);
+            hash_doc(&*b, state);
+        }
+        pretty::Doc::Group(a) => {
+            2u8.hash(state);
+            hash_doc(&*a, state);
+        }
+        pretty::Doc::Nest(a, b) => {
+            3u8.hash(state);
+            a.hash(state);
+            hash_doc(&*b, state);
+        }
+        pretty::Doc::Space => 4u8.hash(state),
+        pretty::Doc::Newline => 5u8.hash(state),
+        pretty::Doc::Text(t) => {
+            6u8.hash(state);
+            t.hash(state);
+        }
+        pretty::Doc::Annotated(a, b) => {
+            7u8.hash(state);
+            a.hash(state);
+            hash_doc(&*b, state);
+        }
+    }
+}
+
+impl std::hash::Hash for DebugDoc {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        hash_doc(&self.inner, state);
+    }
+}

crates/nu-source/src/term_colored.rs

@@ -0,0 +1,51 @@
+use crate::pretty::ShellAnnotation;
+use pretty::{Render, RenderAnnotated};
+use std::io;
+use termcolor::WriteColor;
+
+pub struct TermColored<'a, W> {
+    color_stack: Vec<ShellAnnotation>,
+    upstream: &'a mut W,
+}
+
+impl<'a, W> TermColored<'a, W> {
+    pub fn new(upstream: &'a mut W) -> TermColored<'a, W> {
+        TermColored {
+            color_stack: Vec::new(),
+            upstream,
+        }
+    }
+}
+
+impl<'a, W> Render for TermColored<'a, W>
+where
+    W: io::Write,
+{
+    type Error = io::Error;
+
+    fn write_str(&mut self, s: &str) -> io::Result<usize> {
+        self.upstream.write(s.as_bytes())
+    }
+
+    fn write_str_all(&mut self, s: &str) -> io::Result<()> {
+        self.upstream.write_all(s.as_bytes())
+    }
+}
+
+impl<'a, W> RenderAnnotated<ShellAnnotation> for TermColored<'a, W>
+where
+    W: WriteColor,
+{
+    fn push_annotation(&mut self, ann: &ShellAnnotation) -> Result<(), Self::Error> {
+        self.color_stack.push(*ann);
+        self.upstream.set_color(&(*ann).into())
+    }
+
+    fn pop_annotation(&mut self) -> Result<(), Self::Error> {
+        self.color_stack.pop();
+        match self.color_stack.last() {
+            Some(previous) => self.upstream.set_color(&(*previous).into()),
+            None => self.upstream.reset(),
+        }
+    }
+}

crates/nu-source/src/text.rs

@@ -0,0 +1,229 @@
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use std::cmp::Ordering;
+use std::hash::Hash;
+use std::hash::Hasher;
+use std::ops::Range;
+use std::sync::Arc;
+
+/// A "Text" is like a string except that it can be cheaply cloned.
+/// You can also "extract" subtexts quite cheaply. You can also deref
+/// an `&Text` into a `&str` for interoperability.
+///
+/// Used to represent the value of an input file.
+#[derive(Clone)]
+pub struct Text {
+    text: Arc<String>,
+    start: usize,
+    end: usize,
+}
+
+impl Text {
+    /// Modifies this restrict to a subset of its current range.
+    pub fn select(&mut self, range: Range<usize>) {
+        let len = range.end - range.start;
+        let new_start = self.start + range.start;
+        let new_end = new_start + len;
+        assert!(new_end <= self.end);
+
+        self.start = new_start;
+        self.end = new_end;
+    }
+
+    /// Extract a new `Text` that is a subset of an old `Text`
+    /// -- `text.extract(1..3)` is similar to `&foo[1..3]` except that
+    /// it gives back an owned value instead of a borrowed value.
+    pub fn slice(&self, range: Range<usize>) -> Self {
+        let mut result = self.clone();
+        result.select(range);
+        result
+    }
+}
+
+impl From<Arc<String>> for Text {
+    fn from(text: Arc<String>) -> Self {
+        let end = text.len();
+        Self {
+            text,
+            start: 0,
+            end,
+        }
+    }
+}
+
+impl AsRef<str> for Text {
+    fn as_ref(&self) -> &str {
+        &*self
+    }
+}
+
+impl From<String> for Text {
+    fn from(text: String) -> Self {
+        Text::from(Arc::new(text))
+    }
+}
+
+impl From<&String> for Text {
+    fn from(text: &String) -> Self {
+        Text::from(text.to_string())
+    }
+}
+
+impl From<&str> for Text {
+    fn from(text: &str) -> Self {
+        Text::from(text.to_string())
+    }
+}
+
+impl From<&Text> for Text {
+    fn from(text: &Text) -> Self {
+        text.clone()
+    }
+}
+
+impl std::borrow::Borrow<str> for Text {
+    fn borrow(&self) -> &str {
+        &*self
+    }
+}
+
+impl std::ops::Deref for Text {
+    type Target = str;
+
+    fn deref(&self) -> &str {
+        &self.text[self.start..self.end]
+    }
+}
+
+impl std::fmt::Display for Text {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        <str as std::fmt::Display>::fmt(self, fmt)
+    }
+}
+
+impl std::fmt::Debug for Text {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        <str as std::fmt::Debug>::fmt(self, fmt)
+    }
+}
+
+impl PartialEq<Text> for Text {
+    fn eq(&self, other: &Text) -> bool {
+        let this: &str = self;
+        let other: &str = other;
+        this == other
+    }
+}
+
+impl Eq for Text {}
+
+impl PartialEq<str> for Text {
+    fn eq(&self, other: &str) -> bool {
+        let this: &str = self;
+        this == other
+    }
+}
+
+impl PartialEq<String> for Text {
+    fn eq(&self, other: &String) -> bool {
+        let this: &str = self;
+        let other: &str = other;
+        this == other
+    }
+}
+
+impl PartialEq<Text> for str {
+    fn eq(&self, other: &Text) -> bool {
+        other == self
+    }
+}
+
+impl PartialEq<Text> for String {
+    fn eq(&self, other: &Text) -> bool {
+        other == self
+    }
+}
+
+impl<T: ?Sized> PartialEq<&T> for Text
+where
+    Text: PartialEq<T>,
+{
+    fn eq(&self, other: &&T) -> bool {
+        self == *other
+    }
+}
+
+impl Hash for Text {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        <str as Hash>::hash(self, state)
+    }
+}
+
+impl PartialOrd<Text> for Text {
+    fn partial_cmp(&self, other: &Text) -> Option<Ordering> {
+        let this: &str = self;
+        let other: &str = other;
+        this.partial_cmp(other)
+    }
+}
+
+impl Ord for Text {
+    fn cmp(&self, other: &Text) -> Ordering {
+        let this: &str = self;
+        let other: &str = other;
+        this.cmp(other)
+    }
+}
+
+impl PartialOrd<str> for Text {
+    fn partial_cmp(&self, other: &str) -> Option<Ordering> {
+        let this: &str = self;
+        this.partial_cmp(other)
+    }
+}
+
+impl PartialOrd<String> for Text {
+    fn partial_cmp(&self, other: &String) -> Option<Ordering> {
+        let this: &str = self;
+        let other: &str = other;
+        this.partial_cmp(other)
+    }
+}
+
+impl PartialOrd<Text> for str {
+    fn partial_cmp(&self, other: &Text) -> Option<Ordering> {
+        other.partial_cmp(self)
+    }
+}
+
+impl PartialOrd<Text> for String {
+    fn partial_cmp(&self, other: &Text) -> Option<Ordering> {
+        other.partial_cmp(self)
+    }
+}
+
+impl<T: ?Sized> PartialOrd<&T> for Text
+where
+    Text: PartialOrd<T>,
+{
+    fn partial_cmp(&self, other: &&T) -> Option<Ordering> {
+        self.partial_cmp(*other)
+    }
+}
+
+impl Serialize for Text {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        self.as_ref().serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for Text {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Ok(Text::from(String::deserialize(deserializer)?))
+    }
+}

crates/nu-source/src/tracable.rs

@@ -0,0 +1,32 @@
+use derive_new::new;
+use nom_locate::LocatedSpanEx;
+use nom_tracable::{HasTracableInfo, TracableInfo};
+
+pub type NomSpan<'a> = LocatedSpanEx<&'a str, TracableContext>;
+
+#[derive(Debug, Clone, Copy, PartialEq, new)]
+pub struct TracableContext {
+    pub(crate) info: TracableInfo,
+}
+
+impl HasTracableInfo for TracableContext {
+    fn get_tracable_info(&self) -> TracableInfo {
+        self.info
+    }
+
+    fn set_tracable_info(self, info: TracableInfo) -> Self {
+        TracableContext { info }
+    }
+}
+
+impl std::ops::Deref for TracableContext {
+    type Target = TracableInfo;
+
+    fn deref(&self) -> &TracableInfo {
+        &self.info
+    }
+}
+
+pub fn nom_input(s: &str) -> NomSpan<'_> {
+    LocatedSpanEx::new_extra(s, TracableContext::new(TracableInfo::new()))
+}