Extract nu_source into a crate

This commit extracts Tag, Span, Text, as well as source-related debug
facilities into a new crate called nu_source.

This change is much bigger than one might have expected because the
previous code relied heavily on implementing inherent methods on
`Tagged<T>` and `Spanned<T>`, which is no longer possible.

As a result, this change creates more concrete types instead of using
`Tagged<T>`. One notable example: Tagged<Value> became Value, and Value
became UntaggedValue.

This change clarifies the intent of the code in many places, but it does
make it a big change.

src/shell/completer.rs

@@ -1,4 +1,5 @@
-use crate::prelude::*;
+use crate::context::CommandRegistry;
+
 use derive_new::new;
 use rustyline::completion::{Completer, FilenameCompleter};
 

src/shell/filesystem_shell.rs

@@ -8,6 +8,7 @@ use crate::prelude::*;
 use crate::shell::completer::NuCompleter;
 use crate::shell::shell::Shell;
 use crate::utils::FileStructure;
+use nu_source::Tagged;
 use rustyline::completion::FilenameCompleter;
 use rustyline::hint::{Hinter, HistoryHinter};
 use std::path::{Path, PathBuf};
@@ -1098,8 +1099,8 @@ impl Shell for FilesystemShell {
 
         let mut stream = VecDeque::new();
         stream.push_back(ReturnSuccess::value(
-            Value::Primitive(Primitive::String(p.to_string_lossy().to_string()))
-                .tagged(&args.call_info.name_tag),
+            UntaggedValue::Primitive(Primitive::String(p.to_string_lossy().to_string()))
+                .into_value(&args.call_info.name_tag),
         ));
 
         Ok(stream.into())

src/shell/help_shell.rs

@@ -6,13 +6,14 @@ use crate::commands::rm::RemoveArgs;
 use crate::data::{command_dict, TaggedDictBuilder};
 use crate::prelude::*;
 use crate::shell::shell::Shell;
+use nu_source::Tagged;
 use std::ffi::OsStr;
 use std::path::PathBuf;
 
 #[derive(Clone, Debug)]
 pub struct HelpShell {
     pub(crate) path: String,
-    pub(crate) value: Tagged<Value>,
+    pub(crate) value: Value,
 }
 
 impl HelpShell {
@@ -24,8 +25,8 @@ impl HelpShell {
             let mut spec = TaggedDictBuilder::new(Tag::unknown());
             let value = command_dict(registry.get_command(&cmd).unwrap(), Tag::unknown());
 
-            spec.insert("name", cmd);
-            spec.insert(
+            spec.insert_untagged("name", cmd);
+            spec.insert_untagged(
                 "description",
                 value
                     .get_data_by_key("usage".spanned_unknown())
@@ -33,27 +34,27 @@ impl HelpShell {
                     .as_string()
                     .unwrap(),
             );
-            spec.insert_tagged("details", value);
+            spec.insert_value("details", value);
 
-            specs.push(spec.into_tagged_value());
+            specs.push(spec.into_value());
         }
 
-        cmds.insert("help", Value::Table(specs));
+        cmds.insert_untagged("help", UntaggedValue::Table(specs));
 
         Ok(HelpShell {
             path: "/help".to_string(),
-            value: cmds.into_tagged_value(),
+            value: cmds.into_value(),
         })
     }
 
     pub fn for_command(
-        cmd: Tagged<Value>,
+        cmd: Value,
         registry: &CommandRegistry,
     ) -> Result<HelpShell, std::io::Error> {
         let mut sh = HelpShell::index(&registry)?;
 
-        if let Tagged {
-            item: Value::Primitive(Primitive::String(name)),
+        if let Value {
+            value: UntaggedValue::Primitive(Primitive::String(name)),
             ..
         } = cmd
         {
@@ -63,7 +64,7 @@ impl HelpShell {
         Ok(sh)
     }
 
-    fn commands(&self) -> VecDeque<Tagged<Value>> {
+    fn commands(&self) -> VecDeque<Value> {
         let mut cmds = VecDeque::new();
         let full_path = PathBuf::from(&self.path);
 
@@ -83,8 +84,8 @@ impl HelpShell {
             }
         }
         match viewed {
-            Tagged {
-                item: Value::Table(l),
+            Value {
+                value: UntaggedValue::Table(l),
                 ..
             } => {
                 for item in l {
@@ -107,7 +108,7 @@ impl Shell for HelpShell {
             "{}",
             match anchor_name {
                 Some(x) => format!("{{{}}}", x),
-                None => format!("<{}>", self.value.item.type_name(),),
+                None => format!("<{}>", self.value.type_name()),
             }
         )
     }
@@ -197,8 +198,8 @@ impl Shell for HelpShell {
         let commands = self.commands();
         for cmd in commands {
             match cmd {
-                Tagged { item, .. } => {
-                    for desc in item.data_descriptors() {
+                Value { value, .. } => {
+                    for desc in value.data_descriptors() {
                         possible_completion.push(desc);
                     }
                 }

src/shell/helper.rs

@@ -1,11 +1,10 @@
 use crate::context::Context;
 use crate::parser::hir::syntax_shape::{color_fallible_syntax, FlatShape, PipelineShape};
 use crate::parser::hir::TokensIterator;
-use crate::parser::nom_input;
 use crate::parser::parse::token_tree::TokenNode;
-use crate::{HasSpan, Spanned, SpannedItem, Tag, Tagged, Text};
 use ansi_term::Color;
 use log::{log_enabled, trace};
+use nu_source::{nom_input, HasSpan, Spanned, Tag, Tagged, Text};
 use rustyline::completion::Completer;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::Highlighter;
@@ -75,8 +74,8 @@ impl Highlighter for Helper {
                     Ok(v) => v,
                 };
 
-                let tokens = vec![TokenNode::Pipeline(pipeline.clone().spanned(v.span()))];
-                let mut tokens = TokensIterator::all(&tokens[..], v.span());
+                let tokens = vec![TokenNode::Pipeline(pipeline.clone())];
+                let mut tokens = TokensIterator::all(&tokens[..], Text::from(line), v.span());
 
                 let text = Text::from(line);
                 let expand_context = self.context.expand_context(&text);

src/shell/shell.rs

@@ -6,6 +6,7 @@ use crate::commands::rm::RemoveArgs;
 use crate::errors::ShellError;
 use crate::prelude::*;
 use crate::stream::OutputStream;
+use nu_source::Tagged;
 use std::path::PathBuf;
 
 pub trait Shell: std::fmt::Debug {

src/shell/shell_manager.rs

@@ -8,6 +8,7 @@ use crate::prelude::*;
 use crate::shell::filesystem_shell::FilesystemShell;
 use crate::shell::shell::Shell;
 use crate::stream::OutputStream;
+use nu_source::Tagged;
 use std::error::Error;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicUsize, Ordering};

src/shell/value_shell.rs

@@ -6,6 +6,7 @@ use crate::commands::rm::RemoveArgs;
 use crate::prelude::*;
 use crate::shell::shell::Shell;
 use crate::utils::ValueStructure;
+use nu_source::Tagged;
 use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
 
@@ -13,7 +14,7 @@ use std::path::{Path, PathBuf};
 pub struct ValueShell {
     pub(crate) path: String,
     pub(crate) last_path: String,
-    pub(crate) value: Tagged<Value>,
+    pub(crate) value: Value,
 }
 
 impl std::fmt::Debug for ValueShell {
@@ -23,7 +24,7 @@ impl std::fmt::Debug for ValueShell {
 }
 
 impl ValueShell {
-    pub fn new(value: Tagged<Value>) -> ValueShell {
+    pub fn new(value: Value) -> ValueShell {
         ValueShell {
             path: "/".to_string(),
             last_path: "/".to_string(),
@@ -31,7 +32,7 @@ impl ValueShell {
         }
     }
 
-    fn members_under(&self, path: &Path) -> VecDeque<Tagged<Value>> {
+    fn members_under(&self, path: &Path) -> VecDeque<Value> {
         let mut shell_entries = VecDeque::new();
         let full_path = path.to_path_buf();
         let mut viewed = self.value.clone();
@@ -49,8 +50,8 @@ impl ValueShell {
             }
         }
         match viewed {
-            Tagged {
-                item: Value::Table(l),
+            Value {
+                value: UntaggedValue::Table(l),
                 ..
             } => {
                 for item in l {
@@ -65,7 +66,7 @@ impl ValueShell {
         shell_entries
     }
 
-    fn members(&self) -> VecDeque<Tagged<Value>> {
+    fn members(&self) -> VecDeque<Value> {
         self.members_under(Path::new("."))
     }
 }
@@ -77,7 +78,7 @@ impl Shell for ValueShell {
             "{}",
             match anchor_name {
                 Some(x) => format!("{{{}}}", x),
-                None => format!("<{}>", self.value.item.type_name(),),
+                None => format!("<{}>", self.value.type_name()),
             }
         )
     }
@@ -215,7 +216,7 @@ impl Shell for ValueShell {
     fn pwd(&self, args: EvaluatedWholeStreamCommandArgs) -> Result<OutputStream, ShellError> {
         let mut stream = VecDeque::new();
         stream.push_back(ReturnSuccess::value(
-            Value::string(self.path()).tagged(&args.call_info.name_tag),
+            UntaggedValue::string(self.path()).into_value(&args.call_info.name_tag),
         ));
         Ok(stream.into())
     }
@@ -237,8 +238,8 @@ impl Shell for ValueShell {
         let members = self.members();
         for member in members {
             match member {
-                Tagged { item, .. } => {
-                    for desc in item.data_descriptors() {
+                Value { value, .. } => {
+                    for desc in value.data_descriptors() {
                         possible_completion.push(desc);
                     }
                 }