Add support for ~ expansion

This ended up being a bit of a yak shave. The basic idea in this commit is to
expand `~` in paths, but only in paths.

The way this is accomplished is by doing the expansion inside of the code that
parses literal syntax for `SyntaxType::Path`.

As a quick refresher: every command is entitled to expand its arguments in a
custom way. While this could in theory be used for general-purpose macros,
today the expansion facility is limited to syntactic hints.

For example, the syntax `where cpu > 0` expands under the hood to
`where { $it.cpu > 0 }`. This happens because the first argument to `where`
is defined as a `SyntaxType::Block`, and the parser coerces binary expressions
whose left-hand-side looks like a member into a block when the command is
expecting one.

This is mildly more magical than what most programming languages would do,
but we believe that it makes sense to allow commands to fine-tune the syntax
because of the domain nushell is in (command-line shells).

The syntactic expansions supported by this facility are relatively limited.
For example, we don't allow `$it` to become a bare word, simply because the
command asks for a string in the relevant position. That would quickly
become more confusing than it's worth.

This PR adds a new `SyntaxType` rule: `SyntaxType::Path`. When a command
declares a parameter as a `SyntaxType::Path`, string literals and bare
words passed as an argument to that parameter are processed using the
path expansion rules. Right now, that only means that `~` is expanded into
the home directory, but additional rules are possible in the future.

By restricting this expansion to a syntactic expansion when passed as an
argument to a command expecting a path, we avoid making `~` a generally
reserved character. This will also allow us to give good tab completion
for paths with `~` characters in them when a command is expecting a path.

In order to accomplish the above, this commit changes the parsing functions
to take a `Context` instead of just a `CommandRegistry`. From the perspective
of macro expansion, you can think of the `CommandRegistry` as a dictionary
of in-scope macros, and the `Context` as the compile-time state used in
expansion. This could gain additional functionality over time as we find
more uses for the expansion system.

src/commands/classified.rs

@@ -122,10 +122,11 @@ impl InternalCommand {
             self.name_span.clone(),
             context.source_map.clone(),
             self.args,
-            source,
+            &source,
             objects,
         );
 
+        let result = trace_out_stream!(target: "nu::trace_stream::internal", source: &source, "output" = result);
         let mut result = result.values;
 
         let mut stream = VecDeque::new();

src/commands/command.rs

@@ -422,6 +422,25 @@ pub enum CommandAction {
     LeaveShell,
 }
 
+impl ToDebug for CommandAction {
+    fn fmt_debug(&self, f: &mut fmt::Formatter, _source: &str) -> fmt::Result {
+        match self {
+            CommandAction::ChangePath(s) => write!(f, "action:change-path={}", s),
+            CommandAction::AddSpanSource(u, source) => {
+                write!(f, "action:add-span-source={}@{:?}", u, source)
+            }
+            CommandAction::Exit => write!(f, "action:exit"),
+            CommandAction::EnterShell(s) => write!(f, "action:enter-shell={}", s),
+            CommandAction::EnterValueShell(t) => {
+                write!(f, "action:enter-value-shell={:?}", t.debug())
+            }
+            CommandAction::PreviousShell => write!(f, "action:previous-shell"),
+            CommandAction::NextShell => write!(f, "action:next-shell"),
+            CommandAction::LeaveShell => write!(f, "action:leave-shell"),
+        }
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub enum ReturnSuccess {
     Value(Tagged<Value>),
@@ -430,6 +449,16 @@ pub enum ReturnSuccess {
 
 pub type ReturnValue = Result<ReturnSuccess, ShellError>;
 
+impl ToDebug for ReturnValue {
+    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+        match self {
+            Err(err) => write!(f, "{}", err.debug(source)),
+            Ok(ReturnSuccess::Value(v)) => write!(f, "{:?}", v.debug()),
+            Ok(ReturnSuccess::Action(a)) => write!(f, "{}", a.debug(source)),
+        }
+    }
+}
+
 impl From<Tagged<Value>> for ReturnValue {
     fn from(input: Tagged<Value>) -> ReturnValue {
         Ok(ReturnSuccess::Value(input))
@@ -469,7 +498,7 @@ pub trait WholeStreamCommand: Send + Sync {
         Signature {
             name: self.name().to_string(),
             positional: vec![],
-            rest_positional: true,
+            rest_positional: None,
             named: indexmap::IndexMap::new(),
             is_filter: true,
         }
@@ -491,7 +520,7 @@ pub trait PerItemCommand: Send + Sync {
         Signature {
             name: self.name().to_string(),
             positional: vec![],
-            rest_positional: true,
+            rest_positional: None,
             named: indexmap::IndexMap::new(),
             is_filter: true,
         }

src/commands/get.rs

@@ -22,33 +22,41 @@ impl WholeStreamCommand for Get {
     ) -> Result<OutputStream, ShellError> {
         args.process(registry, get)?.run()
     }
+
     fn signature(&self) -> Signature {
-        Signature::build("get").rest()
+        Signature::build("get").rest(SyntaxType::Member)
     }
 }
 
 fn get_member(path: &Tagged<String>, obj: &Tagged<Value>) -> Result<Tagged<Value>, ShellError> {
-    let mut current = obj;
+    let mut current = Some(obj);
     for p in path.split(".") {
-        match current.get_data_by_key(p) {
-            Some(v) => current = v,
-            None => {
+        if let Some(obj) = current {
+            current = match obj.get_data_by_key(p) {
+                Some(v) => Some(v),
+                None =>
                 // Before we give up, see if they gave us a path that matches a field name by itself
-                match obj.get_data_by_key(&path.item) {
-                    Some(v) => return Ok(v.clone()),
-                    None => {
-                        return Err(ShellError::labeled_error(
-                            "Unknown column",
-                            "table missing column",
-                            path.span(),
-                        ));
+                {
+                    match obj.get_data_by_key(&path.item) {
+                        Some(v) => return Ok(v.clone()),
+                        None => {
+                            return Err(ShellError::labeled_error(
+                                "Unknown column",
+                                "table missing column",
+                                path.span(),
+                            ));
+                        }
                     }
                 }
             }
         }
     }
 
-    Ok(current.clone())
+    match current {
+        Some(v) => Ok(v.clone()),
+        None => Ok(Value::nothing().tagged(obj.tag)),
+    }
+    // Ok(current.clone())
 }
 
 pub fn get(

src/commands/mkdir.rs

@@ -27,7 +27,7 @@ impl PerItemCommand for Mkdir {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("mkdir").rest()
+        Signature::build("mkdir").rest(SyntaxType::Path)
     }
 }
 

src/commands/open.rs

@@ -44,7 +44,8 @@ fn run(call_info: &CallInfo, shell_manager: &ShellManager) -> Result<OutputStrea
     {
         file => file,
     };
-    let path_str = path.as_string()?;
+    let path_buf = path.as_path()?;
+    let path_str = path_buf.display().to_string();
     let path_span = path.span();
     let name_span = call_info.name_span;
     let has_raw = call_info.args.has("raw");
@@ -426,14 +427,16 @@ pub fn parse_string_as_value(
     name_span: Span,
 ) -> Result<Tagged<Value>, ShellError> {
     match extension {
-        Some(x) if x == "csv" => crate::commands::from_csv::from_csv_string_to_value(
-            contents,
-            false,
-            contents_tag,
-        )
-        .map_err(move |_| {
-            ShellError::labeled_error("Could not open as CSV", "could not open as CSV", name_span)
-        }),
+        Some(x) if x == "csv" => {
+            crate::commands::from_csv::from_csv_string_to_value(contents, false, contents_tag)
+                .map_err(move |_| {
+                    ShellError::labeled_error(
+                        "Could not open as CSV",
+                        "could not open as CSV",
+                        name_span,
+                    )
+                })
+        }
         Some(x) if x == "toml" => {
             crate::commands::from_toml::from_toml_string_to_value(contents, contents_tag).map_err(
                 move |_| {
@@ -507,9 +510,9 @@ pub fn parse_binary_as_value(
             crate::commands::from_bson::from_bson_bytes_to_value(contents, contents_tag).map_err(
                 move |_| {
                     ShellError::labeled_error(
-                       "Could not open as BSON",
-                       "could not open as BSON",
-                       name_span,
+                        "Could not open as BSON",
+                        "could not open as BSON",
+                        name_span,
                     )
                 },
             )

src/commands/pick.rs

@@ -17,7 +17,7 @@ impl WholeStreamCommand for Pick {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("pick").rest()
+        Signature::build("pick").rest(SyntaxType::Any)
     }
 
     fn run(

src/commands/plugin.rs

@@ -3,6 +3,7 @@ use crate::errors::ShellError;
 use crate::parser::registry;
 use crate::prelude::*;
 use derive_new::new;
+use log::trace;
 use serde::{self, Deserialize, Serialize};
 use std::io::prelude::*;
 use std::io::BufReader;
@@ -64,6 +65,8 @@ pub fn filter_plugin(
     args: CommandArgs,
     registry: &CommandRegistry,
 ) -> Result<OutputStream, ShellError> {
+    trace!("filter_plugin :: {}", path);
+
     let args = args.evaluate_once(registry)?;
 
     let mut child = std::process::Command::new(path)
@@ -80,6 +83,8 @@ pub fn filter_plugin(
 
     let call_info = args.call_info.clone();
 
+    trace!("filtering :: {:?}", call_info);
+
     let stream = bos
         .chain(args.input.values)
         .chain(eos)
@@ -95,7 +100,14 @@ pub fn filter_plugin(
 
                 let request = JsonRpc::new("begin_filter", call_info.clone());
                 let request_raw = serde_json::to_string(&request).unwrap();
-                let _ = stdin.write(format!("{}\n", request_raw).as_bytes()); // TODO: Handle error
+                match stdin.write(format!("{}\n", request_raw).as_bytes()) {
+                    Ok(_) => {}
+                    Err(err) => {
+                        let mut result = VecDeque::new();
+                        result.push_back(Err(ShellError::unexpected(format!("{}", err))));
+                        return result;
+                    }
+                }
 
                 let mut input = String::new();
                 match reader.read_line(&mut input) {
@@ -140,7 +152,15 @@ pub fn filter_plugin(
                 let mut reader = BufReader::new(stdout);
 
                 let request: JsonRpc<std::vec::Vec<Value>> = JsonRpc::new("end_filter", vec![]);
-                let request_raw = serde_json::to_string(&request).unwrap();
+                let request_raw = match serde_json::to_string(&request) {
+                    Ok(req) => req,
+                    Err(err) => {
+                        let mut result = VecDeque::new();
+                        result.push_back(Err(ShellError::unexpected(format!("{}", err))));
+                        return result;
+                    }
+                };
+
                 let _ = stdin.write(format!("{}\n", request_raw).as_bytes()); // TODO: Handle error
 
                 let mut input = String::new();

src/commands/reject.rs

@@ -24,7 +24,7 @@ impl WholeStreamCommand for Reject {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("reject").rest()
+        Signature::build("reject").rest(SyntaxType::Member)
     }
 }
 

src/commands/sort_by.rs

@@ -4,13 +4,19 @@ use crate::prelude::*;
 
 pub struct SortBy;
 
+#[derive(Deserialize)]
+pub struct SortByArgs {
+    rest: Vec<Tagged<String>>,
+    reverse: bool,
+}
+
 impl WholeStreamCommand for SortBy {
     fn run(
         &self,
         args: CommandArgs,
         registry: &CommandRegistry,
     ) -> Result<OutputStream, ShellError> {
-        sort_by(args, registry)
+        args.process(registry, sort_by)?.run()
     }
 
     fn name(&self) -> &str {
@@ -18,43 +24,32 @@ impl WholeStreamCommand for SortBy {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("sort-by").switch("reverse")
+        Signature::build("sort-by")
+            .rest(SyntaxType::String)
+            .switch("reverse")
     }
 }
 
-fn sort_by(args: CommandArgs, registry: &CommandRegistry) -> Result<OutputStream, ShellError> {
-    let args = args.evaluate_once(registry)?;
-    let (input, args) = args.parts();
+fn sort_by(
+    SortByArgs { reverse, rest }: SortByArgs,
+    mut context: RunnableContext,
+) -> Result<OutputStream, ShellError> {
+    Ok(OutputStream::new(async_stream_block! {
+        let mut vec = context.input.drain_vec().await;
 
-    let fields: Result<Vec<_>, _> = args
-        .positional
-        .iter()
-        .flatten()
-        .map(|a| a.as_string())
-        .collect();
-
-    let fields = fields?;
-
-    let output = input.values.collect::<Vec<_>>();
-
-    let reverse = args.has("reverse");
-    let output = output.map(move |mut vec| {
         let calc_key = |item: &Tagged<Value>| {
-            fields
-                .iter()
+            rest.iter()
                 .map(|f| item.get_data_by_key(f).map(|i| i.clone()))
                 .collect::<Vec<Option<Tagged<Value>>>>()
         };
         if reverse {
-            vec.sort_by_cached_key(|item| {
-                std::cmp::Reverse(calc_key(item))
-            });
+            vec.sort_by_cached_key(|item| std::cmp::Reverse(calc_key(item)));
         } else {
             vec.sort_by_cached_key(calc_key);
+        };
+
+        for item in vec {
+            yield item.into();
         }
-
-        vec.into_iter().collect::<VecDeque<_>>()
-    });
-
-    Ok(output.flatten_stream().from_input_stream())
+    }))
 }

src/commands/split_column.rs

@@ -28,7 +28,7 @@ impl WholeStreamCommand for SplitColumn {
     fn signature(&self) -> Signature {
         Signature::build("split-column")
             .required("separator", SyntaxType::Any)
-            .rest()
+            .rest(SyntaxType::Member)
     }
 }