Add support for external escape valve (^dir)

This commit makes it possible to force nu to treat a command as an external command by prefixing it with `^`. For example `^dir` will force `dir` to run an external command, even if `dir` is also a registered nu command.

This ensures that users don't need to leave nu just because we happened to use a command they need.

This commit adds a new token type for external commands, which, among other things, makes it pretty straight forward to syntax highlight external commands uniquely, and generally to treat them as special.

src/cli.rs

@@ -11,7 +11,7 @@ crate use crate::errors::ShellError;
 use crate::git::current_branch;
 use crate::object::Value;
 use crate::parser::registry::Signature;
-use crate::parser::{hir, Pipeline, PipelineElement, TokenNode};
+use crate::parser::{hir, CallNode, Pipeline, PipelineElement, TokenNode};
 use crate::prelude::*;
 
 use log::{debug, trace};
@@ -158,7 +158,6 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
             command("cd", Box::new(cd::cd)),
             command("size", Box::new(size::size)),
             command("from-yaml", Box::new(from_yaml::from_yaml)),
-            //command("enter", Box::new(enter::enter)),
             command("nth", Box::new(nth::nth)),
             command("n", Box::new(next::next)),
             command("p", Box::new(prev::prev)),
@@ -201,7 +200,6 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
     let _ = load_plugins(&mut context);
 
     let config = Config::builder().color_mode(ColorMode::Forced).build();
-    //let h = crate::shell::Helper::new(context.clone_commands());
     let mut rl: Editor<_> = Editor::with_config(config);
 
     #[cfg(windows)]
@@ -209,7 +207,6 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
         let _ = ansi_term::enable_ansi_support();
     }
 
-    //rl.set_helper(Some(h));
     let _ = rl.load_history("history.txt");
 
     let ctrl_c = Arc::new(AtomicBool::new(false));
@@ -477,11 +474,21 @@ fn classify_command(
     let call = command.call();
 
     match call {
+        // If the command starts with `^`, treat it as an external command no matter what
+        call if call.head().is_external() => {
+            let name_span = call.head().expect_external();
+            let name = name_span.slice(source);
+
+            Ok(external_command(call, source, name.tagged(name_span)))
+        }
+
+        // Otherwise, if the command is a bare word, we'll need to triage it
         call if call.head().is_bare() => {
             let head = call.head();
             let name = head.source(source);
 
             match context.has_command(name) {
+                // if the command is in the registry, it's an internal command
                 true => {
                     let command = context.get_command(name);
                     let config = command.signature();
@@ -496,9 +503,27 @@ fn classify_command(
                         args,
                     }))
                 }
-                false => {
+
+                // otherwise, it's an external command
+                false => Ok(external_command(call, source, name.tagged(head.span()))),
+            }
+        }
+
+        // If the command is something else (like a number or a variable), that is currently unsupported.
+        // We might support `$somevar` as a curried command in the future.
+        call => Err(ShellError::invalid_command(call.head().span())),
+    }
+}
+
+// Classify this command as an external command, which doesn't give special meaning
+// to nu syntactic constructs, and passes all arguments to the external command as
+// strings.
+fn external_command(
+    call: &Tagged<CallNode>,
+    source: &Text,
+    name: Tagged<&str>,
+) -> ClassifiedCommand {
     let arg_list_strings: Vec<Tagged<String>> = match call.children() {
-                        //Some(args) => args.iter().map(|i| i.as_external_arg(source)).collect(),
         Some(args) => args
             .iter()
             .filter_map(|i| match i {
@@ -512,21 +537,11 @@ fn classify_command(
         None => vec![],
     };
 
-                    Ok(ClassifiedCommand::External(ExternalCommand {
+    let (name, tag) = name.into_parts();
-                        name: name.to_string(),
-                        name_span: head.span().clone(),
-                        args: arg_list_strings,
-                    }))
-                }
-            }
-        }
 
-        call => Err(ShellError::diagnostic(
+    ClassifiedCommand::External(ExternalCommand {
-            language_reporting::Diagnostic::new(
+        name: name.to_string(),
-                language_reporting::Severity::Error,
+        name_span: tag.span,
-                "Invalid command",
+        args: arg_list_strings,
-            )
+    })
-            .with_label(language_reporting::Label::new_primary(call.head().span())),
-        )),
-    }
 }

src/errors.rs

@@ -77,6 +77,20 @@ impl ShellError {
         .start()
     }
 
+    crate fn syntax_error(problem: Tagged<impl Into<String>>) -> ShellError {
+        ProximateShellError::SyntaxError {
+            problem: problem.map(|p| p.into()),
+        }
+        .start()
+    }
+
+    crate fn invalid_command(problem: impl Into<Tag>) -> ShellError {
+        ProximateShellError::InvalidCommand {
+            command: problem.into(),
+        }
+        .start()
+    }
+
     crate fn coerce_error(
         left: Tagged<impl Into<String>>,
         right: Tagged<impl Into<String>>,
@@ -130,6 +144,10 @@ impl ShellError {
             ProximateShellError::String(StringError { title, .. }) => {
                 Diagnostic::new(Severity::Error, title)
             }
+            ProximateShellError::InvalidCommand { command } => {
+                Diagnostic::new(Severity::Error, "Invalid command")
+                    .with_label(Label::new_primary(command.span))
+            }
             ProximateShellError::ArgumentError {
                 command,
                 error,
@@ -188,6 +206,15 @@ impl ShellError {
             } => Diagnostic::new(Severity::Error, "Type Error")
                 .with_label(Label::new_primary(span).with_message(expected)),
 
+            ProximateShellError::SyntaxError {
+                problem:
+                    Tagged {
+                        tag: Tag { span, .. },
+                        ..
+                    },
+            } => Diagnostic::new(Severity::Error, "Syntax Error")
+                .with_label(Label::new_primary(span).with_message("Unexpected external command")),
+
             ProximateShellError::MissingProperty { subpath, expr } => {
                 let subpath = subpath.into_label();
                 let expr = expr.into_label();
@@ -258,6 +285,12 @@ impl ShellError {
 #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)]
 pub enum ProximateShellError {
     String(StringError),
+    SyntaxError {
+        problem: Tagged<String>,
+    },
+    InvalidCommand {
+        command: Tag,
+    },
     TypeError {
         expected: String,
         actual: Tagged<Option<String>>,
@@ -339,7 +372,9 @@ impl std::fmt::Display for ShellError {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         match &self.error {
             ProximateShellError::String(s) => write!(f, "{}", &s.title),
+            ProximateShellError::InvalidCommand { .. } => write!(f, "InvalidCommand"),
             ProximateShellError::TypeError { .. } => write!(f, "TypeError"),
+            ProximateShellError::SyntaxError { .. } => write!(f, "SyntaxError"),
             ProximateShellError::MissingProperty { .. } => write!(f, "MissingProperty"),
             ProximateShellError::ArgumentError { .. } => write!(f, "ArgumentError"),
             ProximateShellError::Diagnostic(_) => write!(f, "<diagnostic>"),

src/evaluate/evaluator.rs

@@ -41,6 +41,7 @@ crate fn evaluate_baseline_expr(
         RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(*literal), source)),
         RawExpression::Synthetic(hir::Synthetic::String(s)) => Ok(Value::string(s).tagged_unknown()),
         RawExpression::Variable(var) => evaluate_reference(var, scope, source),
+        RawExpression::ExternalCommand(external) => evaluate_external(external, scope, source),
         RawExpression::Binary(binary) => {
             let left = evaluate_baseline_expr(binary.left(), registry, scope, source)?;
             let right = evaluate_baseline_expr(binary.right(), registry, scope, source)?;
@@ -127,3 +128,13 @@ fn evaluate_reference(
             .unwrap_or_else(|| Value::nothing().simple_spanned(span))),
     }
 }
+
+fn evaluate_external(
+    external: &hir::ExternalCommand,
+    _scope: &Scope,
+    _source: &Text,
+) -> Result<Tagged<Value>, ShellError> {
+    Err(ShellError::syntax_error(
+        "Unexpected external command".tagged(external.name()),
+    ))
+}

src/object/meta.rs

@@ -120,6 +120,10 @@ impl<T> Tagged<T> {
     pub fn item(&self) -> &T {
         &self.item
     }
+
+    pub fn into_parts(self) -> (T, Tag) {
+        (self.item, self.tag)
+    }
 }
 
 impl<T> From<&Tagged<T>> for Span {
@@ -178,6 +182,21 @@ pub struct Tag {
     pub span: Span,
 }
 
+impl From<Span> for Tag {
+    fn from(span: Span) -> Self {
+        Tag { origin: None, span }
+    }
+}
+
+impl From<&Span> for Tag {
+    fn from(span: &Span) -> Self {
+        Tag {
+            origin: None,
+            span: *span,
+        }
+    }
+}
+
 impl Tag {
     pub fn unknown_origin(span: Span) -> Tag {
         Tag { origin: None, span }

src/parser/hir.rs

@@ -1,10 +1,10 @@
 crate mod baseline_parse;
 crate mod baseline_parse_tokens;
 crate mod binary;
+crate mod external_command;
 crate mod named;
 crate mod path;
 
-use crate::evaluate::Scope;
 use crate::parser::{registry, Unit};
 use crate::prelude::*;
 use derive_new::new;
@@ -12,11 +12,14 @@ use getset::Getters;
 use serde::{Deserialize, Serialize};
 use std::fmt;
 
-crate use baseline_parse::{baseline_parse_single_token, baseline_parse_token_as_string};
+use crate::evaluate::Scope;
-crate use baseline_parse_tokens::{baseline_parse_next_expr, SyntaxType, TokensIterator};
+
-crate use binary::Binary;
+crate use self::baseline_parse::{baseline_parse_single_token, baseline_parse_token_as_string};
-crate use named::NamedArguments;
+crate use self::baseline_parse_tokens::{baseline_parse_next_expr, SyntaxType, TokensIterator};
-crate use path::Path;
+crate use self::binary::Binary;
+crate use self::external_command::ExternalCommand;
+crate use self::named::NamedArguments;
+crate use self::path::Path;
 
 pub fn path(head: impl Into<Expression>, tail: Vec<Tagged<impl Into<String>>>) -> Path {
     Path::new(
@@ -78,6 +81,7 @@ pub enum RawExpression {
     Block(Vec<Expression>),
     List(Vec<Expression>),
     Path(Box<Path>),
+    ExternalCommand(ExternalCommand),
 
     #[allow(unused)]
     Boolean(bool),
@@ -107,6 +111,7 @@ impl RawExpression {
             RawExpression::Block(..) => "block",
             RawExpression::Path(..) => "path",
             RawExpression::Boolean(..) => "boolean",
+            RawExpression::ExternalCommand(..) => "external",
         }
     }
 }
@@ -147,6 +152,13 @@ impl Expression {
         )
     }
 
+    crate fn external_command(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
+        Tagged::from_simple_spanned_item(
+            RawExpression::ExternalCommand(ExternalCommand::new(inner.into())),
+            outer.into(),
+        )
+    }
+
     crate fn it_variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
         Tagged::from_simple_spanned_item(
             RawExpression::Variable(Variable::It(inner.into())),
@@ -163,6 +175,7 @@ impl ToDebug for Expression {
             RawExpression::Variable(Variable::It(_)) => write!(f, "$it"),
             RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)),
             RawExpression::Binary(b) => write!(f, "{}", b.debug(source)),
+            RawExpression::ExternalCommand(c) => write!(f, "^{}", c.name().slice(source)),
             RawExpression::Block(exprs) => {
                 write!(f, "{{ ")?;
 

src/parser/hir/baseline_parse.rs

@@ -10,6 +10,7 @@ pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Express
             hir::Expression::it_variable(span, token.span())
         }
         RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
+        RawToken::External(span) => hir::Expression::external_command(span, token.span()),
         RawToken::Bare => hir::Expression::bare(token.span()),
     }
 }
@@ -19,6 +20,7 @@ pub fn baseline_parse_token_as_string(token: &Token, source: &Text) -> hir::Expr
         RawToken::Variable(span) if span.slice(source) == "it" => {
             hir::Expression::it_variable(span, token.span())
         }
+        RawToken::External(span) => hir::Expression::external_command(span, token.span()),
         RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
         RawToken::Integer(_) => hir::Expression::bare(token.span()),
         RawToken::Size(_, _) => hir::Expression::bare(token.span()),

src/parser/hir/baseline_parse_tokens.rs

@@ -235,7 +235,10 @@ pub fn baseline_parse_path(
             TokenNode::Token(token) => match token.item() {
                 RawToken::Bare => token.span().slice(source),
                 RawToken::String(span) => span.slice(source),
-                RawToken::Integer(_) | RawToken::Size(..) | RawToken::Variable(_) => {
+                RawToken::Integer(_)
+                | RawToken::Size(..)
+                | RawToken::Variable(_)
+                | RawToken::External(_) => {
                     return Err(ShellError::type_error(
                         "String",
                         token.type_name().simple_spanned(part),

src/parser/hir/external_command.rs

@@ -0,0 +1,21 @@
+use crate::prelude::*;
+use derive_new::new;
+use getset::Getters;
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+#[derive(
+    Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new,
+)]
+#[get = "crate"]
+pub struct ExternalCommand {
+    name: Span,
+}
+
+impl ToDebug for ExternalCommand {
+    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+        write!(f, "{}", self.name.slice(source))?;
+
+        Ok(())
+    }
+}

src/parser/parse/parser.rs

@@ -81,15 +81,6 @@ pub fn raw_integer(input: NomSpan) -> IResult<NomSpan, Tagged<i64>> {
         ))
     })
 }
-/*
-pub fn integer(input: NomSpan) -> IResult<NomSpan, TokenNode> {
-    trace_step(input, "integer", move |input| {
-        let (input, int) = raw_integer(input)?;
-
-        Ok((input, TokenTreeBuilder::spanned_int(*int, int.span())))
-    })
-}
-*/
 
 pub fn operator(input: NomSpan) -> IResult<NomSpan, TokenNode> {
     trace_step(input, "operator", |input| {
@@ -138,6 +129,20 @@ pub fn string(input: NomSpan) -> IResult<NomSpan, TokenNode> {
     })
 }
 
+pub fn external(input: NomSpan) -> IResult<NomSpan, TokenNode> {
+    trace_step(input, "external", move |input| {
+        let start = input.offset;
+        let (input, _) = tag("^")(input)?;
+        let (input, bare) = take_while(is_bare_char)(input)?;
+        let end = input.offset;
+
+        Ok((
+            input,
+            TokenTreeBuilder::spanned_external(bare, (start, end)),
+        ))
+    })
+}
+
 pub fn bare(input: NomSpan) -> IResult<NomSpan, TokenNode> {
     trace_step(input, "bare", move |input| {
         let start = input.offset;
@@ -268,7 +273,8 @@ pub fn size(input: NomSpan) -> IResult<NomSpan, TokenNode> {
 
 pub fn leaf(input: NomSpan) -> IResult<NomSpan, TokenNode> {
     trace_step(input, "leaf", move |input| {
-        let (input, node) = alt((size, string, operator, flag, shorthand, var, bare))(input)?;
+        let (input, node) =
+            alt((size, string, operator, flag, shorthand, var, external, bare))(input)?;
 
         Ok((input, node))
     })
@@ -736,6 +742,14 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_external() {
+        assert_leaf! {
+            parsers [ external ]
+            "^ls" -> 0..3 { External(span(1, 3)) }
+        }
+    }
+
     #[test]
     fn test_delimited_paren() {
         assert_eq!(

src/parser/parse/token_tree.rs

@@ -137,6 +137,26 @@ impl TokenNode {
         }
     }
 
+    pub fn is_external(&self) -> bool {
+        match self {
+            TokenNode::Token(Tagged {
+                item: RawToken::External(..),
+                ..
+            }) => true,
+            _ => false,
+        }
+    }
+
+    pub fn expect_external(&self) -> Span {
+        match self {
+            TokenNode::Token(Tagged {
+                item: RawToken::External(span),
+                ..
+            }) => *span,
+            _ => panic!("Only call expect_external if you checked is_external first"),
+        }
+    }
+
     crate fn as_flag(&self, value: &str, source: &Text) -> Option<Tagged<Flag>> {
         match self {
             TokenNode::Flag(

src/parser/parse/token_tree_builder.rs

@@ -152,6 +152,13 @@ impl TokenTreeBuilder {
         ))
     }
 
+    pub fn spanned_external(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
+        TokenNode::Token(Tagged::from_simple_spanned_item(
+            RawToken::External(input.into()),
+            span.into(),
+        ))
+    }
+
     pub fn int(input: impl Into<i64>) -> CurriedToken {
         let int = input.into();
 

src/parser/parse/tokens.rs

@@ -8,6 +8,7 @@ pub enum RawToken {
     Size(i64, Unit),
     String(Span),
     Variable(Span),
+    External(Span),
     Bare,
 }
 
@@ -18,6 +19,7 @@ impl RawToken {
             RawToken::Size(..) => "Size",
             RawToken::String(_) => "String",
             RawToken::Variable(_) => "Variable",
+            RawToken::External(_) => "External",
             RawToken::Bare => "String",
         }
     }

src/shell/helper.rs

@@ -135,6 +135,10 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String {
             item: RawToken::Bare,
             ..
         }) => Color::Green.normal().paint(token_node.span().slice(line)),
+        TokenNode::Token(Tagged {
+            item: RawToken::External(..),
+            ..
+        }) => Color::Cyan.bold().paint(token_node.span().slice(line)),
     };
 
     styled.to_string()

tests/external_tests.rs

@@ -0,0 +1,12 @@
+mod helpers;
+
+use helpers::in_directory as cwd;
+
+#[test]
+fn external_command() {
+    // Echo should exist on all currently supported platforms. A better approach might
+    // be to generate a dummy executable as part of the tests with known semantics.
+    nu!(output, cwd("tests/fixtures"), "echo 1");
+
+    assert!(output.contains("1"));
+}