Merge pull request #632 from nushell/improve-external-words

Close a bunch of holes in external command args

src/commands/cp.rs

@@ -21,7 +21,7 @@ impl PerItemCommand for Cpy {
 
     fn signature(&self) -> Signature {
         Signature::build("cp")
-            .required("src", SyntaxType::Path)
+            .required("src", SyntaxType::Pattern)
             .required("dst", SyntaxType::Path)
             .named("file", SyntaxType::Any)
             .switch("recursive")

src/commands/ls.rs

@@ -10,7 +10,7 @@ impl WholeStreamCommand for LS {
     }
 
     fn signature(&self) -> Signature {
-        Signature::build("ls").optional("path", SyntaxType::Path)
+        Signature::build("ls").optional("path", SyntaxType::Pattern)
     }
 
     fn usage(&self) -> &str {

src/commands/to_bson.rs

@@ -50,6 +50,7 @@ pub fn value_to_bson_value(v: &Tagged<Value>) -> Result<Bson, ShellError> {
         }
         Value::Primitive(Primitive::Nothing) => Bson::Null,
         Value::Primitive(Primitive::String(s)) => Bson::String(s.clone()),
+        Value::Primitive(Primitive::Pattern(p)) => Bson::String(p.clone()),
         Value::Primitive(Primitive::Path(s)) => Bson::String(s.display().to_string()),
         Value::Table(l) => Bson::Array(
             l.iter()

src/commands/to_json.rs

@@ -45,6 +45,7 @@ pub fn value_to_json_value(v: &Tagged<Value>) -> Result<serde_json::Value, Shell
             CoerceInto::<i64>::coerce_into(i.tagged(v.tag), "converting to JSON number")?,
         )),
         Value::Primitive(Primitive::Nothing) => serde_json::Value::Null,
+        Value::Primitive(Primitive::Pattern(s)) => serde_json::Value::String(s.clone()),
         Value::Primitive(Primitive::String(s)) => serde_json::Value::String(s.clone()),
         Value::Primitive(Primitive::Path(s)) => serde_json::Value::String(s.display().to_string()),
 

src/commands/to_sqlite.rs

@@ -91,6 +91,7 @@ fn nu_value_to_sqlite_string(v: Value) -> String {
             Primitive::Int(i) => format!("{}", i),
             Primitive::Decimal(f) => format!("{}", f),
             Primitive::Bytes(u) => format!("{}", u),
+            Primitive::Pattern(s) => format!("'{}'", s.replace("'", "''")),
             Primitive::String(s) => format!("'{}'", s.replace("'", "''")),
             Primitive::Boolean(true) => "1".into(),
             Primitive::Boolean(_) => "0".into(),

src/commands/to_toml.rs

@@ -44,6 +44,7 @@ pub fn value_to_toml_value(v: &Tagged<Value>) -> Result<toml::Value, ShellError>
             toml::Value::Integer(i.tagged(v.tag).coerce_into("converting to TOML integer")?)
         }
         Value::Primitive(Primitive::Nothing) => toml::Value::String("<Nothing>".to_string()),
+        Value::Primitive(Primitive::Pattern(s)) => toml::Value::String(s.clone()),
         Value::Primitive(Primitive::String(s)) => toml::Value::String(s.clone()),
         Value::Primitive(Primitive::Path(s)) => toml::Value::String(s.display().to_string()),
 

src/commands/to_yaml.rs

@@ -42,6 +42,7 @@ pub fn value_to_yaml_value(v: &Tagged<Value>) -> Result<serde_yaml::Value, Shell
             CoerceInto::<i64>::coerce_into(i.tagged(v.tag), "converting to YAML number")?,
         )),
         Value::Primitive(Primitive::Nothing) => serde_yaml::Value::Null,
+        Value::Primitive(Primitive::Pattern(s)) => serde_yaml::Value::String(s.clone()),
         Value::Primitive(Primitive::String(s)) => serde_yaml::Value::String(s.clone()),
         Value::Primitive(Primitive::Path(s)) => serde_yaml::Value::String(s.display().to_string()),
 

src/data/base.rs

@@ -20,6 +20,7 @@ pub enum Primitive {
     Decimal(BigDecimal),
     Bytes(u64),
     String(String),
+    Pattern(String),
     Boolean(bool),
     Date(DateTime<Utc>),
     Path(PathBuf),
@@ -53,6 +54,7 @@ impl Primitive {
             Int(_) => "int",
             Decimal(_) => "decimal",
             Bytes(_) => "bytes",
+            Pattern(_) => "pattern",
             String(_) => "string",
             Boolean(_) => "boolean",
             Date(_) => "date",
@@ -71,6 +73,7 @@ impl Primitive {
             Path(path) => write!(f, "{}", path.display()),
             Decimal(decimal) => write!(f, "{}", decimal),
             Bytes(bytes) => write!(f, "{}", bytes),
+            Pattern(string) => write!(f, "{:?}", string),
             String(string) => write!(f, "{:?}", string),
             Boolean(boolean) => write!(f, "{}", boolean),
             Date(date) => write!(f, "{}", date),
@@ -108,6 +111,7 @@ impl Primitive {
             }
             Primitive::Int(i) => format!("{}", i),
             Primitive::Decimal(decimal) => format!("{}", decimal),
+            Primitive::Pattern(s) => format!("{}", s),
             Primitive::String(s) => format!("{}", s),
             Primitive::Boolean(b) => match (b, field_name) {
                 (true, None) => format!("Yes"),
@@ -578,6 +582,10 @@ impl Value {
         Value::Primitive(Primitive::String(s.into()))
     }
 
+    pub fn pattern(s: impl Into<String>) -> Value {
+        Value::Primitive(Primitive::String(s.into()))
+    }
+
     pub fn path(s: impl Into<PathBuf>) -> Value {
         Value::Primitive(Primitive::Path(s.into()))
     }

src/errors.rs

@@ -38,6 +38,7 @@ pub enum ArgumentError {
     MissingMandatoryFlag(String),
     MissingMandatoryPositional(String),
     MissingValueForName(String),
+    InvalidExternalWord,
 }
 
 #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)]
@@ -136,6 +137,16 @@ impl ShellError {
         .start()
     }
 
+    pub(crate) fn invalid_external_word(span: Span) -> ShellError {
+        ProximateShellError::ArgumentError {
+            command: "Invalid argument to Nu command (did you mean to call an external command?)"
+                .into(),
+            error: ArgumentError::InvalidExternalWord,
+            span,
+        }
+        .start()
+    }
+
     pub(crate) fn parse_error(
         error: nom::Err<(nom5_locate::LocatedSpan<&str>, nom::error::ErrorKind)>,
     ) -> ShellError {
@@ -190,6 +201,10 @@ impl ShellError {
                 error,
                 span,
             } => match error {
+                ArgumentError::InvalidExternalWord => Diagnostic::new(
+                    Severity::Error,
+                    format!("Invalid bare word for Nu command (did you intend to invoke an external command?)"))
+                .with_label(Label::new_primary(span)),
                 ArgumentError::MissingMandatoryFlag(name) => Diagnostic::new(
                     Severity::Error,
                     format!(

src/evaluate/evaluator.rs

@@ -1,5 +1,5 @@
 use crate::data::base::Block;
-use crate::errors::Description;
+use crate::errors::{ArgumentError, Description};
 use crate::parser::{
     hir::{self, Expression, RawExpression},
     CommandRegistry, Text,
@@ -39,6 +39,11 @@ pub(crate) fn evaluate_baseline_expr(
 ) -> Result<Tagged<Value>, ShellError> {
     match &expr.item {
         RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(literal), source)),
+        RawExpression::ExternalWord => Err(ShellError::argument_error(
+            "Invalid external word",
+            ArgumentError::InvalidExternalWord,
+            expr.span(),
+        )),
         RawExpression::FilePath(path) => Ok(Value::path(path.clone()).tagged(expr.span())),
         RawExpression::Synthetic(hir::Synthetic::String(s)) => Ok(Value::string(s).tagged_unknown()),
         RawExpression::Variable(var) => evaluate_reference(var, scope, source),
@@ -109,6 +114,7 @@ fn evaluate_literal(literal: Tagged<&hir::Literal>, source: &Text) -> Tagged<Val
         hir::Literal::Number(int) => int.into(),
         hir::Literal::Size(int, unit) => unit.compute(int),
         hir::Literal::String(span) => Value::string(span.slice(source)),
+        hir::Literal::GlobPattern => Value::pattern(literal.span().slice(source)),
         hir::Literal::Bare => Value::string(literal.span().slice(source)),
     };
 

src/parser/hir.rs

@@ -17,7 +17,7 @@ use crate::evaluate::Scope;
 
 pub(crate) use self::baseline_parse::{
     baseline_parse_single_token, baseline_parse_token_as_number, baseline_parse_token_as_path,
-    baseline_parse_token_as_string,
+    baseline_parse_token_as_pattern, baseline_parse_token_as_string,
 };
 pub(crate) use self::baseline_parse_tokens::{baseline_parse_next_expr, TokensIterator};
 pub(crate) use self::binary::Binary;
@@ -83,12 +83,14 @@ impl ToDebug for Call {
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
 pub enum RawExpression {
     Literal(Literal),
+    ExternalWord,
     Synthetic(Synthetic),
     Variable(Variable),
     Binary(Box<Binary>),
     Block(Vec<Expression>),
     List(Vec<Expression>),
     Path(Box<Path>),
+
     FilePath(PathBuf),
     ExternalCommand(ExternalCommand),
 
@@ -113,6 +115,7 @@ impl RawExpression {
         match self {
             RawExpression::Literal(literal) => literal.type_name(),
             RawExpression::Synthetic(synthetic) => synthetic.type_name(),
+            RawExpression::ExternalWord => "externalword",
             RawExpression::FilePath(..) => "filepath",
             RawExpression::Variable(..) => "variable",
             RawExpression::List(..) => "list",
@@ -162,6 +165,10 @@ impl Expression {
         Tagged::from_simple_spanned_item(RawExpression::Literal(Literal::Bare), span.into())
     }
 
+    pub(crate) fn pattern(tag: impl Into<Tag>) -> Expression {
+        RawExpression::Literal(Literal::GlobPattern).tagged(tag.into())
+    }
+
     pub(crate) fn variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
         Tagged::from_simple_spanned_item(
             RawExpression::Variable(Variable::Other(inner.into())),
@@ -189,6 +196,7 @@ impl ToDebug for Expression {
         match self.item() {
             RawExpression::Literal(l) => l.tagged(self.span()).fmt_debug(f, source),
             RawExpression::FilePath(p) => write!(f, "{}", p.display()),
+            RawExpression::ExternalWord => write!(f, "{}", self.span().slice(source)),
             RawExpression::Synthetic(Synthetic::String(s)) => write!(f, "{:?}", s),
             RawExpression::Variable(Variable::It(_)) => write!(f, "$it"),
             RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)),
@@ -225,11 +233,17 @@ impl From<Tagged<Path>> for Expression {
     }
 }
 
+/// Literals are expressions that are:
+///
+/// 1. Copy
+/// 2. Can be evaluated without additional context
+/// 3. Evaluation cannot produce an error
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
 pub enum Literal {
     Number(Number),
     Size(Number, Unit),
     String(Span),
+    GlobPattern,
     Bare,
 }
 
@@ -239,6 +253,7 @@ impl ToDebug for Tagged<&Literal> {
             Literal::Number(number) => write!(f, "{:?}", *number),
             Literal::Size(number, unit) => write!(f, "{:?}{:?}", *number, unit),
             Literal::String(span) => write!(f, "{}", span.slice(source)),
+            Literal::GlobPattern => write!(f, "{}", self.span().slice(source)),
             Literal::Bare => write!(f, "{}", self.span().slice(source)),
         }
     }
@@ -251,6 +266,7 @@ impl Literal {
             Literal::Size(..) => "size",
             Literal::String(..) => "string",
             Literal::Bare => "string",
+            Literal::GlobPattern => "pattern",
         }
     }
 }

src/parser/hir/baseline_parse.rs

@@ -1,10 +1,15 @@
 use crate::context::Context;
+use crate::errors::ShellError;
 use crate::parser::{hir, RawToken, Token};
+use crate::TaggedItem;
 use crate::Text;
 use std::path::PathBuf;
 
-pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Expression {
+pub fn baseline_parse_single_token(
-    match *token.item() {
+    token: &Token,
+    source: &Text,
+) -> Result<hir::Expression, ShellError> {
+    Ok(match *token.item() {
         RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.span()),
         RawToken::Size(int, unit) => {
             hir::Expression::size(int.to_number(source), unit, token.span())
@@ -14,51 +19,74 @@ pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Express
             hir::Expression::it_variable(span, token.span())
         }
         RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
-        RawToken::External(span) => hir::Expression::external_command(span, token.span()),
+        RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
+        RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
+        RawToken::GlobPattern => hir::Expression::pattern(token.span()),
         RawToken::Bare => hir::Expression::bare(token.span()),
-    }
+    })
 }
 
-pub fn baseline_parse_token_as_number(token: &Token, source: &Text) -> hir::Expression {
+pub fn baseline_parse_token_as_number(
-    match *token.item() {
+    token: &Token,
+    source: &Text,
+) -> Result<hir::Expression, ShellError> {
+    Ok(match *token.item() {
         RawToken::Variable(span) if span.slice(source) == "it" => {
             hir::Expression::it_variable(span, token.span())
         }
-        RawToken::External(span) => hir::Expression::external_command(span, token.span()),
+        RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
+        RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
         RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
         RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.span()),
         RawToken::Size(number, unit) => {
             hir::Expression::size(number.to_number(source), unit, token.span())
         }
         RawToken::Bare => hir::Expression::bare(token.span()),
+        RawToken::GlobPattern => {
+            return Err(ShellError::type_error(
+                "Number",
+                "glob pattern".to_string().tagged(token.tag()),
+            ))
+        }
         RawToken::String(span) => hir::Expression::string(span, token.span()),
-    }
+    })
 }
 
-pub fn baseline_parse_token_as_string(token: &Token, source: &Text) -> hir::Expression {
+pub fn baseline_parse_token_as_string(
-    match *token.item() {
+    token: &Token,
+    source: &Text,
+) -> Result<hir::Expression, ShellError> {
+    Ok(match *token.item() {
         RawToken::Variable(span) if span.slice(source) == "it" => {
             hir::Expression::it_variable(span, token.span())
         }
-        RawToken::External(span) => hir::Expression::external_command(span, token.span()),
+        RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
+        RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
         RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
         RawToken::Number(_) => hir::Expression::bare(token.span()),
         RawToken::Size(_, _) => hir::Expression::bare(token.span()),
         RawToken::Bare => hir::Expression::bare(token.span()),
+        RawToken::GlobPattern => {
+            return Err(ShellError::type_error(
+                "String",
+                "glob pattern".tagged(token.tag()),
+            ))
+        }
         RawToken::String(span) => hir::Expression::string(span, token.span()),
-    }
+    })
 }
 
 pub fn baseline_parse_token_as_path(
     token: &Token,
     context: &Context,
     source: &Text,
-) -> hir::Expression {
+) -> Result<hir::Expression, ShellError> {
-    match *token.item() {
+    Ok(match *token.item() {
         RawToken::Variable(span) if span.slice(source) == "it" => {
             hir::Expression::it_variable(span, token.span())
         }
-        RawToken::External(span) => hir::Expression::external_command(span, token.span()),
+        RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
+        RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
         RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
         RawToken::Number(_) => hir::Expression::bare(token.span()),
         RawToken::Size(_, _) => hir::Expression::bare(token.span()),
@@ -66,10 +94,45 @@ pub fn baseline_parse_token_as_path(
             expand_path(token.span().slice(source), context),
             token.span(),
         ),
+        RawToken::GlobPattern => {
+            return Err(ShellError::type_error(
+                "Path",
+                "glob pattern".tagged(token.tag()),
+            ))
+        }
         RawToken::String(span) => {
             hir::Expression::file_path(expand_path(span.slice(source), context), token.span())
         }
-    }
+    })
+}
+
+pub fn baseline_parse_token_as_pattern(
+    token: &Token,
+    context: &Context,
+    source: &Text,
+) -> Result<hir::Expression, ShellError> {
+    Ok(match *token.item() {
+        RawToken::Variable(span) if span.slice(source) == "it" => {
+            hir::Expression::it_variable(span, token.span())
+        }
+        RawToken::ExternalCommand(_) => {
+            return Err(ShellError::syntax_error(
+                "Invalid external command".to_string().tagged(token.tag()),
+            ))
+        }
+        RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
+        RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
+        RawToken::Number(_) => hir::Expression::bare(token.span()),
+        RawToken::Size(_, _) => hir::Expression::bare(token.span()),
+        RawToken::GlobPattern => hir::Expression::pattern(token.span()),
+        RawToken::Bare => hir::Expression::file_path(
+            expand_path(token.span().slice(source), context),
+            token.span(),
+        ),
+        RawToken::String(span) => {
+            hir::Expression::file_path(expand_path(span.slice(source), context), token.span())
+        }
+    })
 }
 
 pub fn expand_path(string: &str, context: &Context) -> PathBuf {

src/parser/hir/baseline_parse_tokens.rs

@@ -4,7 +4,7 @@ use crate::parser::{
     hir,
     hir::{
         baseline_parse_single_token, baseline_parse_token_as_number, baseline_parse_token_as_path,
-        baseline_parse_token_as_string,
+        baseline_parse_token_as_pattern, baseline_parse_token_as_string,
     },
     DelimitedNode, Delimiter, PathNode, RawToken, TokenNode,
 };
@@ -33,7 +33,6 @@ pub fn baseline_parse_tokens(
     Ok(exprs)
 }
 
-
 #[derive(Debug, Copy, Clone, Serialize, Deserialize)]
 pub enum SyntaxType {
     Any,
@@ -44,6 +43,7 @@ pub enum SyntaxType {
     Variable,
     Number,
     Path,
+    Pattern,
     Binary,
     Block,
     Boolean,
@@ -60,9 +60,10 @@ impl std::fmt::Display for SyntaxType {
             SyntaxType::Variable => write!(f, "Variable"),
             SyntaxType::Number => write!(f, "Number"),
             SyntaxType::Path => write!(f, "Path"),
+            SyntaxType::Pattern => write!(f, "Pattern"),
             SyntaxType::Binary => write!(f, "Binary"),
             SyntaxType::Block => write!(f, "Block"),
-            SyntaxType::Boolean => write!(f, "Boolean")
+            SyntaxType::Boolean => write!(f, "Boolean"),
         }
     }
 }
@@ -81,7 +82,7 @@ pub fn baseline_parse_next_expr(
 
     match (syntax_type, next) {
         (SyntaxType::Path, TokenNode::Token(token)) => {
-            return Ok(baseline_parse_token_as_path(token, context, source))
+            return baseline_parse_token_as_path(token, context, source)
         }
 
         (SyntaxType::Path, token) => {
@@ -91,8 +92,19 @@ pub fn baseline_parse_next_expr(
             ))
         }
 
+        (SyntaxType::Pattern, TokenNode::Token(token)) => {
+            return baseline_parse_token_as_pattern(token, context, source)
+        }
+
+        (SyntaxType::Pattern, token) => {
+            return Err(ShellError::type_error(
+                "Path",
+                token.type_name().simple_spanned(token.span()),
+            ))
+        }
+
         (SyntaxType::String, TokenNode::Token(token)) => {
-            return Ok(baseline_parse_token_as_string(token, source));
+            return baseline_parse_token_as_string(token, source);
         }
 
         (SyntaxType::String, token) => {
@@ -103,7 +115,7 @@ pub fn baseline_parse_next_expr(
         }
 
         (SyntaxType::Number, TokenNode::Token(token)) => {
-            return Ok(baseline_parse_token_as_number(token, source));
+            return Ok(baseline_parse_token_as_number(token, source)?);
         }
 
         (SyntaxType::Number, token) => {
@@ -115,7 +127,7 @@ pub fn baseline_parse_next_expr(
 
         // TODO: More legit member processing
         (SyntaxType::Member, TokenNode::Token(token)) => {
-            return Ok(baseline_parse_token_as_string(token, source));
+            return baseline_parse_token_as_string(token, source);
         }
 
         (SyntaxType::Member, token) => {
@@ -245,7 +257,7 @@ pub fn baseline_parse_semantic_token(
     source: &Text,
 ) -> Result<hir::Expression, ShellError> {
     match token {
-        TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)),
+        TokenNode::Token(token) => baseline_parse_single_token(token, source),
         TokenNode::Call(_call) => unimplemented!(),
         TokenNode::Delimited(delimited) => baseline_parse_delimited(delimited, context, source),
         TokenNode::Pipeline(_pipeline) => unimplemented!(),
@@ -315,7 +327,9 @@ pub fn baseline_parse_path(
                 RawToken::Number(_)
                 | RawToken::Size(..)
                 | RawToken::Variable(_)
-                | RawToken::External(_) => {
+                | RawToken::ExternalCommand(_)
+                | RawToken::GlobPattern
+                | RawToken::ExternalWord => {
                     return Err(ShellError::type_error(
                         "String",
                         token.type_name().simple_spanned(part),

src/parser/parse/call_node.rs

@@ -1,5 +1,7 @@
 use crate::parser::TokenNode;
+use crate::traits::ToDebug;
 use getset::Getters;
+use std::fmt;
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
 pub struct CallNode {
@@ -24,3 +26,17 @@ impl CallNode {
         }
     }
 }
+
+impl ToDebug for CallNode {
+    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+        write!(f, "{}", self.head.debug(source))?;
+
+        if let Some(children) = &self.children {
+            for child in children {
+                write!(f, "{}", child.debug(source))?
+            }
+        }
+
+        Ok(())
+    }
+}

src/parser/parse/parser.rs

@@ -231,17 +231,62 @@ pub fn external(input: NomSpan) -> IResult<NomSpan, TokenNode> {
     })
 }
 
+pub fn pattern(input: NomSpan) -> IResult<NomSpan, TokenNode> {
+    trace_step(input, "bare", move |input| {
+        let start = input.offset;
+        let (input, _) = take_while1(is_start_glob_char)(input)?;
+        let (input, _) = take_while(is_glob_char)(input)?;
+
+        let next_char = &input.fragment.chars().nth(0);
+
+        if let Some(next_char) = next_char {
+            if is_external_word_char(*next_char) {
+                return Err(nom::Err::Error(nom::error::make_error(
+                    input,
+                    nom::error::ErrorKind::TakeWhile1,
+                )));
+            }
+        }
+
+        let end = input.offset;
+
+        Ok((input, TokenTreeBuilder::spanned_pattern((start, end))))
+    })
+}
+
 pub fn bare(input: NomSpan) -> IResult<NomSpan, TokenNode> {
     trace_step(input, "bare", move |input| {
         let start = input.offset;
         let (input, _) = take_while1(is_start_bare_char)(input)?;
         let (input, _) = take_while(is_bare_char)(input)?;
+
+        let next_char = &input.fragment.chars().nth(0);
+
+        if let Some(next_char) = next_char {
+            if is_external_word_char(*next_char) || *next_char == '*' {
+                return Err(nom::Err::Error(nom::error::make_error(
+                    input,
+                    nom::error::ErrorKind::TakeWhile1,
+                )));
+            }
+        }
+
         let end = input.offset;
 
         Ok((input, TokenTreeBuilder::spanned_bare((start, end))))
     })
 }
 
+pub fn external_word(input: NomSpan) -> IResult<NomSpan, TokenNode> {
+    trace_step(input, "bare", move |input| {
+        let start = input.offset;
+        let (input, _) = take_while1(is_external_word_char)(input)?;
+        let end = input.offset;
+
+        Ok((input, TokenTreeBuilder::spanned_external_word((start, end))))
+    })
+}
+
 pub fn var(input: NomSpan) -> IResult<NomSpan, TokenNode> {
     trace_step(input, "var", move |input| {
         let start = input.offset;
@@ -364,8 +409,18 @@ pub fn size(input: NomSpan) -> IResult<NomSpan, TokenNode> {
 
 pub fn leaf(input: NomSpan) -> IResult<NomSpan, TokenNode> {
     trace_step(input, "leaf", move |input| {
-        let (input, node) =
+        let (input, node) = alt((
-            alt((size, string, operator, flag, shorthand, var, external, bare))(input)?;
+            size,
+            string,
+            operator,
+            flag,
+            shorthand,
+            var,
+            external,
+            bare,
+            pattern,
+            external_word,
+        ))(input)?;
 
         Ok((input, node))
     })
@@ -582,26 +637,13 @@ pub fn pipeline(input: NomSpan) -> IResult<NomSpan, TokenNode> {
 }
 
 fn make_call_list(
-    head: Option<(
+    head: Option<(Option<NomSpan>, Tagged<CallNode>, Option<NomSpan>)>,
-        Option<NomSpan>,
+    items: Vec<(NomSpan, Option<NomSpan>, Tagged<CallNode>, Option<NomSpan>)>,
-        Tagged<CallNode>,
-        Option<NomSpan>
-    )>,
-    items: Vec<(
-        NomSpan,
-        Option<NomSpan>,
-        Tagged<CallNode>,
-        Option<NomSpan>,
-    )>,
 ) -> Vec<PipelineElement> {
     let mut out = vec![];
 
     if let Some(head) = head {
-        let el = PipelineElement::new(
+        let el = PipelineElement::new(None, head.0.map(Span::from), head.1, head.2.map(Span::from));
-            None,
-            head.0.map(Span::from),
-            head.1,
-            head.2.map(Span::from));
 
         out.push(el);
     }
@@ -611,7 +653,8 @@ fn make_call_list(
             Some(pipe).map(Span::from),
             ws1.map(Span::from),
             call,
-            ws2.map(Span::from));
+            ws2.map(Span::from),
+        );
 
         out.push(el);
     }
@@ -628,40 +671,49 @@ fn int<T>(frag: &str, neg: Option<T>) -> i64 {
     }
 }
 
+fn is_external_word_char(c: char) -> bool {
+    match c {
+        ';' | '|' | '#' | '-' | '"' | '\'' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '`' => false,
+        other if other.is_whitespace() => false,
+        _ => true,
+    }
+}
+
+fn is_start_glob_char(c: char) -> bool {
+    is_start_bare_char(c) || c == '*'
+}
+
+fn is_glob_char(c: char) -> bool {
+    is_bare_char(c) || c == '*'
+}
+
 fn is_start_bare_char(c: char) -> bool {
     match c {
+        '+' => false,
         _ if c.is_alphabetic() => true,
-        _ if c.is_numeric() => true,
         '.' => true,
         '\\' => true,
         '/' => true,
         '_' => true,
         '-' => true,
-        '@' => true,
-        '*' => true,
-        '?' => true,
         '~' => true,
-        '+' => true,
         _ => false,
     }
 }
 
 fn is_bare_char(c: char) -> bool {
     match c {
+        '+' => false,
         _ if c.is_alphanumeric() => true,
-        ':' => true,
         '.' => true,
         '\\' => true,
         '/' => true,
         '_' => true,
         '-' => true,
-        '@' => true,
-        '*' => true,
-        '?' => true,
         '=' => true,
         '~' => true,
-        '+' => true,
+        ':' => true,
-        '%' => true,
+        '?' => true,
         _ => false,
     }
 }
@@ -724,6 +776,44 @@ mod tests {
         }
     }
 
+    macro_rules! equal_tokens {
+        ($source:tt -> $tokens:expr) => {
+            let result = apply(pipeline, "pipeline", $source);
+            let (expected_tree, expected_source) = TokenTreeBuilder::build($tokens);
+
+            if result != expected_tree {
+                let debug_result = format!("{}", result.debug($source));
+                let debug_expected = format!("{}", expected_tree.debug(&expected_source));
+
+                if debug_result == debug_expected {
+                    assert_eq!(
+                        result, expected_tree,
+                        "NOTE: actual and expected had equivalent debug serializations, source={:?}, debug_expected={:?}",
+                        $source,
+                        debug_expected
+                    )
+                } else {
+                    assert_eq!(debug_result, debug_expected)
+                }
+            }
+
+            // apply(pipeline, "pipeline", r#"cargo +nightly run"#),
+            // build_token(b::pipeline(vec![(
+            //     None,
+            //     b::call(
+            //         b::bare("cargo"),
+            //         vec![
+            //             b::sp(),
+            //             b::external_word("+nightly"),
+            //             b::sp(),
+            //             b::bare("run")
+            //         ]
+            //     ),
+            //     None
+            // )]))
+        };
+    }
+
     #[test]
     fn test_integer() {
         assert_leaf! {
@@ -854,7 +944,7 @@ mod tests {
     fn test_external() {
         assert_leaf! {
             parsers [ external ]
-            "^ls" -> 0..3 { External(span(1, 3)) }
+            "^ls" -> 0..3 { ExternalCommand(span(1, 3)) }
         }
     }
 
@@ -1058,6 +1148,46 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_external_word() {
+        let _ = pretty_env_logger::try_init();
+
+        equal_tokens!(
+            "cargo +nightly run" ->
+            b::pipeline(vec![(
+                None,
+                b::call(
+                    b::bare("cargo"),
+                    vec![
+                        b::sp(),
+                        b::external_word("+nightly"),
+                        b::sp(),
+                        b::bare("run")
+                    ]
+                ),
+                None
+            )])
+        );
+
+        equal_tokens!(
+            "rm foo%bar" ->
+            b::pipeline(vec![(
+                None,
+                b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]),
+                None
+            )])
+        );
+
+        equal_tokens!(
+            "rm foo%bar" ->
+            b::pipeline(vec![(
+                None,
+                b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]),
+                None
+            )])
+        );
+    }
+
     #[test]
     fn test_smoke_pipeline() {
         let _ = pretty_env_logger::try_init();
@@ -1178,7 +1308,6 @@ mod tests {
     }
 
     fn build_token(block: CurriedToken) -> TokenNode {
-        let mut builder = TokenTreeBuilder::new();
+        TokenTreeBuilder::build(block).0
-        block(&mut builder)
     }
 }

src/parser/parse/pipeline.rs

@@ -1,7 +1,9 @@
 use crate::parser::CallNode;
+use crate::traits::ToDebug;
 use crate::{Span, Tagged};
 use derive_new::new;
 use getset::Getters;
+use std::fmt;
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)]
 pub struct Pipeline {
@@ -9,6 +11,20 @@ pub struct Pipeline {
     pub(crate) post_ws: Option<Span>,
 }
 
+impl ToDebug for Pipeline {
+    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+        for part in &self.parts {
+            write!(f, "{}", part.debug(source))?;
+        }
+
+        if let Some(post_ws) = self.post_ws {
+            write!(f, "{}", post_ws.slice(source))?
+        }
+
+        Ok(())
+    }
+}
+
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
 pub struct PipelineElement {
     pub pipe: Option<Span>,
@@ -17,3 +33,23 @@ pub struct PipelineElement {
     call: Tagged<CallNode>,
     pub post_ws: Option<Span>,
 }
+
+impl ToDebug for PipelineElement {
+    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+        if let Some(pipe) = self.pipe {
+            write!(f, "{}", pipe.slice(source))?;
+        }
+
+        if let Some(pre_ws) = self.pre_ws {
+            write!(f, "{}", pre_ws.slice(source))?;
+        }
+
+        write!(f, "{}", self.call.debug(source))?;
+
+        if let Some(post_ws) = self.post_ws {
+            write!(f, "{}", post_ws.slice(source))?;
+        }
+
+        Ok(())
+    }
+}

src/parser/parse/token_tree.rs

@@ -1,5 +1,6 @@
 use crate::errors::ShellError;
 use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
+use crate::traits::ToDebug;
 use crate::{Span, Tagged, Text};
 use derive_new::new;
 use enum_utils::FromStr;
@@ -22,6 +23,12 @@ pub enum TokenNode {
     Path(Tagged<PathNode>),
 }
 
+impl ToDebug for TokenNode {
+    fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
+        write!(f, "{:?}", self.old_debug(&Text::from(source)))
+    }
+}
+
 pub struct DebugTokenNode<'a> {
     node: &'a TokenNode,
     source: &'a Text,
@@ -34,11 +41,11 @@ impl fmt::Debug for DebugTokenNode<'_> {
             TokenNode::Call(s) => {
                 write!(f, "(")?;
 
-                write!(f, "{:?}", s.head().debug(self.source))?;
+                write!(f, "{}", s.head().debug(self.source))?;
 
                 if let Some(children) = s.children() {
                     for child in children {
-                        write!(f, "{:?}", child.debug(self.source))?;
+                        write!(f, "{}", child.debug(self.source))?;
                     }
                 }
 
@@ -57,7 +64,7 @@ impl fmt::Debug for DebugTokenNode<'_> {
                 )?;
 
                 for child in d.children() {
-                    write!(f, "{:?}", child.debug(self.source))?;
+                    write!(f, "{:?}", child.old_debug(self.source))?;
                 }
 
                 write!(
@@ -70,7 +77,7 @@ impl fmt::Debug for DebugTokenNode<'_> {
                     }
                 )
             }
-            TokenNode::Pipeline(_) => write!(f, "<todo:pipeline>"),
+            TokenNode::Pipeline(pipeline) => write!(f, "{}", pipeline.debug(self.source)),
             TokenNode::Error(s) => write!(f, "<error> for {:?}", s.span().slice(self.source)),
             rest => write!(f, "{}", rest.span().slice(self.source)),
         }
@@ -115,7 +122,7 @@ impl TokenNode {
         .to_string()
     }
 
-    pub fn debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> {
+    pub fn old_debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> {
         DebugTokenNode { node: self, source }
     }
 
@@ -140,7 +147,7 @@ impl TokenNode {
     pub fn is_external(&self) -> bool {
         match self {
             TokenNode::Token(Tagged {
-                item: RawToken::External(..),
+                item: RawToken::ExternalCommand(..),
                 ..
             }) => true,
             _ => false,
@@ -150,7 +157,7 @@ impl TokenNode {
     pub fn expect_external(&self) -> Span {
         match self {
             TokenNode::Token(Tagged {
-                item: RawToken::External(span),
+                item: RawToken::ExternalCommand(span),
                 ..
             }) => *span,
             _ => panic!("Only call expect_external if you checked is_external first"),

src/parser/parse/token_tree_builder.rs

@@ -14,15 +14,19 @@ use derive_new::new;
 pub struct TokenTreeBuilder {
     #[new(default)]
     pos: usize,
+
+    #[new(default)]
+    output: String,
 }
 
 pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> TokenNode + 'static>;
 pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Tagged<CallNode> + 'static>;
 
 impl TokenTreeBuilder {
-    pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> TokenNode {
+    pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) {
         let mut builder = TokenTreeBuilder::new();
-        block(&mut builder)
+        let node = block(&mut builder);
+        (node, builder.output)
     }
 
     pub fn pipeline(input: Vec<(Option<&str>, CurriedCall, Option<&str>)>) -> CurriedToken {
@@ -56,7 +60,8 @@ impl TokenTreeBuilder {
                 pipe,
                 pre_span.map(Span::from),
                 call,
-                post_span.map(Span::from)));
+                post_span.map(Span::from),
+            ));
 
             loop {
                 match input.next() {
@@ -147,9 +152,45 @@ impl TokenTreeBuilder {
         ))
     }
 
+    pub fn pattern(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&input);
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_pattern((start, end))
+        })
+    }
+
+    pub fn spanned_pattern(input: impl Into<Span>) -> TokenNode {
+        TokenNode::Token(Tagged::from_simple_spanned_item(
+            RawToken::Bare,
+            input.into(),
+        ))
+    }
+
+    pub fn external_word(input: impl Into<String>) -> CurriedToken {
+        let input = input.into();
+
+        Box::new(move |b| {
+            let (start, end) = b.consume(&input);
+            b.pos = end;
+
+            TokenTreeBuilder::spanned_external_word((start, end))
+        })
+    }
+
+    pub fn spanned_external_word(input: impl Into<Span>) -> TokenNode {
+        TokenNode::Token(Tagged::from_simple_spanned_item(
+            RawToken::ExternalWord,
+            input.into(),
+        ))
+    }
+
     pub fn spanned_external(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
         TokenNode::Token(Tagged::from_simple_spanned_item(
-            RawToken::External(input.into()),
+            RawToken::ExternalCommand(input.into()),
             span.into(),
         ))
     }
@@ -422,6 +463,7 @@ impl TokenTreeBuilder {
     fn consume(&mut self, input: &str) -> (usize, usize) {
         let start = self.pos;
         self.pos += input.len();
+        self.output.push_str(input);
         (start, self.pos)
     }
 }

src/parser/parse/tokens.rs

@@ -10,7 +10,9 @@ pub enum RawToken {
     Size(RawNumber, Unit),
     String(Span),
     Variable(Span),
-    External(Span),
+    ExternalCommand(Span),
+    ExternalWord,
+    GlobPattern,
     Bare,
 }
 
@@ -50,7 +52,9 @@ impl RawToken {
             RawToken::Size(..) => "Size",
             RawToken::String(_) => "String",
             RawToken::Variable(_) => "Variable",
-            RawToken::External(_) => "External",
+            RawToken::ExternalCommand(_) => "ExternalCommand",
+            RawToken::ExternalWord => "ExternalWord",
+            RawToken::GlobPattern => "GlobPattern",
             RawToken::Bare => "String",
         }
     }

src/parser/parse_command.rs

@@ -6,6 +6,7 @@ use crate::parser::{
     hir::{self, NamedArguments},
     Flag, RawToken, TokenNode,
 };
+use crate::traits::ToDebug;
 use crate::{Span, Tag, Tagged, Text};
 use log::trace;
 
@@ -248,7 +249,7 @@ pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source
         itertools::join(
             tail.debug_remaining()
                 .iter()
-                .map(|i| format!("%{:?}%", i.debug(source))),
+                .map(|i| format!("%{}%", i.debug(&source))),
             " "
         )
     );

src/shell/helper.rs

@@ -123,6 +123,10 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String {
             item: RawToken::Size(..),
             ..
         }) => Color::Purple.bold().paint(token_node.span().slice(line)),
+        TokenNode::Token(Tagged {
+            item: RawToken::GlobPattern,
+            ..
+        }) => Color::Cyan.normal().paint(token_node.span().slice(line)),
         TokenNode::Token(Tagged {
             item: RawToken::String(..),
             ..
@@ -136,9 +140,13 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String {
             ..
         }) => Color::Green.normal().paint(token_node.span().slice(line)),
         TokenNode::Token(Tagged {
-            item: RawToken::External(..),
+            item: RawToken::ExternalCommand(..),
             ..
         }) => Color::Cyan.bold().paint(token_node.span().slice(line)),
+        TokenNode::Token(Tagged {
+            item: RawToken::ExternalWord,
+            ..
+        }) => Color::Black.bold().paint(token_node.span().slice(line)),
     };
 
     styled.to_string()