extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2024-11-22 08:23:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

Close a bunch of holes in external command args

Browse Source 
Previously, there was a single parsing rule for "bare words" that
applied to both internal and external commands.

This meant that, because `cargo +nightly` needed to work, we needed to
add `+` as a valid character in bare words.

The number of characters continued to grow, and the situation was
becoming untenable. The current strategy would eventually eat up all
syntax and make it impossible to add syntax like `@foo` to internal
commands.

This patch significantly restricts bare words and introduces a new token
type (`ExternalWord`). An `ExternalWord` expands to an error in the
internal syntax, but expands to a bare word in the external syntax.

`ExternalWords` are highlighted in grey in the shell.
This commit is contained in:
Yehuda Katz 2019-09-09 10:43:10 -07:00
parent 1878bb8a54
commit 4d3e7efe25
13 changed files with 300 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/errors.rs
View File

@ -38,6 +38,7 @@ pub enum ArgumentError {
MissingMandatoryFlag(String),
MissingMandatoryPositional(String),
MissingValueForName(String),
InvalidExternalWord,
}
#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)]
@ -136,6 +137,16 @@ impl ShellError {
.start()
}
pub(crate) fn invalid_external_word(span: Span) -> ShellError {
ProximateShellError::ArgumentError {
command: "Invalid argument to Nu command (did you mean to call an external command?)"
.into(),
error: ArgumentError::InvalidExternalWord,
span,
}
.start()
}
pub(crate) fn parse_error(
error: nom::Err<(nom5_locate::LocatedSpan<&str>, nom::error::ErrorKind)>,
) -> ShellError {
@ -190,6 +201,10 @@ impl ShellError {
error,
span,
} => match error {
ArgumentError::InvalidExternalWord => Diagnostic::new(
Severity::Error,
format!("Invalid bare word for Nu command (did you intend to invoke an external command?)"))
.with_label(Label::new_primary(span)),
ArgumentError::MissingMandatoryFlag(name) => Diagnostic::new(
Severity::Error,
format!(

7
src/evaluate/evaluator.rs
View File

@ -1,5 +1,5 @@
use crate::data::base::Block;
use crate::errors::Description;
use crate::errors::{ArgumentError, Description};
use crate::parser::{
hir::{self, Expression, RawExpression},
CommandRegistry, Text,
@ -39,6 +39,11 @@ pub(crate) fn evaluate_baseline_expr(
) -> Result<Tagged<Value>, ShellError> {
match &expr.item {
RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(literal), source)),
RawExpression::ExternalWord => Err(ShellError::argument_error(
"Invalid external word",
ArgumentError::InvalidExternalWord,
expr.span(),
)),
RawExpression::FilePath(path) => Ok(Value::path(path.clone()).tagged(expr.span())),
RawExpression::Synthetic(hir::Synthetic::String(s)) => Ok(Value::string(s).tagged_unknown()),
RawExpression::Variable(var) => evaluate_reference(var, scope, source),

8
src/parser/hir.rs
View File

@ -83,6 +83,7 @@ impl ToDebug for Call {
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
pub enum RawExpression {
Literal(Literal),
ExternalWord,
Synthetic(Synthetic),
Variable(Variable),
Binary(Box<Binary>),
@ -113,6 +114,7 @@ impl RawExpression {
match self {
RawExpression::Literal(literal) => literal.type_name(),
RawExpression::Synthetic(synthetic) => synthetic.type_name(),
RawExpression::ExternalWord => "externalword",
RawExpression::FilePath(..) => "filepath",
RawExpression::Variable(..) => "variable",
RawExpression::List(..) => "list",
@ -189,6 +191,7 @@ impl ToDebug for Expression {
match self.item() {
RawExpression::Literal(l) => l.tagged(self.span()).fmt_debug(f, source),
RawExpression::FilePath(p) => write!(f, "{}", p.display()),
RawExpression::ExternalWord => write!(f, "{}", self.span().slice(source)),
RawExpression::Synthetic(Synthetic::String(s)) => write!(f, "{:?}", s),
RawExpression::Variable(Variable::It(_)) => write!(f, "$it"),
RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)),
@ -225,6 +228,11 @@ impl From<Tagged<Path>> for Expression {
}
}
/// Literals are expressions that are:
///
/// 1. Copy
/// 2. Can be evaluated without additional context
/// 3. Evaluation cannot produce an error
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
pub enum Literal {
Number(Number),

46
src/parser/hir/baseline_parse.rs
View File

@ -1,10 +1,14 @@
use crate::context::Context;
use crate::errors::ShellError;
use crate::parser::{hir, RawToken, Token};
use crate::Text;
use std::path::PathBuf;
pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Expression {
match *token.item() {
pub fn baseline_parse_single_token(
token: &Token,
source: &Text,
) -> Result<hir::Expression, ShellError> {
Ok(match *token.item() {
RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.span()),
RawToken::Size(int, unit) => {
hir::Expression::size(int.to_number(source), unit, token.span())
@ -14,17 +18,22 @@ pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Express
hir::Expression::it_variable(span, token.span())
}
RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
RawToken::External(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
RawToken::Bare => hir::Expression::bare(token.span()),
}
})
}
pub fn baseline_parse_token_as_number(token: &Token, source: &Text) -> hir::Expression {
match *token.item() {
pub fn baseline_parse_token_as_number(
token: &Token,
source: &Text,
) -> Result<hir::Expression, ShellError> {
Ok(match *token.item() {
RawToken::Variable(span) if span.slice(source) == "it" => {
hir::Expression::it_variable(span, token.span())
}
RawToken::External(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.span()),
RawToken::Size(number, unit) => {
@ -32,33 +41,38 @@ pub fn baseline_parse_token_as_number(token: &Token, source: &Text) -> hir::Expr
}
RawToken::Bare => hir::Expression::bare(token.span()),
RawToken::String(span) => hir::Expression::string(span, token.span()),
}
})
}
pub fn baseline_parse_token_as_string(token: &Token, source: &Text) -> hir::Expression {
match *token.item() {
pub fn baseline_parse_token_as_string(
token: &Token,
source: &Text,
) -> Result<hir::Expression, ShellError> {
Ok(match *token.item() {
RawToken::Variable(span) if span.slice(source) == "it" => {
hir::Expression::it_variable(span, token.span())
}
RawToken::External(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
RawToken::Number(_) => hir::Expression::bare(token.span()),
RawToken::Size(_, _) => hir::Expression::bare(token.span()),
RawToken::Bare => hir::Expression::bare(token.span()),
RawToken::String(span) => hir::Expression::string(span, token.span()),
}
})
}
pub fn baseline_parse_token_as_path(
token: &Token,
context: &Context,
source: &Text,
) -> hir::Expression {
match *token.item() {
) -> Result<hir::Expression, ShellError> {
Ok(match *token.item() {
RawToken::Variable(span) if span.slice(source) == "it" => {
hir::Expression::it_variable(span, token.span())
}
RawToken::External(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
RawToken::Number(_) => hir::Expression::bare(token.span()),
RawToken::Size(_, _) => hir::Expression::bare(token.span()),
@ -69,7 +83,7 @@ pub fn baseline_parse_token_as_path(
RawToken::String(span) => {
hir::Expression::file_path(expand_path(span.slice(source), context), token.span())
}
}
})
}
pub fn expand_path(string: &str, context: &Context) -> PathBuf {

16
src/parser/hir/baseline_parse_tokens.rs
View File

@ -33,7 +33,6 @@ pub fn baseline_parse_tokens(
Ok(exprs)
}
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub enum SyntaxType {
Any,
@ -62,7 +61,7 @@ impl std::fmt::Display for SyntaxType {
SyntaxType::Path => write!(f, "Path"),
SyntaxType::Binary => write!(f, "Binary"),
SyntaxType::Block => write!(f, "Block"),
SyntaxType::Boolean => write!(f, "Boolean")
SyntaxType::Boolean => write!(f, "Boolean"),
}
}
}
@ -81,7 +80,7 @@ pub fn baseline_parse_next_expr(
match (syntax_type, next) {
(SyntaxType::Path, TokenNode::Token(token)) => {
return Ok(baseline_parse_token_as_path(token, context, source))
return baseline_parse_token_as_path(token, context, source)
}
(SyntaxType::Path, token) => {
@ -92,7 +91,7 @@ pub fn baseline_parse_next_expr(
}
(SyntaxType::String, TokenNode::Token(token)) => {
return Ok(baseline_parse_token_as_string(token, source));
return baseline_parse_token_as_string(token, source);
}
(SyntaxType::String, token) => {
@ -103,7 +102,7 @@ pub fn baseline_parse_next_expr(
}
(SyntaxType::Number, TokenNode::Token(token)) => {
return Ok(baseline_parse_token_as_number(token, source));
return Ok(baseline_parse_token_as_number(token, source)?);
}
(SyntaxType::Number, token) => {
@ -115,7 +114,7 @@ pub fn baseline_parse_next_expr(
// TODO: More legit member processing
(SyntaxType::Member, TokenNode::Token(token)) => {
return Ok(baseline_parse_token_as_string(token, source));
return baseline_parse_token_as_string(token, source);
}
(SyntaxType::Member, token) => {
@ -245,7 +244,7 @@ pub fn baseline_parse_semantic_token(
source: &Text,
) -> Result<hir::Expression, ShellError> {
match token {
TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)),
TokenNode::Token(token) => baseline_parse_single_token(token, source),
TokenNode::Call(_call) => unimplemented!(),
TokenNode::Delimited(delimited) => baseline_parse_delimited(delimited, context, source),
TokenNode::Pipeline(_pipeline) => unimplemented!(),
@ -315,7 +314,8 @@ pub fn baseline_parse_path(
RawToken::Number(_)
| RawToken::Size(..)
| RawToken::Variable(_)
| RawToken::External(_) => {
| RawToken::ExternalCommand(_)
| RawToken::ExternalWord => {
return Err(ShellError::type_error(
"String",
token.type_name().simple_spanned(part),

16
src/parser/parse/call_node.rs
View File

@ -1,5 +1,7 @@
use crate::parser::TokenNode;
use crate::traits::ToDebug;
use getset::Getters;
use std::fmt;
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
pub struct CallNode {
@ -24,3 +26,17 @@ impl CallNode {
}
}
}
impl ToDebug for CallNode {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
write!(f, "{}", self.head.debug(source))?;
if let Some(children) = &self.children {
for child in children {
write!(f, "{}", child.debug(source))?
}
}
Ok(())
}
}

161
src/parser/parse/parser.rs
View File

@ -236,12 +236,34 @@ pub fn bare(input: NomSpan) -> IResult<NomSpan, TokenNode> {
let start = input.offset;
let (input, _) = take_while1(is_start_bare_char)(input)?;
let (input, _) = take_while(is_bare_char)(input)?;
let next_char = &input.fragment.chars().nth(0);
if let Some(next_char) = next_char {
if is_external_word_char(*next_char) {
return Err(nom::Err::Error(nom::error::make_error(
input,
nom::error::ErrorKind::TakeWhile1,
)));
}
}
let end = input.offset;
Ok((input, TokenTreeBuilder::spanned_bare((start, end))))
})
}
pub fn external_word(input: NomSpan) -> IResult<NomSpan, TokenNode> {
trace_step(input, "bare", move |input| {
let start = input.offset;
let (input, _) = take_while1(is_external_word_char)(input)?;
let end = input.offset;
Ok((input, TokenTreeBuilder::spanned_external_word((start, end))))
})
}
pub fn var(input: NomSpan) -> IResult<NomSpan, TokenNode> {
trace_step(input, "var", move |input| {
let start = input.offset;
@ -364,8 +386,17 @@ pub fn size(input: NomSpan) -> IResult<NomSpan, TokenNode> {
pub fn leaf(input: NomSpan) -> IResult<NomSpan, TokenNode> {
trace_step(input, "leaf", move |input| {
let (input, node) =
alt((size, string, operator, flag, shorthand, var, external, bare))(input)?;
let (input, node) = alt((
size,
string,
operator,
flag,
shorthand,
var,
external,
bare,
external_word,
))(input)?;
Ok((input, node))
})
@ -582,26 +613,13 @@ pub fn pipeline(input: NomSpan) -> IResult<NomSpan, TokenNode> {
}
fn make_call_list(
head: Option<(
Option<NomSpan>,
Tagged<CallNode>,
Option<NomSpan>
)>,
items: Vec<(
NomSpan,
Option<NomSpan>,
Tagged<CallNode>,
Option<NomSpan>,
)>,
head: Option<(Option<NomSpan>, Tagged<CallNode>, Option<NomSpan>)>,
items: Vec<(NomSpan, Option<NomSpan>, Tagged<CallNode>, Option<NomSpan>)>,
) -> Vec<PipelineElement> {
let mut out = vec![];
if let Some(head) = head {
let el = PipelineElement::new(
None,
head.0.map(Span::from),
head.1,
head.2.map(Span::from));
let el = PipelineElement::new(None, head.0.map(Span::from), head.1, head.2.map(Span::from));
out.push(el);
}
@ -611,7 +629,8 @@ fn make_call_list(
Some(pipe).map(Span::from),
ws1.map(Span::from),
call,
ws2.map(Span::from));
ws2.map(Span::from),
);
out.push(el);
}
@ -628,40 +647,39 @@ fn int<T>(frag: &str, neg: Option<T>) -> i64 {
}
}
fn is_external_word_char(c: char) -> bool {
match c {
';' | '|' | '#' | '-' | '"' | '\'' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '`' => false,
other if other.is_whitespace() => false,
_ => true,
}
}
fn is_start_bare_char(c: char) -> bool {
match c {
'+' => false,
_ if c.is_alphabetic() => true,
_ if c.is_numeric() => true,
'.' => true,
'\\' => true,
'/' => true,
'_' => true,
'-' => true,
'@' => true,
'*' => true,
'?' => true,
'~' => true,
'+' => true,
_ => false,
}
}
fn is_bare_char(c: char) -> bool {
match c {
'+' => false,
_ if c.is_alphanumeric() => true,
':' => true,
'.' => true,
'\\' => true,
'/' => true,
'_' => true,
'-' => true,
'@' => true,
'*' => true,
'?' => true,
'=' => true,
'~' => true,
'+' => true,
'%' => true,
_ => false,
}
}
@ -724,6 +742,44 @@ mod tests {
}
}
macro_rules! equal_tokens {
($source:tt -> $tokens:expr) => {
let result = apply(pipeline, "pipeline", $source);
let (expected_tree, expected_source) = TokenTreeBuilder::build($tokens);
if result != expected_tree {
let debug_result = format!("{}", result.debug($source));
let debug_expected = format!("{}", expected_tree.debug(&expected_source));
if debug_result == debug_expected {
assert_eq!(
result, expected_tree,
"NOTE: actual and expected had equivalent debug serializations, source={:?}, debug_expected={:?}",
$source,
debug_expected
)
} else {
assert_eq!(debug_result, debug_expected)
}
}
// apply(pipeline, "pipeline", r#"cargo +nightly run"#),
// build_token(b::pipeline(vec![(
// None,
// b::call(
// b::bare("cargo"),
// vec![
// b::sp(),
// b::external_word("+nightly"),
// b::sp(),
// b::bare("run")
// ]
// ),
// None
// )]))
};
}
#[test]
fn test_integer() {
assert_leaf! {
@ -854,7 +910,7 @@ mod tests {
fn test_external() {
assert_leaf! {
parsers [ external ]
"^ls" -> 0..3 { External(span(1, 3)) }
"^ls" -> 0..3 { ExternalCommand(span(1, 3)) }
}
}
@ -1058,6 +1114,46 @@ mod tests {
);
}
#[test]
fn test_external_word() {
let _ = pretty_env_logger::try_init();
equal_tokens!(
"cargo +nightly run" ->
b::pipeline(vec![(
None,
b::call(
b::bare("cargo"),
vec![
b::sp(),
b::external_word("+nightly"),
b::sp(),
b::bare("run")
]
),
None
)])
);
equal_tokens!(
"rm foo%bar" ->
b::pipeline(vec![(
None,
b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]),
None
)])
);
equal_tokens!(
"rm foo%bar" ->
b::pipeline(vec![(
None,
b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]),
None
)])
);
}
#[test]
fn test_smoke_pipeline() {
let _ = pretty_env_logger::try_init();
@ -1178,7 +1274,6 @@ mod tests {
}
fn build_token(block: CurriedToken) -> TokenNode {
let mut builder = TokenTreeBuilder::new();
block(&mut builder)
TokenTreeBuilder::build(block).0
}
}

36
src/parser/parse/pipeline.rs
View File

@ -1,7 +1,9 @@
use crate::parser::CallNode;
use crate::traits::ToDebug;
use crate::{Span, Tagged};
use derive_new::new;
use getset::Getters;
use std::fmt;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)]
pub struct Pipeline {
@ -9,6 +11,20 @@ pub struct Pipeline {
pub(crate) post_ws: Option<Span>,
}
impl ToDebug for Pipeline {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
for part in &self.parts {
write!(f, "{}", part.debug(source))?;
}
if let Some(post_ws) = self.post_ws {
write!(f, "{}", post_ws.slice(source))?
}
Ok(())
}
}
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
pub struct PipelineElement {
pub pipe: Option<Span>,
@ -17,3 +33,23 @@ pub struct PipelineElement {
call: Tagged<CallNode>,
pub post_ws: Option<Span>,
}
impl ToDebug for PipelineElement {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
if let Some(pipe) = self.pipe {
write!(f, "{}", pipe.slice(source))?;
}
if let Some(pre_ws) = self.pre_ws {
write!(f, "{}", pre_ws.slice(source))?;
}
write!(f, "{}", self.call.debug(source))?;
if let Some(post_ws) = self.post_ws {
write!(f, "{}", post_ws.slice(source))?;
}
Ok(())
}
}

21
src/parser/parse/token_tree.rs
View File

@ -1,5 +1,6 @@
use crate::errors::ShellError;
use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
use crate::traits::ToDebug;
use crate::{Span, Tagged, Text};
use derive_new::new;
use enum_utils::FromStr;
@ -22,6 +23,12 @@ pub enum TokenNode {
Path(Tagged<PathNode>),
}
impl ToDebug for TokenNode {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
write!(f, "{:?}", self.old_debug(&Text::from(source)))
}
}
pub struct DebugTokenNode<'a> {
node: &'a TokenNode,
source: &'a Text,
@ -34,11 +41,11 @@ impl fmt::Debug for DebugTokenNode<'_> {
TokenNode::Call(s) => {
write!(f, "(")?;
write!(f, "{:?}", s.head().debug(self.source))?;
write!(f, "{}", s.head().debug(self.source))?;
if let Some(children) = s.children() {
for child in children {
write!(f, "{:?}", child.debug(self.source))?;
write!(f, "{}", child.debug(self.source))?;
}
}
@ -57,7 +64,7 @@ impl fmt::Debug for DebugTokenNode<'_> {
)?;
for child in d.children() {
write!(f, "{:?}", child.debug(self.source))?;
write!(f, "{:?}", child.old_debug(self.source))?;
}
write!(
@ -70,7 +77,7 @@ impl fmt::Debug for DebugTokenNode<'_> {
}
)
}
TokenNode::Pipeline(_) => write!(f, "<todo:pipeline>"),
TokenNode::Pipeline(pipeline) => write!(f, "{}", pipeline.debug(self.source)),
TokenNode::Error(s) => write!(f, "<error> for {:?}", s.span().slice(self.source)),
rest => write!(f, "{}", rest.span().slice(self.source)),
}
@ -115,7 +122,7 @@ impl TokenNode {
.to_string()
}
pub fn debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> {
pub fn old_debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> {
DebugTokenNode { node: self, source }
}
@ -140,7 +147,7 @@ impl TokenNode {
pub fn is_external(&self) -> bool {
match self {
TokenNode::Token(Tagged {
item: RawToken::External(..),
item: RawToken::ExternalCommand(..),
..
}) => true,
_ => false,
@ -150,7 +157,7 @@ impl TokenNode {
pub fn expect_external(&self) -> Span {
match self {
TokenNode::Token(Tagged {
item: RawToken::External(span),
item: RawToken::ExternalCommand(span),
..
}) => *span,
_ => panic!("Only call expect_external if you checked is_external first"),

32
src/parser/parse/token_tree_builder.rs
View File

@ -14,15 +14,19 @@ use derive_new::new;
pub struct TokenTreeBuilder {
#[new(default)]
pos: usize,
#[new(default)]
output: String,
}
pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> TokenNode + 'static>;
pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Tagged<CallNode> + 'static>;
impl TokenTreeBuilder {
pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> TokenNode {
pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) {
let mut builder = TokenTreeBuilder::new();
block(&mut builder)
let node = block(&mut builder);
(node, builder.output)
}
pub fn pipeline(input: Vec<(Option<&str>, CurriedCall, Option<&str>)>) -> CurriedToken {
@ -56,7 +60,8 @@ impl TokenTreeBuilder {
pipe,
pre_span.map(Span::from),
call,
post_span.map(Span::from)));
post_span.map(Span::from),
));
loop {
match input.next() {
@ -147,9 +152,27 @@ impl TokenTreeBuilder {
))
}
pub fn external_word(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, end) = b.consume(&input);
b.pos = end;
TokenTreeBuilder::spanned_external_word((start, end))
})
}
pub fn spanned_external_word(input: impl Into<Span>) -> TokenNode {
TokenNode::Token(Tagged::from_simple_spanned_item(
RawToken::ExternalWord,
input.into(),
))
}
pub fn spanned_external(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Token(Tagged::from_simple_spanned_item(
RawToken::External(input.into()),
RawToken::ExternalCommand(input.into()),
span.into(),
))
}
@ -422,6 +445,7 @@ impl TokenTreeBuilder {
fn consume(&mut self, input: &str) -> (usize, usize) {
let start = self.pos;
self.pos += input.len();
self.output.push_str(input);
(start, self.pos)
}
}

6
src/parser/parse/tokens.rs
View File

@ -10,7 +10,8 @@ pub enum RawToken {
Size(RawNumber, Unit),
String(Span),
Variable(Span),
External(Span),
ExternalCommand(Span),
ExternalWord,
Bare,
}
@ -50,7 +51,8 @@ impl RawToken {
RawToken::Size(..) => "Size",
RawToken::String(_) => "String",
RawToken::Variable(_) => "Variable",
RawToken::External(_) => "External",
RawToken::ExternalCommand(_) => "ExternalCommand",
RawToken::ExternalWord => "ExternalWord",
RawToken::Bare => "String",
}
}

3
src/parser/parse_command.rs
View File

@ -6,6 +6,7 @@ use crate::parser::{
hir::{self, NamedArguments},
Flag, RawToken, TokenNode,
};
use crate::traits::ToDebug;
use crate::{Span, Tag, Tagged, Text};
use log::trace;
@ -248,7 +249,7 @@ pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source
itertools::join(
tail.debug_remaining()
.iter()
.map(|i| format!("%{:?}%", i.debug(source))),
.map(|i| format!("%{}%", i.debug(&source))),
" "
)
);

6
src/shell/helper.rs
View File

@ -136,9 +136,13 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String {
..
}) => Color::Green.normal().paint(token_node.span().slice(line)),
TokenNode::Token(Tagged {
item: RawToken::External(..),
item: RawToken::ExternalCommand(..),
..
}) => Color::Cyan.bold().paint(token_node.span().slice(line)),
TokenNode::Token(Tagged {
item: RawToken::ExternalWord,
..
}) => Color::Black.bold().paint(token_node.span().slice(line)),
};
styled.to_string()