extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-01-08 23:40:17 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #632 from nushell/improve-external-words

Browse Source 
Close a bunch of holes in external command args
This commit is contained in:
Andrés N. Robalino 2019-09-10 12:37:43 -05:00 committed by GitHub
commit f47349c1a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 447 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/commands/cp.rs
View File

@ -21,7 +21,7 @@ impl PerItemCommand for Cpy {
fn signature(&self) -> Signature {
Signature::build("cp")
.required("src", SyntaxType::Path)
.required("src", SyntaxType::Pattern)
.required("dst", SyntaxType::Path)
.named("file", SyntaxType::Any)
.switch("recursive")

2
src/commands/ls.rs
View File

@ -10,7 +10,7 @@ impl WholeStreamCommand for LS {
}
fn signature(&self) -> Signature {
Signature::build("ls").optional("path", SyntaxType::Path)
Signature::build("ls").optional("path", SyntaxType::Pattern)
}
fn usage(&self) -> &str {

1
src/commands/to_bson.rs
View File

@ -50,6 +50,7 @@ pub fn value_to_bson_value(v: &Tagged<Value>) -> Result<Bson, ShellError> {
}
Value::Primitive(Primitive::Nothing) => Bson::Null,
Value::Primitive(Primitive::String(s)) => Bson::String(s.clone()),
Value::Primitive(Primitive::Pattern(p)) => Bson::String(p.clone()),
Value::Primitive(Primitive::Path(s)) => Bson::String(s.display().to_string()),
Value::Table(l) => Bson::Array(
l.iter()

1
src/commands/to_json.rs
View File

@ -45,6 +45,7 @@ pub fn value_to_json_value(v: &Tagged<Value>) -> Result<serde_json::Value, Shell
CoerceInto::<i64>::coerce_into(i.tagged(v.tag), "converting to JSON number")?,
)),
Value::Primitive(Primitive::Nothing) => serde_json::Value::Null,
Value::Primitive(Primitive::Pattern(s)) => serde_json::Value::String(s.clone()),
Value::Primitive(Primitive::String(s)) => serde_json::Value::String(s.clone()),
Value::Primitive(Primitive::Path(s)) => serde_json::Value::String(s.display().to_string()),

1
src/commands/to_sqlite.rs
View File

@ -91,6 +91,7 @@ fn nu_value_to_sqlite_string(v: Value) -> String {
Primitive::Int(i) => format!("{}", i),
Primitive::Decimal(f) => format!("{}", f),
Primitive::Bytes(u) => format!("{}", u),
Primitive::Pattern(s) => format!("'{}'", s.replace("'", "''")),
Primitive::String(s) => format!("'{}'", s.replace("'", "''")),
Primitive::Boolean(true) => "1".into(),
Primitive::Boolean(_) => "0".into(),

1
src/commands/to_toml.rs
View File

@ -44,6 +44,7 @@ pub fn value_to_toml_value(v: &Tagged<Value>) -> Result<toml::Value, ShellError>
toml::Value::Integer(i.tagged(v.tag).coerce_into("converting to TOML integer")?)
}
Value::Primitive(Primitive::Nothing) => toml::Value::String("<Nothing>".to_string()),
Value::Primitive(Primitive::Pattern(s)) => toml::Value::String(s.clone()),
Value::Primitive(Primitive::String(s)) => toml::Value::String(s.clone()),
Value::Primitive(Primitive::Path(s)) => toml::Value::String(s.display().to_string()),

1
src/commands/to_yaml.rs
View File

@ -42,6 +42,7 @@ pub fn value_to_yaml_value(v: &Tagged<Value>) -> Result<serde_yaml::Value, Shell
CoerceInto::<i64>::coerce_into(i.tagged(v.tag), "converting to YAML number")?,
)),
Value::Primitive(Primitive::Nothing) => serde_yaml::Value::Null,
Value::Primitive(Primitive::Pattern(s)) => serde_yaml::Value::String(s.clone()),
Value::Primitive(Primitive::String(s)) => serde_yaml::Value::String(s.clone()),
Value::Primitive(Primitive::Path(s)) => serde_yaml::Value::String(s.display().to_string()),

8
src/data/base.rs
View File

@ -20,6 +20,7 @@ pub enum Primitive {
Decimal(BigDecimal),
Bytes(u64),
String(String),
Pattern(String),
Boolean(bool),
Date(DateTime<Utc>),
Path(PathBuf),
@ -53,6 +54,7 @@ impl Primitive {
Int(_) => "int",
Decimal(_) => "decimal",
Bytes(_) => "bytes",
Pattern(_) => "pattern",
String(_) => "string",
Boolean(_) => "boolean",
Date(_) => "date",
@ -71,6 +73,7 @@ impl Primitive {
Path(path) => write!(f, "{}", path.display()),
Decimal(decimal) => write!(f, "{}", decimal),
Bytes(bytes) => write!(f, "{}", bytes),
Pattern(string) => write!(f, "{:?}", string),
String(string) => write!(f, "{:?}", string),
Boolean(boolean) => write!(f, "{}", boolean),
Date(date) => write!(f, "{}", date),
@ -108,6 +111,7 @@ impl Primitive {
}
Primitive::Int(i) => format!("{}", i),
Primitive::Decimal(decimal) => format!("{}", decimal),
Primitive::Pattern(s) => format!("{}", s),
Primitive::String(s) => format!("{}", s),
Primitive::Boolean(b) => match (b, field_name) {
(true, None) => format!("Yes"),
@ -578,6 +582,10 @@ impl Value {
Value::Primitive(Primitive::String(s.into()))
}
pub fn pattern(s: impl Into<String>) -> Value {
Value::Primitive(Primitive::String(s.into()))
}
pub fn path(s: impl Into<PathBuf>) -> Value {
Value::Primitive(Primitive::Path(s.into()))
}

15
src/errors.rs
View File

@ -38,6 +38,7 @@ pub enum ArgumentError {
MissingMandatoryFlag(String),
MissingMandatoryPositional(String),
MissingValueForName(String),
InvalidExternalWord,
}
#[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)]
@ -136,6 +137,16 @@ impl ShellError {
.start()
}
pub(crate) fn invalid_external_word(span: Span) -> ShellError {
ProximateShellError::ArgumentError {
command: "Invalid argument to Nu command (did you mean to call an external command?)"
.into(),
error: ArgumentError::InvalidExternalWord,
span,
}
.start()
}
pub(crate) fn parse_error(
error: nom::Err<(nom5_locate::LocatedSpan<&str>, nom::error::ErrorKind)>,
) -> ShellError {
@ -190,6 +201,10 @@ impl ShellError {
error,
span,
} => match error {
ArgumentError::InvalidExternalWord => Diagnostic::new(
Severity::Error,
format!("Invalid bare word for Nu command (did you intend to invoke an external command?)"))
.with_label(Label::new_primary(span)),
ArgumentError::MissingMandatoryFlag(name) => Diagnostic::new(
Severity::Error,
format!(

8
src/evaluate/evaluator.rs
View File

@ -1,5 +1,5 @@
use crate::data::base::Block;
use crate::errors::Description;
use crate::errors::{ArgumentError, Description};
use crate::parser::{
hir::{self, Expression, RawExpression},
CommandRegistry, Text,
@ -39,6 +39,11 @@ pub(crate) fn evaluate_baseline_expr(
) -> Result<Tagged<Value>, ShellError> {
match &expr.item {
RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(literal), source)),
RawExpression::ExternalWord => Err(ShellError::argument_error(
"Invalid external word",
ArgumentError::InvalidExternalWord,
expr.span(),
)),
RawExpression::FilePath(path) => Ok(Value::path(path.clone()).tagged(expr.span())),
RawExpression::Synthetic(hir::Synthetic::String(s)) => Ok(Value::string(s).tagged_unknown()),
RawExpression::Variable(var) => evaluate_reference(var, scope, source),
@ -109,6 +114,7 @@ fn evaluate_literal(literal: Tagged<&hir::Literal>, source: &Text) -> Tagged<Val
hir::Literal::Number(int) => int.into(),
hir::Literal::Size(int, unit) => unit.compute(int),
hir::Literal::String(span) => Value::string(span.slice(source)),
hir::Literal::GlobPattern => Value::pattern(literal.span().slice(source)),
hir::Literal::Bare => Value::string(literal.span().slice(source)),
};

18
src/parser/hir.rs
View File

@ -17,7 +17,7 @@ use crate::evaluate::Scope;
pub(crate) use self::baseline_parse::{
baseline_parse_single_token, baseline_parse_token_as_number, baseline_parse_token_as_path,
baseline_parse_token_as_string,
baseline_parse_token_as_pattern, baseline_parse_token_as_string,
};
pub(crate) use self::baseline_parse_tokens::{baseline_parse_next_expr, TokensIterator};
pub(crate) use self::binary::Binary;
@ -83,12 +83,14 @@ impl ToDebug for Call {
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
pub enum RawExpression {
Literal(Literal),
ExternalWord,
Synthetic(Synthetic),
Variable(Variable),
Binary(Box<Binary>),
Block(Vec<Expression>),
List(Vec<Expression>),
Path(Box<Path>),
FilePath(PathBuf),
ExternalCommand(ExternalCommand),
@ -113,6 +115,7 @@ impl RawExpression {
match self {
RawExpression::Literal(literal) => literal.type_name(),
RawExpression::Synthetic(synthetic) => synthetic.type_name(),
RawExpression::ExternalWord => "externalword",
RawExpression::FilePath(..) => "filepath",
RawExpression::Variable(..) => "variable",
RawExpression::List(..) => "list",
@ -162,6 +165,10 @@ impl Expression {
Tagged::from_simple_spanned_item(RawExpression::Literal(Literal::Bare), span.into())
}
pub(crate) fn pattern(tag: impl Into<Tag>) -> Expression {
RawExpression::Literal(Literal::GlobPattern).tagged(tag.into())
}
pub(crate) fn variable(inner: impl Into<Span>, outer: impl Into<Span>) -> Expression {
Tagged::from_simple_spanned_item(
RawExpression::Variable(Variable::Other(inner.into())),
@ -189,6 +196,7 @@ impl ToDebug for Expression {
match self.item() {
RawExpression::Literal(l) => l.tagged(self.span()).fmt_debug(f, source),
RawExpression::FilePath(p) => write!(f, "{}", p.display()),
RawExpression::ExternalWord => write!(f, "{}", self.span().slice(source)),
RawExpression::Synthetic(Synthetic::String(s)) => write!(f, "{:?}", s),
RawExpression::Variable(Variable::It(_)) => write!(f, "$it"),
RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)),
@ -225,11 +233,17 @@ impl From<Tagged<Path>> for Expression {
}
}
/// Literals are expressions that are:
///
/// 1. Copy
/// 2. Can be evaluated without additional context
/// 3. Evaluation cannot produce an error
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
pub enum Literal {
Number(Number),
Size(Number, Unit),
String(Span),
GlobPattern,
Bare,
}
@ -239,6 +253,7 @@ impl ToDebug for Tagged<&Literal> {
Literal::Number(number) => write!(f, "{:?}", *number),
Literal::Size(number, unit) => write!(f, "{:?}{:?}", *number, unit),
Literal::String(span) => write!(f, "{}", span.slice(source)),
Literal::GlobPattern => write!(f, "{}", self.span().slice(source)),
Literal::Bare => write!(f, "{}", self.span().slice(source)),
}
}
@ -251,6 +266,7 @@ impl Literal {
Literal::Size(..) => "size",
Literal::String(..) => "string",
Literal::Bare => "string",
Literal::GlobPattern => "pattern",
}
}
}

95
src/parser/hir/baseline_parse.rs
View File

@ -1,10 +1,15 @@
use crate::context::Context;
use crate::errors::ShellError;
use crate::parser::{hir, RawToken, Token};
use crate::TaggedItem;
use crate::Text;
use std::path::PathBuf;
pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Expression {
match *token.item() {
pub fn baseline_parse_single_token(
token: &Token,
source: &Text,
) -> Result<hir::Expression, ShellError> {
Ok(match *token.item() {
RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.span()),
RawToken::Size(int, unit) => {
hir::Expression::size(int.to_number(source), unit, token.span())
@ -14,51 +19,74 @@ pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Express
hir::Expression::it_variable(span, token.span())
}
RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
RawToken::External(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
RawToken::GlobPattern => hir::Expression::pattern(token.span()),
RawToken::Bare => hir::Expression::bare(token.span()),
}
})
}
pub fn baseline_parse_token_as_number(token: &Token, source: &Text) -> hir::Expression {
match *token.item() {
pub fn baseline_parse_token_as_number(
token: &Token,
source: &Text,
) -> Result<hir::Expression, ShellError> {
Ok(match *token.item() {
RawToken::Variable(span) if span.slice(source) == "it" => {
hir::Expression::it_variable(span, token.span())
}
RawToken::External(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.span()),
RawToken::Size(number, unit) => {
hir::Expression::size(number.to_number(source), unit, token.span())
}
RawToken::Bare => hir::Expression::bare(token.span()),
RawToken::GlobPattern => {
return Err(ShellError::type_error(
"Number",
"glob pattern".to_string().tagged(token.tag()),
))
}
RawToken::String(span) => hir::Expression::string(span, token.span()),
}
})
}
pub fn baseline_parse_token_as_string(token: &Token, source: &Text) -> hir::Expression {
match *token.item() {
pub fn baseline_parse_token_as_string(
token: &Token,
source: &Text,
) -> Result<hir::Expression, ShellError> {
Ok(match *token.item() {
RawToken::Variable(span) if span.slice(source) == "it" => {
hir::Expression::it_variable(span, token.span())
}
RawToken::External(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
RawToken::Number(_) => hir::Expression::bare(token.span()),
RawToken::Size(_, _) => hir::Expression::bare(token.span()),
RawToken::Bare => hir::Expression::bare(token.span()),
RawToken::GlobPattern => {
return Err(ShellError::type_error(
"String",
"glob pattern".tagged(token.tag()),
))
}
RawToken::String(span) => hir::Expression::string(span, token.span()),
}
})
}
pub fn baseline_parse_token_as_path(
token: &Token,
context: &Context,
source: &Text,
) -> hir::Expression {
match *token.item() {
) -> Result<hir::Expression, ShellError> {
Ok(match *token.item() {
RawToken::Variable(span) if span.slice(source) == "it" => {
hir::Expression::it_variable(span, token.span())
}
RawToken::External(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()),
RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
RawToken::Number(_) => hir::Expression::bare(token.span()),
RawToken::Size(_, _) => hir::Expression::bare(token.span()),
@ -66,10 +94,45 @@ pub fn baseline_parse_token_as_path(
expand_path(token.span().slice(source), context),
token.span(),
),
RawToken::GlobPattern => {
return Err(ShellError::type_error(
"Path",
"glob pattern".tagged(token.tag()),
))
}
RawToken::String(span) => {
hir::Expression::file_path(expand_path(span.slice(source), context), token.span())
}
}
})
}
pub fn baseline_parse_token_as_pattern(
token: &Token,
context: &Context,
source: &Text,
) -> Result<hir::Expression, ShellError> {
Ok(match *token.item() {
RawToken::Variable(span) if span.slice(source) == "it" => {
hir::Expression::it_variable(span, token.span())
}
RawToken::ExternalCommand(_) => {
return Err(ShellError::syntax_error(
"Invalid external command".to_string().tagged(token.tag()),
))
}
RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())),
RawToken::Variable(span) => hir::Expression::variable(span, token.span()),
RawToken::Number(_) => hir::Expression::bare(token.span()),
RawToken::Size(_, _) => hir::Expression::bare(token.span()),
RawToken::GlobPattern => hir::Expression::pattern(token.span()),
RawToken::Bare => hir::Expression::file_path(
expand_path(token.span().slice(source), context),
token.span(),
),
RawToken::String(span) => {
hir::Expression::file_path(expand_path(span.slice(source), context), token.span())
}
})
}
pub fn expand_path(string: &str, context: &Context) -> PathBuf {

32
src/parser/hir/baseline_parse_tokens.rs
View File

@ -4,7 +4,7 @@ use crate::parser::{
hir,
hir::{
baseline_parse_single_token, baseline_parse_token_as_number, baseline_parse_token_as_path,
baseline_parse_token_as_string,
baseline_parse_token_as_pattern, baseline_parse_token_as_string,
},
DelimitedNode, Delimiter, PathNode, RawToken, TokenNode,
};
@ -33,7 +33,6 @@ pub fn baseline_parse_tokens(
Ok(exprs)
}
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub enum SyntaxType {
Any,
@ -44,6 +43,7 @@ pub enum SyntaxType {
Variable,
Number,
Path,
Pattern,
Binary,
Block,
Boolean,
@ -60,9 +60,10 @@ impl std::fmt::Display for SyntaxType {
SyntaxType::Variable => write!(f, "Variable"),
SyntaxType::Number => write!(f, "Number"),
SyntaxType::Path => write!(f, "Path"),
SyntaxType::Pattern => write!(f, "Pattern"),
SyntaxType::Binary => write!(f, "Binary"),
SyntaxType::Block => write!(f, "Block"),
SyntaxType::Boolean => write!(f, "Boolean")
SyntaxType::Boolean => write!(f, "Boolean"),
}
}
}
@ -81,7 +82,7 @@ pub fn baseline_parse_next_expr(
match (syntax_type, next) {
(SyntaxType::Path, TokenNode::Token(token)) => {
return Ok(baseline_parse_token_as_path(token, context, source))
return baseline_parse_token_as_path(token, context, source)
}
(SyntaxType::Path, token) => {
@ -91,8 +92,19 @@ pub fn baseline_parse_next_expr(
))
}
(SyntaxType::Pattern, TokenNode::Token(token)) => {
return baseline_parse_token_as_pattern(token, context, source)
}
(SyntaxType::Pattern, token) => {
return Err(ShellError::type_error(
"Path",
token.type_name().simple_spanned(token.span()),
))
}
(SyntaxType::String, TokenNode::Token(token)) => {
return Ok(baseline_parse_token_as_string(token, source));
return baseline_parse_token_as_string(token, source);
}
(SyntaxType::String, token) => {
@ -103,7 +115,7 @@ pub fn baseline_parse_next_expr(
}
(SyntaxType::Number, TokenNode::Token(token)) => {
return Ok(baseline_parse_token_as_number(token, source));
return Ok(baseline_parse_token_as_number(token, source)?);
}
(SyntaxType::Number, token) => {
@ -115,7 +127,7 @@ pub fn baseline_parse_next_expr(
// TODO: More legit member processing
(SyntaxType::Member, TokenNode::Token(token)) => {
return Ok(baseline_parse_token_as_string(token, source));
return baseline_parse_token_as_string(token, source);
}
(SyntaxType::Member, token) => {
@ -245,7 +257,7 @@ pub fn baseline_parse_semantic_token(
source: &Text,
) -> Result<hir::Expression, ShellError> {
match token {
TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)),
TokenNode::Token(token) => baseline_parse_single_token(token, source),
TokenNode::Call(_call) => unimplemented!(),
TokenNode::Delimited(delimited) => baseline_parse_delimited(delimited, context, source),
TokenNode::Pipeline(_pipeline) => unimplemented!(),
@ -315,7 +327,9 @@ pub fn baseline_parse_path(
RawToken::Number(_)
| RawToken::Size(..)
| RawToken::Variable(_)
| RawToken::External(_) => {
| RawToken::ExternalCommand(_)
| RawToken::GlobPattern
| RawToken::ExternalWord => {
return Err(ShellError::type_error(
"String",
token.type_name().simple_spanned(part),

16
src/parser/parse/call_node.rs
View File

@ -1,5 +1,7 @@
use crate::parser::TokenNode;
use crate::traits::ToDebug;
use getset::Getters;
use std::fmt;
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)]
pub struct CallNode {
@ -24,3 +26,17 @@ impl CallNode {
}
}
}
impl ToDebug for CallNode {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
write!(f, "{}", self.head.debug(source))?;
if let Some(children) = &self.children {
for child in children {
write!(f, "{}", child.debug(source))?
}
}
Ok(())
}
}

195
src/parser/parse/parser.rs
View File

@ -231,17 +231,62 @@ pub fn external(input: NomSpan) -> IResult<NomSpan, TokenNode> {
})
}
pub fn pattern(input: NomSpan) -> IResult<NomSpan, TokenNode> {
trace_step(input, "bare", move |input| {
let start = input.offset;
let (input, _) = take_while1(is_start_glob_char)(input)?;
let (input, _) = take_while(is_glob_char)(input)?;
let next_char = &input.fragment.chars().nth(0);
if let Some(next_char) = next_char {
if is_external_word_char(*next_char) {
return Err(nom::Err::Error(nom::error::make_error(
input,
nom::error::ErrorKind::TakeWhile1,
)));
}
}
let end = input.offset;
Ok((input, TokenTreeBuilder::spanned_pattern((start, end))))
})
}
pub fn bare(input: NomSpan) -> IResult<NomSpan, TokenNode> {
trace_step(input, "bare", move |input| {
let start = input.offset;
let (input, _) = take_while1(is_start_bare_char)(input)?;
let (input, _) = take_while(is_bare_char)(input)?;
let next_char = &input.fragment.chars().nth(0);
if let Some(next_char) = next_char {
if is_external_word_char(*next_char) || *next_char == '*' {
return Err(nom::Err::Error(nom::error::make_error(
input,
nom::error::ErrorKind::TakeWhile1,
)));
}
}
let end = input.offset;
Ok((input, TokenTreeBuilder::spanned_bare((start, end))))
})
}
pub fn external_word(input: NomSpan) -> IResult<NomSpan, TokenNode> {
trace_step(input, "bare", move |input| {
let start = input.offset;
let (input, _) = take_while1(is_external_word_char)(input)?;
let end = input.offset;
Ok((input, TokenTreeBuilder::spanned_external_word((start, end))))
})
}
pub fn var(input: NomSpan) -> IResult<NomSpan, TokenNode> {
trace_step(input, "var", move |input| {
let start = input.offset;
@ -364,8 +409,18 @@ pub fn size(input: NomSpan) -> IResult<NomSpan, TokenNode> {
pub fn leaf(input: NomSpan) -> IResult<NomSpan, TokenNode> {
trace_step(input, "leaf", move |input| {
let (input, node) =
alt((size, string, operator, flag, shorthand, var, external, bare))(input)?;
let (input, node) = alt((
size,
string,
operator,
flag,
shorthand,
var,
external,
bare,
pattern,
external_word,
))(input)?;
Ok((input, node))
})
@ -582,26 +637,13 @@ pub fn pipeline(input: NomSpan) -> IResult<NomSpan, TokenNode> {
}
fn make_call_list(
head: Option<(
Option<NomSpan>,
Tagged<CallNode>,
Option<NomSpan>
)>,
items: Vec<(
NomSpan,
Option<NomSpan>,
Tagged<CallNode>,
Option<NomSpan>,
)>,
head: Option<(Option<NomSpan>, Tagged<CallNode>, Option<NomSpan>)>,
items: Vec<(NomSpan, Option<NomSpan>, Tagged<CallNode>, Option<NomSpan>)>,
) -> Vec<PipelineElement> {
let mut out = vec![];
if let Some(head) = head {
let el = PipelineElement::new(
None,
head.0.map(Span::from),
head.1,
head.2.map(Span::from));
let el = PipelineElement::new(None, head.0.map(Span::from), head.1, head.2.map(Span::from));
out.push(el);
}
@ -611,7 +653,8 @@ fn make_call_list(
Some(pipe).map(Span::from),
ws1.map(Span::from),
call,
ws2.map(Span::from));
ws2.map(Span::from),
);
out.push(el);
}
@ -628,40 +671,49 @@ fn int<T>(frag: &str, neg: Option<T>) -> i64 {
}
}
fn is_external_word_char(c: char) -> bool {
match c {
';' | '|' | '#' | '-' | '"' | '\'' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '`' => false,
other if other.is_whitespace() => false,
_ => true,
}
}
fn is_start_glob_char(c: char) -> bool {
is_start_bare_char(c) || c == '*'
}
fn is_glob_char(c: char) -> bool {
is_bare_char(c) || c == '*'
}
fn is_start_bare_char(c: char) -> bool {
match c {
'+' => false,
_ if c.is_alphabetic() => true,
_ if c.is_numeric() => true,
'.' => true,
'\\' => true,
'/' => true,
'_' => true,
'-' => true,
'@' => true,
'*' => true,
'?' => true,
'~' => true,
'+' => true,
_ => false,
}
}
fn is_bare_char(c: char) -> bool {
match c {
'+' => false,
_ if c.is_alphanumeric() => true,
':' => true,
'.' => true,
'\\' => true,
'/' => true,
'_' => true,
'-' => true,
'@' => true,
'*' => true,
'?' => true,
'=' => true,
'~' => true,
'+' => true,
'%' => true,
':' => true,
'?' => true,
_ => false,
}
}
@ -724,6 +776,44 @@ mod tests {
}
}
macro_rules! equal_tokens {
($source:tt -> $tokens:expr) => {
let result = apply(pipeline, "pipeline", $source);
let (expected_tree, expected_source) = TokenTreeBuilder::build($tokens);
if result != expected_tree {
let debug_result = format!("{}", result.debug($source));
let debug_expected = format!("{}", expected_tree.debug(&expected_source));
if debug_result == debug_expected {
assert_eq!(
result, expected_tree,
"NOTE: actual and expected had equivalent debug serializations, source={:?}, debug_expected={:?}",
$source,
debug_expected
)
} else {
assert_eq!(debug_result, debug_expected)
}
}
// apply(pipeline, "pipeline", r#"cargo +nightly run"#),
// build_token(b::pipeline(vec![(
// None,
// b::call(
// b::bare("cargo"),
// vec![
// b::sp(),
// b::external_word("+nightly"),
// b::sp(),
// b::bare("run")
// ]
// ),
// None
// )]))
};
}
#[test]
fn test_integer() {
assert_leaf! {
@ -854,7 +944,7 @@ mod tests {
fn test_external() {
assert_leaf! {
parsers [ external ]
"^ls" -> 0..3 { External(span(1, 3)) }
"^ls" -> 0..3 { ExternalCommand(span(1, 3)) }
}
}
@ -1058,6 +1148,46 @@ mod tests {
);
}
#[test]
fn test_external_word() {
let _ = pretty_env_logger::try_init();
equal_tokens!(
"cargo +nightly run" ->
b::pipeline(vec![(
None,
b::call(
b::bare("cargo"),
vec![
b::sp(),
b::external_word("+nightly"),
b::sp(),
b::bare("run")
]
),
None
)])
);
equal_tokens!(
"rm foo%bar" ->
b::pipeline(vec![(
None,
b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]),
None
)])
);
equal_tokens!(
"rm foo%bar" ->
b::pipeline(vec![(
None,
b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]),
None
)])
);
}
#[test]
fn test_smoke_pipeline() {
let _ = pretty_env_logger::try_init();
@ -1178,7 +1308,6 @@ mod tests {
}
fn build_token(block: CurriedToken) -> TokenNode {
let mut builder = TokenTreeBuilder::new();
block(&mut builder)
TokenTreeBuilder::build(block).0
}
}

36
src/parser/parse/pipeline.rs
View File

@ -1,7 +1,9 @@
use crate::parser::CallNode;
use crate::traits::ToDebug;
use crate::{Span, Tagged};
use derive_new::new;
use getset::Getters;
use std::fmt;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)]
pub struct Pipeline {
@ -9,6 +11,20 @@ pub struct Pipeline {
pub(crate) post_ws: Option<Span>,
}
impl ToDebug for Pipeline {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
for part in &self.parts {
write!(f, "{}", part.debug(source))?;
}
if let Some(post_ws) = self.post_ws {
write!(f, "{}", post_ws.slice(source))?
}
Ok(())
}
}
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)]
pub struct PipelineElement {
pub pipe: Option<Span>,
@ -17,3 +33,23 @@ pub struct PipelineElement {
call: Tagged<CallNode>,
pub post_ws: Option<Span>,
}
impl ToDebug for PipelineElement {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
if let Some(pipe) = self.pipe {
write!(f, "{}", pipe.slice(source))?;
}
if let Some(pre_ws) = self.pre_ws {
write!(f, "{}", pre_ws.slice(source))?;
}
write!(f, "{}", self.call.debug(source))?;
if let Some(post_ws) = self.post_ws {
write!(f, "{}", post_ws.slice(source))?;
}
Ok(())
}
}

21
src/parser/parse/token_tree.rs
View File

@ -1,5 +1,6 @@
use crate::errors::ShellError;
use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*};
use crate::traits::ToDebug;
use crate::{Span, Tagged, Text};
use derive_new::new;
use enum_utils::FromStr;
@ -22,6 +23,12 @@ pub enum TokenNode {
Path(Tagged<PathNode>),
}
impl ToDebug for TokenNode {
fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result {
write!(f, "{:?}", self.old_debug(&Text::from(source)))
}
}
pub struct DebugTokenNode<'a> {
node: &'a TokenNode,
source: &'a Text,
@ -34,11 +41,11 @@ impl fmt::Debug for DebugTokenNode<'_> {
TokenNode::Call(s) => {
write!(f, "(")?;
write!(f, "{:?}", s.head().debug(self.source))?;
write!(f, "{}", s.head().debug(self.source))?;
if let Some(children) = s.children() {
for child in children {
write!(f, "{:?}", child.debug(self.source))?;
write!(f, "{}", child.debug(self.source))?;
}
}
@ -57,7 +64,7 @@ impl fmt::Debug for DebugTokenNode<'_> {
)?;
for child in d.children() {
write!(f, "{:?}", child.debug(self.source))?;
write!(f, "{:?}", child.old_debug(self.source))?;
}
write!(
@ -70,7 +77,7 @@ impl fmt::Debug for DebugTokenNode<'_> {
}
)
}
TokenNode::Pipeline(_) => write!(f, "<todo:pipeline>"),
TokenNode::Pipeline(pipeline) => write!(f, "{}", pipeline.debug(self.source)),
TokenNode::Error(s) => write!(f, "<error> for {:?}", s.span().slice(self.source)),
rest => write!(f, "{}", rest.span().slice(self.source)),
}
@ -115,7 +122,7 @@ impl TokenNode {
.to_string()
}
pub fn debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> {
pub fn old_debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> {
DebugTokenNode { node: self, source }
}
@ -140,7 +147,7 @@ impl TokenNode {
pub fn is_external(&self) -> bool {
match self {
TokenNode::Token(Tagged {
item: RawToken::External(..),
item: RawToken::ExternalCommand(..),
..
}) => true,
_ => false,
@ -150,7 +157,7 @@ impl TokenNode {
pub fn expect_external(&self) -> Span {
match self {
TokenNode::Token(Tagged {
item: RawToken::External(span),
item: RawToken::ExternalCommand(span),
..
}) => *span,
_ => panic!("Only call expect_external if you checked is_external first"),

50
src/parser/parse/token_tree_builder.rs
View File

@ -14,15 +14,19 @@ use derive_new::new;
pub struct TokenTreeBuilder {
#[new(default)]
pos: usize,
#[new(default)]
output: String,
}
pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> TokenNode + 'static>;
pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Tagged<CallNode> + 'static>;
impl TokenTreeBuilder {
pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> TokenNode {
pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) {
let mut builder = TokenTreeBuilder::new();
block(&mut builder)
let node = block(&mut builder);
(node, builder.output)
}
pub fn pipeline(input: Vec<(Option<&str>, CurriedCall, Option<&str>)>) -> CurriedToken {
@ -56,7 +60,8 @@ impl TokenTreeBuilder {
pipe,
pre_span.map(Span::from),
call,
post_span.map(Span::from)));
post_span.map(Span::from),
));
loop {
match input.next() {
@ -147,9 +152,45 @@ impl TokenTreeBuilder {
))
}
pub fn pattern(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, end) = b.consume(&input);
b.pos = end;
TokenTreeBuilder::spanned_pattern((start, end))
})
}
pub fn spanned_pattern(input: impl Into<Span>) -> TokenNode {
TokenNode::Token(Tagged::from_simple_spanned_item(
RawToken::Bare,
input.into(),
))
}
pub fn external_word(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, end) = b.consume(&input);
b.pos = end;
TokenTreeBuilder::spanned_external_word((start, end))
})
}
pub fn spanned_external_word(input: impl Into<Span>) -> TokenNode {
TokenNode::Token(Tagged::from_simple_spanned_item(
RawToken::ExternalWord,
input.into(),
))
}
pub fn spanned_external(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Token(Tagged::from_simple_spanned_item(
RawToken::External(input.into()),
RawToken::ExternalCommand(input.into()),
span.into(),
))
}
@ -422,6 +463,7 @@ impl TokenTreeBuilder {
fn consume(&mut self, input: &str) -> (usize, usize) {
let start = self.pos;
self.pos += input.len();
self.output.push_str(input);
(start, self.pos)
}
}

8
src/parser/parse/tokens.rs
View File

@ -10,7 +10,9 @@ pub enum RawToken {
Size(RawNumber, Unit),
String(Span),
Variable(Span),
External(Span),
ExternalCommand(Span),
ExternalWord,
GlobPattern,
Bare,
}
@ -50,7 +52,9 @@ impl RawToken {
RawToken::Size(..) => "Size",
RawToken::String(_) => "String",
RawToken::Variable(_) => "Variable",
RawToken::External(_) => "External",
RawToken::ExternalCommand(_) => "ExternalCommand",
RawToken::ExternalWord => "ExternalWord",
RawToken::GlobPattern => "GlobPattern",
RawToken::Bare => "String",
}
}

3
src/parser/parse_command.rs
View File

@ -6,6 +6,7 @@ use crate::parser::{
hir::{self, NamedArguments},
Flag, RawToken, TokenNode,
};
use crate::traits::ToDebug;
use crate::{Span, Tag, Tagged, Text};
use log::trace;
@ -248,7 +249,7 @@ pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source
itertools::join(
tail.debug_remaining()
.iter()
.map(|i| format!("%{:?}%", i.debug(source))),
.map(|i| format!("%{}%", i.debug(&source))),
" "
)
);

10
src/shell/helper.rs
View File

@ -123,6 +123,10 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String {
item: RawToken::Size(..),
..
}) => Color::Purple.bold().paint(token_node.span().slice(line)),
TokenNode::Token(Tagged {
item: RawToken::GlobPattern,
..
}) => Color::Cyan.normal().paint(token_node.span().slice(line)),
TokenNode::Token(Tagged {
item: RawToken::String(..),
..
@ -136,9 +140,13 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String {
..
}) => Color::Green.normal().paint(token_node.span().slice(line)),
TokenNode::Token(Tagged {
item: RawToken::External(..),
item: RawToken::ExternalCommand(..),
..
}) => Color::Cyan.bold().paint(token_node.span().slice(line)),
TokenNode::Token(Tagged {
item: RawToken::ExternalWord,
..
}) => Color::Black.bold().paint(token_node.span().slice(line)),
};
styled.to_string()