extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2024-10-05 09:52:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #87 from wycats/better-parser

Browse Source 
Span all the things
This commit is contained in:
Jonathan Turner 2019-06-06 20:00:37 +12:00 committed by GitHub
commit f31c08e941
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 4006 additions and 1760 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
src/cli.rs
View File

@ -10,8 +10,8 @@ use crate::evaluate::Scope;
crate use crate::format::{EntriesListView, GenericView};
use crate::git::current_branch;
use crate::object::Value;
use crate::parser::ast::{Expression, Leaf};
use crate::parser::{Args, ParsedCommand, Pipeline};
use crate::parser::ast::{Expression, Leaf, RawExpression};
use crate::parser::{Args, Pipeline};
use crate::stream::empty_stream;
use log::debug;
@ -324,43 +324,51 @@ fn classify_command(
// let command_name = &command.name[..];
// let args = &command.args;
if let Expression::Call(call) = command {
if let Expression {
expr: RawExpression::Call(call),
..
} = command
{
match (&call.name, &call.args) {
(Expression::Leaf(Leaf::Bare(name)), args) => {
match context.has_command(&name.to_string()) {
true => {
let command = context.get_command(&name.to_string());
let config = command.config();
let scope = Scope::empty();
(
Expression {
expr: RawExpression::Leaf(Leaf::Bare(name)),
..
},
args,
) => match context.has_command(&name.to_string()) {
true => {
let command = context.get_command(&name.to_string());
let config = command.config();
let scope = Scope::empty();
let args = match args {
Some(args) => config.evaluate_args(args.iter(), &scope)?,
None => Args::default(),
};
let args = match args {
Some(args) => config.evaluate_args(args.iter(), &scope)?,
None => Args::default(),
};
Ok(ClassifiedCommand::Internal(InternalCommand {
command,
args,
}))
}
false => {
let arg_list_strings: Vec<String> = match args {
Some(args) => args.iter().map(|i| i.as_external_arg()).collect(),
None => vec![],
};
Ok(ClassifiedCommand::External(ExternalCommand {
name: name.to_string(),
args: arg_list_strings,
}))
}
Ok(ClassifiedCommand::Internal(InternalCommand {
command,
args,
}))
}
}
false => {
let arg_list_strings: Vec<String> = match args {
Some(args) => args.iter().map(|i| i.as_external_arg()).collect(),
None => vec![],
};
Ok(ClassifiedCommand::External(ExternalCommand {
name: name.to_string(),
args: arg_list_strings,
}))
}
},
(_, None) => Err(ShellError::string(
"Unimplemented command that is just an expression (1)",
)),
(_, Some(args)) => Err(ShellError::string("Unimplemented dynamic command")),
(_, Some(_)) => Err(ShellError::string("Unimplemented dynamic command")),
}
} else {
Err(ShellError::string(&format!(

1
src/commands/classified.rs
View File

@ -76,6 +76,7 @@ crate struct ClassifiedPipeline {
}
crate enum ClassifiedCommand {
#[allow(unused)]
Expr(Expression),
Internal(InternalCommand),
External(ExternalCommand),

1
src/errors.rs
View File

@ -63,6 +63,7 @@ pub struct ShellDiagnostic {
}
impl ShellDiagnostic {
#[allow(unused)]
crate fn simple_diagnostic(
span: impl Into<Span>,
source: impl Into<String>,

24
src/evaluate/evaluator.rs
View File

@ -22,15 +22,15 @@ impl Scope {
crate fn evaluate_expr(expr: &ast::Expression, scope: &Scope) -> Result<Value, ShellError> {
use ast::*;
match expr {
Expression::Call(c) => Err(ShellError::unimplemented("Evaluating call expression")),
Expression::Leaf(l) => Ok(evaluate_leaf(l)),
Expression::Parenthesized(p) => evaluate_expr(&p.expr, scope),
Expression::Flag(f) => Ok(Value::Primitive(Primitive::String(f.print()))),
Expression::Block(b) => evaluate_block(&b, scope),
Expression::Path(p) => evaluate_path(&p, scope),
Expression::Binary(b) => evaluate_binary(b, scope),
Expression::VariableReference(r) => evaluate_reference(r, scope),
match &expr.expr {
RawExpression::Call(_) => Err(ShellError::unimplemented("Evaluating call expression")),
RawExpression::Leaf(l) => Ok(evaluate_leaf(l)),
RawExpression::Parenthesized(p) => evaluate_expr(&p.expr, scope),
RawExpression::Flag(f) => Ok(Value::Primitive(Primitive::String(f.print()))),
RawExpression::Block(b) => evaluate_block(&b, scope),
RawExpression::Path(p) => evaluate_path(&p, scope),
RawExpression::Binary(b) => evaluate_binary(b, scope),
RawExpression::VariableReference(r) => evaluate_reference(r, scope),
}
}
@ -63,7 +63,7 @@ fn evaluate_binary(binary: &ast::Binary, scope: &Scope) -> Result<Value, ShellEr
let left = evaluate_expr(&binary.left, scope)?;
let right = evaluate_expr(&binary.right, scope)?;
match left.compare(binary.operator, &right) {
match left.compare(&binary.operator, &right) {
Some(v) => Ok(Value::boolean(v)),
None => Err(ShellError::TypeError(format!(
"Can't compare {} and {}",
@ -83,8 +83,8 @@ fn evaluate_path(path: &ast::Path, scope: &Scope) -> Result<Value, ShellError> {
let mut seen = vec![];
for name in path.tail() {
let next = value.get_data_by_key(&name);
seen.push(name);
let next = value.get_data_by_key(&name.item);
seen.push(name.item.clone());
match next {
None => {

9
src/object/base.rs
View File

@ -134,9 +134,10 @@ impl Deserialize<'de> for Block {
where
D: Deserializer<'de>,
{
Ok(Block::new(ast::Expression::Leaf(ast::Leaf::String(
format!("Unserializable block"),
))))
let mut builder = ast::ExpressionBuilder::new();
let expr: ast::Expression = builder.string("Unserializable block");
Ok(Block::new(expr))
}
}
@ -219,7 +220,7 @@ impl Value {
}
}
crate fn compare(&self, operator: ast::Operator, other: &Value) -> Option<bool> {
crate fn compare(&self, operator: &ast::Operator, other: &Value) -> Option<bool> {
match operator {
_ => {
let coerced = coerce_compare(self, other)?;

73
src/parser.rs
View File

@ -5,7 +5,7 @@ crate mod parser;
crate mod registry;
crate mod span;
crate use ast::{ParsedCommand, Pipeline};
crate use ast::Pipeline;
crate use registry::{Args, CommandConfig};
use crate::errors::ShellError;
@ -33,7 +33,7 @@ pub fn parse(input: &str) -> Result<Pipeline, ShellError> {
#[cfg(test)]
mod tests {
use super::*;
use crate::parser::ast::{bare, flag, short, unit, var, Expression, Operator, Pipeline};
use crate::parser::ast::Pipeline;
use pretty_assertions::assert_eq;
fn assert_parse(source: &str, expected: Pipeline) {
@ -61,30 +61,44 @@ mod tests {
let printed = parsed.print();
assert_eq!(parsed, expected);
assert_eq!(source, printed);
assert_eq!(printed, source);
}
macro_rules! commands {
( $( ( $name:tt $( $command:expr)* ) )|* ) => {
Pipeline::new(vec![
( $( ( $name:tt $( $command:ident ( $arg:expr ) )* ) )|* ) => {{
use $crate::parser::ast::{Expression, ExpressionBuilder};
let mut builder = crate::parser::ast::ExpressionBuilder::new();
builder.pipeline(vec![
$(
command!($name $($command)*)
(command!($name $($command($arg))*) as (&dyn Fn(&mut ExpressionBuilder) -> Expression))
),*
])
}
}}
}
macro_rules! command {
($name:ident $( $command:expr )*) => {
Expression::call(Expression::bare(stringify!($name)), vec![ $($command.into()),* ])
($name:ident) => {
&|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((
&|b: &mut $crate::parser::ast::ExpressionBuilder| b.bare(stringify!($name)),
vec![]
))
};
($name:ident $( $command:expr )*) => {
Expression::call(Expression::bare(stringify!($name)), vec![ $($command.into()),* ])
($name:ident $( $command:ident ( $body:expr ) )*) => {{
use $crate::parser::ast::{Expression, ExpressionBuilder};
&|b: &mut ExpressionBuilder| b.call((
(&|b: &mut ExpressionBuilder| b.bare(stringify!($name))) as (&dyn Fn(&mut ExpressionBuilder) -> Expression),
vec![$( (&|b: &mut ExpressionBuilder| b.$command($body)) as &dyn Fn(&mut ExpressionBuilder) -> Expression ),* ]))
}};
($name:ident $( $command:ident ( $body:expr ) )*) => {
&|b: &mut $crate::parser::ast::ExpressionBuilder| b.call(|b| b.bare(stringify!($name)), vec![ $( |b| b.$command($body) ),* ])
};
($name:tt $( $command:expr )*) => {
Expression::call(Expression::bare($name), vec![ $($command.into()),* ])
($name:tt $( $command:ident ( $body:expr ) )*) => {
&|b: &mut $crate::parser::ast::ExpressionBuilder| b.call((&|b| b.bare($name), vec![ $( &|b| b.$command($body) ),* ]))
};
}
@ -100,7 +114,7 @@ mod tests {
commands![
(open bare("Cargo.toml"))
| (select bare("package.authors"))
| ("split-row" " ")
| ("split-row" string(" "))
],
);
@ -122,7 +136,7 @@ mod tests {
assert_parse(
"open Cargo.toml -r",
commands![(open bare("Cargo.toml") short("r"))],
commands![(open bare("Cargo.toml") shorthand("r"))],
);
assert_parse(
@ -132,7 +146,7 @@ mod tests {
assert_parse(
r#"config --get "ignore dups" | format-list"#,
commands![(config flag("get") "ignore dups") | ("format-list")],
commands![(config flag("get") string("ignore dups")) | ("format-list")],
);
assert_parse(
@ -147,16 +161,16 @@ mod tests {
assert_parse(
"config --set tabs 2",
commands![(config flag("set") bare("tabs") 2)],
commands![(config flag("set") bare("tabs") int(2))],
);
assert_parse(
r#"ls | skip 1 | first 2 | select "file name" | rm $it"#,
commands![
(ls)
| (skip 1)
| (first 2)
| (select "file name")
| (skip int(1))
| (first int(2))
| (select string("file name"))
| (rm var("it"))
],
);
@ -165,7 +179,7 @@ mod tests {
r#"git branch --merged | split-row "`n" | where $it != "* master""#,
commands![
// TODO: Handle escapes correctly. Should we do ` escape because of paths?
(git bare("branch") flag("merged")) | ("split-row" "`n") | (where binary(var("it"), "!=", "* master"))
(git bare("branch") flag("merged")) | ("split-row" string("`n")) | (where binary((&|b| b.var("it"), &|b| b.op("!="), &|b| b.string("* master"))))
],
);
@ -175,7 +189,7 @@ mod tests {
(open bare("input2.json"))
| ("from-json")
| (select bare("glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso"))
| (where binary(var("it"), ">", "GML"))
| (where binary((&|b| b.var("it"), &|b| b.op(">"), &|b| b.string("GML"))))
]
);
@ -189,16 +203,15 @@ mod tests {
assert_parse(
"ls | where size < 1KB",
commands![
(ls) | (where binary(bare("size"), "<", unit(1, "KB")))
(ls) | (where binary((&|b| b.bare("size"), &|b| b.op("<"), &|b| b.unit((1, "KB")))))
],
);
}
fn binary(
left: impl Into<Expression>,
op: impl Into<Operator>,
right: impl Into<Expression>,
) -> Expression {
Expression::binary(left, op, right)
assert_parse(
"ls | where { $it.size > 100 }",
commands![
(ls) | (where block(&|b| b.binary((&|b| b.path((&|b| b.var("it"), vec!["size"])), &|b| b.op(">"), &|b| b.int(100)))))
],
)
}
}

735
src/parser/ast.rs
View File

@ -1,4 +1,4 @@
use crate::parser::lexer::SpannedToken;
use crate::parser::lexer::{Span, Spanned};
use crate::prelude::*;
use adhoc_derive::FromStr;
use derive_new::new;
@ -7,6 +7,381 @@ use serde_derive::{Deserialize, Serialize};
use std::io::Write;
use std::str::FromStr;
#[derive(new)]
pub struct ExpressionBuilder {
#[new(default)]
pos: usize,
}
#[allow(unused)]
impl ExpressionBuilder {
pub fn op(&mut self, input: impl Into<Operator>) -> Spanned<Operator> {
let input = input.into();
let (start, end) = self.consume(input.as_str());
self.pos = end;
ExpressionBuilder::spanned_op(input, start, end)
}
pub fn spanned_op(input: impl Into<Operator>, start: usize, end: usize) -> Spanned<Operator> {
Spanned {
span: Span::from((start, end)),
item: input.into(),
}
}
pub fn string(&mut self, input: impl Into<String>) -> Expression {
let input = input.into();
let (start, _) = self.consume("\"");
self.consume(&input);
let (_, end) = self.consume("\"");
self.pos = end;
ExpressionBuilder::spanned_string(input, start, end)
}
pub fn spanned_string(input: impl Into<String>, start: usize, end: usize) -> Expression {
let input = input.into();
Expression {
span: Span::from((start, end)),
expr: RawExpression::Leaf(Leaf::String(input)),
}
}
pub fn bare(&mut self, input: impl Into<Bare>) -> Expression {
let input = input.into();
let (start, end) = self.consume(&input.body);
self.pos = end;
ExpressionBuilder::spanned_bare(input, start, end)
}
pub fn spanned_bare(input: impl Into<Bare>, start: usize, end: usize) -> Expression {
let input = input.into();
Expression {
span: Span::from((start, end)),
expr: RawExpression::Leaf(Leaf::Bare(input)),
}
}
pub fn boolean(&mut self, input: impl Into<bool>) -> Expression {
let boolean = input.into();
let (start, end) = match boolean {
true => self.consume("$yes"),
false => self.consume("$no"),
};
self.pos = end;
ExpressionBuilder::spanned_boolean(boolean, start, end)
}
pub fn spanned_boolean(input: impl Into<bool>, start: usize, end: usize) -> Expression {
let input = input.into();
Expression {
span: Span::from((start, end)),
expr: RawExpression::Leaf(Leaf::Boolean(input)),
}
}
pub fn int(&mut self, input: impl Into<i64>) -> Expression {
let int = input.into();
let (start, end) = self.consume(&int.to_string());
self.pos = end;
ExpressionBuilder::spanned_int(int, start, end)
}
pub fn spanned_int(input: impl Into<i64>, start: usize, end: usize) -> Expression {
let input = input.into();
Expression {
span: Span::from((start, end)),
expr: RawExpression::Leaf(Leaf::Int(input)),
}
}
pub fn unit(&mut self, input: (impl Into<i64>, impl Into<Unit>)) -> Expression {
let (int, unit) = (input.0.into(), input.1.into());
let (start, _) = self.consume(&int.to_string());
let (_, end) = self.consume(&unit.to_string());
self.pos = end;
ExpressionBuilder::spanned_unit((int, unit), start, end)
}
pub fn spanned_unit(
input: (impl Into<i64>, impl Into<Unit>),
start: usize,
end: usize,
) -> Expression {
let (int, unit) = (input.0.into(), input.1.into());
Expression {
span: Span::from((start, end)),
expr: RawExpression::Leaf(Leaf::Unit(int, unit)),
}
}
pub fn flag(&mut self, input: impl Into<String>) -> Expression {
let input = input.into();
let (start, _) = self.consume("--");
let (_, end) = self.consume(&input);
self.pos = end;
ExpressionBuilder::spanned_flag(input, start, end)
}
pub fn spanned_flag(input: impl Into<String>, start: usize, end: usize) -> Expression {
let input = input.into();
Expression {
span: Span::from((start, end)),
expr: RawExpression::Flag(Flag::Longhand(input)),
}
}
pub fn shorthand(&mut self, input: impl Into<String>) -> Expression {
let int = input.into();
let size = int.to_string().len();
let start = self.pos;
let end = self.pos + size + 1;
self.pos = end;
ExpressionBuilder::spanned_shorthand(int, start, end)
}
pub fn spanned_shorthand(input: impl Into<String>, start: usize, end: usize) -> Expression {
let input = input.into();
Expression {
span: Span::from((start, end)),
expr: RawExpression::Flag(Flag::Shorthand(input)),
}
}
pub fn parens(
&mut self,
input: impl FnOnce(&mut ExpressionBuilder) -> Expression,
) -> Expression {
let (start, _) = self.consume("(");
let input = input(self);
let (_, end) = self.consume(")");
self.pos = end;
ExpressionBuilder::spanned_parens(input, start, end)
}
pub fn spanned_parens(input: Expression, start: usize, end: usize) -> Expression {
Expression {
span: Span::from((start, end)),
expr: RawExpression::Parenthesized(Box::new(Parenthesized::new(input))),
}
}
pub fn block(&mut self, input: &dyn Fn(&mut ExpressionBuilder) -> Expression) -> Expression {
let (start, _) = self.consume("{ ");
let input = input(self);
let (_, end) = self.consume(" }");
self.pos = end;
ExpressionBuilder::spanned_block(input, start, end)
}
pub fn spanned_block(input: Expression, start: usize, end: usize) -> Expression {
Expression {
span: Span::from((start, end)),
expr: RawExpression::Block(Box::new(Block::new(input))),
}
}
pub fn binary(
&mut self,
input: (
&dyn Fn(&mut ExpressionBuilder) -> Expression,
&dyn Fn(&mut ExpressionBuilder) -> Spanned<Operator>,
&dyn Fn(&mut ExpressionBuilder) -> Expression,
),
) -> Expression {
let start = self.pos;
let left = (input.0)(self);
self.consume(" ");
let operator = (input.1)(self);
self.consume(" ");
let right = (input.2)(self);
let end = self.pos;
ExpressionBuilder::spanned_binary((left, operator, right), start, end)
}
pub fn spanned_binary(
input: (
impl Into<Expression>,
impl Into<Spanned<Operator>>,
impl Into<Expression>,
),
start: usize,
end: usize,
) -> Expression {
let binary = Binary::new(input.0, input.1.into(), input.2);
Expression {
span: Span::from((start, end)),
expr: RawExpression::Binary(Box::new(binary)),
}
}
pub fn path(
&mut self,
input: (
&dyn Fn(&mut ExpressionBuilder) -> Expression,
Vec<impl Into<String>>,
),
) -> Expression {
let start = self.pos;
let head = (input.0)(self);
let mut tail = vec![];
for item in input.1 {
self.consume(".");
let item = item.into();
let (start, end) = self.consume(&item);
tail.push(Spanned::new(Span::from((start, end)), item));
}
let end = self.pos;
ExpressionBuilder::spanned_path((head, tail), start, end)
}
pub fn spanned_path(
input: (impl Into<Expression>, Vec<Spanned<String>>),
start: usize,
end: usize,
) -> Expression {
let path = Path::new(input.0.into(), input.1);
Expression {
span: Span::from((start, end)),
expr: RawExpression::Path(Box::new(path)),
}
}
pub fn call(
&mut self,
input: (
&(dyn Fn(&mut ExpressionBuilder) -> Expression),
Vec<&dyn Fn(&mut ExpressionBuilder) -> Expression>,
),
) -> Expression {
let start = self.pos;
let name = (&input.0)(self);
let mut args = vec![];
for item in input.1 {
self.consume(" ");
args.push(item(self));
}
let end = self.pos;
ExpressionBuilder::spanned_call((name, args), start, end)
}
pub fn spanned_call(input: impl Into<Call>, start: usize, end: usize) -> Expression {
let call = input.into();
Expression {
span: Span::from((start, end)),
expr: RawExpression::Call(Box::new(call)),
}
}
pub fn var(&mut self, input: impl Into<String>) -> Expression {
let input = input.into();
let (start, _) = self.consume("$");
let (_, end) = self.consume(&input);
ExpressionBuilder::spanned_var(input, start, end)
}
pub fn spanned_var(input: impl Into<String>, start: usize, end: usize) -> Expression {
let input = input.into();
let expr = match &input[..] {
"it" => RawExpression::VariableReference(Variable::It),
_ => RawExpression::VariableReference(Variable::Other(input)),
};
Expression {
span: Span::from((start, end)),
expr,
}
}
pub fn pipeline(
&mut self,
input: Vec<&dyn Fn(&mut ExpressionBuilder) -> Expression>,
) -> Pipeline {
let start = self.pos;
let mut exprs = vec![];
let mut input = input.into_iter();
let next = input.next().unwrap();
exprs.push(next(self));
for item in input {
self.consume(" | ");
exprs.push(item(self));
}
let end = self.pos;
ExpressionBuilder::spanned_pipeline(exprs, start, end)
}
pub fn spanned_pipeline(input: Vec<Expression>, start: usize, end: usize) -> Pipeline {
Pipeline {
span: Span::from((start, end)),
commands: input,
}
}
pub fn sp(&mut self) {
self.consume(" ");
}
pub fn ws(&mut self, input: &str) {
self.consume(input);
}
fn consume(&mut self, input: &str) -> (usize, usize) {
let start = self.pos;
self.pos += input.len();
(start, self.pos)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
pub enum Operator {
Equal,
@ -19,13 +394,17 @@ pub enum Operator {
impl Operator {
pub fn print(&self) -> String {
self.as_str().to_string()
}
pub fn as_str(&self) -> &str {
match *self {
Operator::Equal => "==".to_string(),
Operator::NotEqual => "!=".to_string(),
Operator::LessThan => "<".to_string(),
Operator::GreaterThan => ">".to_string(),
Operator::LessThanOrEqual => "<=".to_string(),
Operator::GreaterThanOrEqual => ">=".to_string(),
Operator::Equal => "==",
Operator::NotEqual => "!=",
Operator::LessThan => "<",
Operator::GreaterThan => ">",
Operator::LessThanOrEqual => "<=",
Operator::GreaterThanOrEqual => ">=",
}
}
}
@ -52,140 +431,127 @@ impl FromStr for Operator {
}
#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)]
pub enum Expression {
pub struct Expression {
crate expr: RawExpression,
crate span: Span,
}
impl std::ops::Deref for Expression {
type Target = RawExpression;
fn deref(&self) -> &RawExpression {
&self.expr
}
}
impl Expression {
crate fn print(&self) -> String {
self.expr.print()
}
crate fn as_external_arg(&self) -> String {
self.expr.as_external_arg()
}
}
#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)]
pub enum RawExpression {
Leaf(Leaf),
Flag(Flag),
Parenthesized(Box<Parenthesized>),
Block(Box<Block>),
Binary(Box<Binary>),
Path(Box<Path>),
Call(Box<ParsedCommand>),
Call(Box<Call>),
VariableReference(Variable),
}
impl From<&str> for Expression {
fn from(input: &str) -> Expression {
Expression::Leaf(Leaf::String(input.into()))
}
}
impl RawExpression {
// crate fn leaf(leaf: impl Into<Leaf>) -> Expression {
// Expression::Leaf(leaf.into())
// }
impl From<String> for Expression {
fn from(input: String) -> Expression {
Expression::Leaf(Leaf::String(input.into()))
}
}
// crate fn flag(flag: impl Into<Flag>) -> Expression {
// Expression::Flag(flag.into())
// }
impl From<i64> for Expression {
fn from(input: i64) -> Expression {
Expression::Leaf(Leaf::Int(input.into()))
}
}
// crate fn call(head: Expression, tail: Vec<Expression>) -> Expression {
// if tail.len() == 0 {
// Expression::Call(Box::new(ParsedCommand::new(head.into(), None)))
// } else {
// Expression::Call(Box::new(ParsedCommand::new(head.into(), Some(tail))))
// }
// }
impl From<BarePath> for Expression {
fn from(input: BarePath) -> Expression {
Expression::Leaf(Leaf::Bare(input))
}
}
// crate fn binary(
// left: impl Into<Expression>,
// operator: impl Into<Operator>,
// right: impl Into<Expression>,
// ) -> Expression {
// Expression::Binary(Box::new(Binary {
// left: left.into(),
// operator: operator.into(),
// right: right.into(),
// }))
// }
impl From<Variable> for Expression {
fn from(input: Variable) -> Expression {
Expression::VariableReference(input)
}
}
impl From<Flag> for Expression {
fn from(input: Flag) -> Expression {
Expression::Flag(input)
}
}
impl From<Binary> for Expression {
fn from(input: Binary) -> Expression {
Expression::Binary(Box::new(input))
}
}
impl Expression {
crate fn leaf(leaf: impl Into<Leaf>) -> Expression {
Expression::Leaf(leaf.into())
}
crate fn flag(flag: impl Into<Flag>) -> Expression {
Expression::Flag(flag.into())
}
crate fn call(head: Expression, tail: Vec<Expression>) -> Expression {
if tail.len() == 0 {
Expression::Call(Box::new(ParsedCommand::new(head.into(), None)))
} else {
Expression::Call(Box::new(ParsedCommand::new(head.into(), Some(tail))))
}
}
crate fn binary(
left: impl Into<Expression>,
operator: impl Into<Operator>,
right: impl Into<Expression>,
) -> Expression {
Expression::Binary(Box::new(Binary {
left: left.into(),
operator: operator.into(),
right: right.into(),
}))
}
crate fn block(expr: impl Into<Expression>) -> Expression {
Expression::Block(Box::new(Block::new(expr.into())))
}
// crate fn block(expr: impl Into<Expression>) -> Expression {
// Expression::Block(Box::new(Block::new(expr.into())))
// }
crate fn print(&self) -> String {
match self {
Expression::Call(c) => c.print(),
Expression::Leaf(l) => l.print(),
Expression::Flag(f) => f.print(),
Expression::Parenthesized(p) => p.print(),
Expression::Block(b) => b.print(),
Expression::VariableReference(r) => r.print(),
Expression::Path(p) => p.print(),
Expression::Binary(b) => b.print(),
RawExpression::Call(c) => c.print(),
RawExpression::Leaf(l) => l.print(),
RawExpression::Flag(f) => f.print(),
RawExpression::Parenthesized(p) => p.print(),
RawExpression::Block(b) => b.print(),
RawExpression::VariableReference(r) => r.print(),
RawExpression::Path(p) => p.print(),
RawExpression::Binary(b) => b.print(),
}
}
crate fn as_external_arg(&self) -> String {
match self {
Expression::Call(c) => c.as_external_arg(),
Expression::Leaf(l) => l.as_external_arg(),
Expression::Flag(f) => f.as_external_arg(),
Expression::Parenthesized(p) => p.as_external_arg(),
Expression::Block(b) => b.as_external_arg(),
Expression::VariableReference(r) => r.as_external_arg(),
Expression::Path(p) => p.as_external_arg(),
Expression::Binary(b) => b.as_external_arg(),
RawExpression::Call(c) => c.as_external_arg(),
RawExpression::Leaf(l) => l.as_external_arg(),
RawExpression::Flag(f) => f.as_external_arg(),
RawExpression::Parenthesized(p) => p.as_external_arg(),
RawExpression::Block(b) => b.as_external_arg(),
RawExpression::VariableReference(r) => r.as_external_arg(),
RawExpression::Path(p) => p.as_external_arg(),
RawExpression::Binary(b) => b.as_external_arg(),
}
}
crate fn bare(path: impl Into<BarePath>) -> Expression {
Expression::Leaf(Leaf::Bare(path.into()))
}
crate fn as_string(&self) -> Option<String> {
match self {
Expression::Leaf(Leaf::String(s)) => Some(s.to_string()),
Expression::Leaf(Leaf::Bare(path)) => Some(path.to_string()),
RawExpression::Leaf(Leaf::String(s)) => Some(s.to_string()),
RawExpression::Leaf(Leaf::Bare(path)) => Some(path.to_string()),
_ => None,
}
}
#[allow(unused)]
crate fn as_bare(&self) -> Option<String> {
match self {
Expression::Leaf(Leaf::Bare(p)) => Some(p.to_string()),
RawExpression::Leaf(Leaf::Bare(p)) => Some(p.to_string()),
_ => None,
}
}
#[allow(unused)]
crate fn as_block(&self) -> Option<Block> {
match self {
RawExpression::Block(block) => Some(*block.clone()),
_ => None,
}
}
crate fn is_flag(&self, value: &str) -> bool {
match self {
Expression::Flag(Flag::Longhand(f)) if value == f => true,
RawExpression::Flag(Flag::Longhand(f)) if value == f => true,
_ => false,
}
}
@ -227,7 +593,7 @@ pub struct Path {
head: Expression,
#[get = "crate"]
tail: Vec<String>,
tail: Vec<Spanned<String>>,
}
impl Path {
@ -235,7 +601,7 @@ impl Path {
let mut out = self.head.print();
for item in self.tail.iter() {
out.push_str(&format!(".{}", item));
out.push_str(&format!(".{}", item.item));
}
out
@ -245,7 +611,7 @@ impl Path {
let mut out = self.head.as_external_arg();
for item in self.tail.iter() {
out.push_str(&format!(".{}", item));
out.push_str(&format!(".{}", item.item));
}
out
@ -258,24 +624,7 @@ pub enum Variable {
Other(String),
}
#[cfg(test)]
crate fn var(name: &str) -> Expression {
match name {
"it" => Expression::VariableReference(Variable::It),
other => Expression::VariableReference(Variable::Other(other.to_string())),
}
}
impl Variable {
crate fn from_str(input: &str) -> Expression {
match input {
"it" => Expression::VariableReference(Variable::It),
"yes" => Expression::Leaf(Leaf::Boolean(true)),
"no" => Expression::Leaf(Leaf::Boolean(false)),
other => Expression::VariableReference(Variable::Other(other.to_string())),
}
}
fn print(&self) -> String {
match self {
Variable::It => format!("$it"),
@ -288,45 +637,34 @@ impl Variable {
}
}
pub fn bare(s: impl Into<String>) -> BarePath {
BarePath {
head: s.into(),
tail: vec![],
#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, new)]
pub struct Bare {
body: String,
}
impl From<String> for Bare {
fn from(input: String) -> Bare {
Bare { body: input }
}
}
#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)]
pub struct BarePath {
head: String,
tail: Vec<String>,
}
impl<T: Into<String>> From<T> for BarePath {
fn from(input: T) -> BarePath {
BarePath {
head: input.into(),
tail: vec![],
impl From<&str> for Bare {
fn from(input: &str) -> Bare {
Bare {
body: input.to_string(),
}
}
}
impl BarePath {
crate fn from_token(head: SpannedToken) -> BarePath {
BarePath {
head: head.to_string(),
tail: vec![],
}
}
crate fn from_tokens(head: SpannedToken, tail: Vec<SpannedToken>) -> BarePath {
BarePath {
head: head.to_string(),
tail: tail.iter().map(|i| i.to_string()).collect(),
impl Bare {
crate fn from_string(string: impl Into<String>) -> Bare {
Bare {
body: string.into(),
}
}
crate fn to_string(&self) -> String {
bare_string(&self.head, &self.tail)
self.body.to_string()
}
}
@ -363,30 +701,28 @@ impl Unit {
Unit::PB => size * 1024 * 1024 * 1024 * 1024 * 1024,
})
}
}
#[cfg(test)]
pub fn unit(num: i64, unit: impl Into<Unit>) -> Expression {
Expression::Leaf(Leaf::Unit(num, unit.into()))
crate fn to_string(&self) -> &str {
match self {
Unit::B => "B",
Unit::KB => "KB",
Unit::MB => "MB",
Unit::GB => "GB",
Unit::TB => "TB",
Unit::PB => "PB",
}
}
}
#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)]
pub enum Leaf {
String(String),
Bare(BarePath),
#[allow(unused)]
Bare(Bare),
Boolean(bool),
Int(i64),
Unit(i64, Unit),
}
crate fn bare_string(head: &String, tail: &Vec<String>) -> String {
let mut out = vec![head.clone()];
out.extend(tail.clone());
itertools::join(out, ".")
}
impl Leaf {
fn print(&self) -> String {
match self {
@ -412,14 +748,14 @@ impl Leaf {
#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)]
pub struct Binary {
crate left: Expression,
crate operator: Operator,
crate operator: Spanned<Operator>,
crate right: Expression,
}
impl Binary {
crate fn new(
left: impl Into<Expression>,
operator: Operator,
operator: Spanned<Operator>,
right: impl Into<Expression>,
) -> Binary {
Binary {
@ -456,16 +792,6 @@ pub enum Flag {
Longhand(String),
}
#[cfg(test)]
crate fn flag(s: &str) -> Flag {
Flag::Longhand(s.into())
}
#[cfg(test)]
crate fn short(s: &str) -> Flag {
Flag::Shorthand(s.into())
}
impl Flag {
#[allow(unused)]
crate fn print(&self) -> String {
@ -482,12 +808,34 @@ impl Flag {
}
#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, new)]
pub struct ParsedCommand {
pub struct Call {
crate name: Expression,
crate args: Option<Vec<Expression>>,
}
impl ParsedCommand {
impl From<(Expression, Vec<Expression>)> for Call {
fn from(input: (Expression, Vec<Expression>)) -> Call {
Call {
name: input.0,
args: if input.1.len() == 0 {
None
} else {
Some(input.1)
},
}
}
}
impl From<Expression> for Call {
fn from(input: Expression) -> Call {
Call {
name: input,
args: None,
}
}
}
impl Call {
fn as_external_arg(&self) -> String {
let mut out = vec![];
@ -517,35 +865,26 @@ impl ParsedCommand {
}
}
impl From<&str> for ParsedCommand {
fn from(input: &str) -> ParsedCommand {
ParsedCommand {
name: Expression::Leaf(Leaf::Bare(bare(input))),
args: None,
}
}
}
impl From<(&str, Vec<Expression>)> for ParsedCommand {
fn from(input: (&str, Vec<Expression>)) -> ParsedCommand {
ParsedCommand {
name: Expression::bare(input.0),
args: Some(input.1),
}
}
}
#[derive(new, Debug, Eq, PartialEq)]
pub struct Pipeline {
crate commands: Vec<Expression>,
crate span: Span,
}
impl Pipeline {
crate fn from_parts(command: Expression, rest: Vec<Expression>) -> Pipeline {
crate fn from_parts(
command: Expression,
rest: Vec<Expression>,
start: usize,
end: usize,
) -> Pipeline {
let mut commands = vec![command];
commands.extend(rest);
Pipeline { commands }
Pipeline {
commands,
span: Span::from((start, end)),
}
}
#[allow(unused)]

28
src/parser/lexer.rs
View File

@ -317,8 +317,8 @@ impl logos::Extras for LexerState {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]
pub struct Span {
start: usize,
end: usize,
crate start: usize,
crate end: usize,
// source: &'source str,
}
@ -374,6 +374,26 @@ impl language_reporting::ReportingSpan for Span {
}
}
#[derive(new, Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub struct Spanned<T> {
crate span: Span,
crate item: T,
}
impl<T> std::ops::Deref for Spanned<T> {
type Target = T;
fn deref(&self) -> &T {
&self.item
}
}
impl<T> Spanned<T> {
crate fn from_item(item: T, span: Span) -> Spanned<T> {
Spanned { span, item }
}
}
#[derive(new, Debug, Clone, Eq, PartialEq)]
pub struct SpannedToken<'source> {
crate span: Span,
@ -382,6 +402,10 @@ pub struct SpannedToken<'source> {
}
impl SpannedToken<'source> {
crate fn to_spanned_string(&self) -> Spanned<String> {
Spanned::from_item(self.slice.to_string(), self.span)
}
crate fn to_string(&self) -> String {
self.slice.to_string()
}

78
src/parser/parser.lalrpop
View File

@ -3,7 +3,7 @@
use std::str::FromStr;
use crate::parser::ast::*;
use crate::prelude::*;
use crate::parser::lexer::{SpannedToken, Token};
use crate::parser::lexer::{SpannedToken, Spanned, Span, Token};
use byte_unit::Byte;
// nu's grammar is a little bit different from a lot of other languages, to better match
@ -27,61 +27,69 @@ use byte_unit::Byte;
grammar<'input>;
pub Pipeline: Pipeline = {
<first:PipelineElement> <rest: ( "|" <PipelineElement> )*> => Pipeline::from_parts(first, rest),
<l: @L> <first:PipelineElement> <rest: ( "|" <PipelineElement> )*> <r: @R> => Pipeline::from_parts(first, rest, l, r),
}
PipelineElement: Expression = {
<Bare> => Expression::call(Expression::bare(<>), vec![]),
<l: @L> <bare: BareExpression> <r: @R> => ExpressionBuilder::spanned_call((bare, vec![]), l, r),
<SingleExpression> => <>,
}
// A leaf expression is a single logical token that directly represents an expression
LeafExpression: Expression = {
<String> => <>,
<Int> => Expression::leaf(Leaf::Int(<>)),
<UnitsNum> => <>,
<Var> => <>,
<String>,
<l: @L> <int: Int> <r: @R> => ExpressionBuilder::spanned_int(int, l, r),
<UnitsNum>,
<Var>,
}
pub Call: Expression = {
<expr:Expression> <rest:SingleCallArgument> => Expression::call(expr, vec![rest]),
<expr:Expression> <first:CallArgument> <rest:( <CallArgument> )+> => Expression::call(expr, { let mut rest = rest; let mut v = vec![first]; v.append(&mut rest); v }),
<expr:Bare> <rest:SingleCallArgument> => Expression::call(Expression::bare(expr), vec![rest]),
<expr:Bare> <first:CallArgument> <rest:( <CallArgument> )+> => Expression::call(Expression::bare(expr), { let mut v = vec![first]; let mut rest = rest; v.append(&mut rest); v }),
<l: @L> <expr:Expression> <rest:SingleCallArgument> <r: @R> => ExpressionBuilder::spanned_call((expr, vec![rest]), l, r),
<l: @L> <expr:Expression> <first:CallArgument> <rest:( <CallArgument> )+> <r: @R> => ExpressionBuilder::spanned_call((expr, { let mut rest = rest; let mut v = vec![first]; v.append(&mut rest); v }), l, r),
<l: @L> <expr:BareExpression> <rest:SingleCallArgument> <r: @R> => ExpressionBuilder::spanned_call((expr, vec![rest]), l, r),
<l: @L> <expr:BareExpression> <first:CallArgument> <rest:( <CallArgument> )+> <r: @R> => ExpressionBuilder::spanned_call((expr, { let mut v = vec![first]; let mut rest = rest; v.append(&mut rest); v }), l, r),
}
Binary: Expression = {
<left:ArgumentExpression> <op:Operator> <right:ArgumentExpression> => Expression::binary(left, op, right),
<l: @L> <left:ArgumentExpression> <op:SpannedOperator> <right:ArgumentExpression> <r: @R> => ExpressionBuilder::spanned_binary((left, op, right), l, r)
}
// In a block, a single bare word is interpreted as a call:
//
// foreach { ls }
Block: Expression = {
"{" <SingleExpression> "}" => Expression::block(<>),
"{" <Bare> "}" => Expression::block(Expression::call(Expression::bare(<>), vec![])),
<l: @L> "{" <expr: SingleExpression> "}" <r: @R> => ExpressionBuilder::spanned_block(expr, l, r),
<l: @L> "{" <bare: BareExpression> "}" <r: @R> => {
let call = ExpressionBuilder::spanned_call(bare.clone(), bare.span.start, bare.span.end);
ExpressionBuilder::spanned_block(call, l, r)
}
}
// An `Expression` is the most general kind of expression. It can go anywhere, even right next to another expression, and
// even as the first part of a call.
Expression: Expression = {
MemberHeadExpression: Expression = {
<LeafExpression> => <>,
<Block> => <>,
"(" <Call> ")" => <>,
"(" <Bare> ")" => Expression::call(Expression::bare(<>), vec![]),
"(" <Binary> ")" => <>,
<l: @L> "(" <expr: Call> ")" <r: @R> => ExpressionBuilder::spanned_call(expr, l, r),
<l: @L> "(" <expr: BareExpression> ")" <r: @R> => ExpressionBuilder::spanned_call((expr, vec![]), l, r),
<l: @L> "(" <expr:Binary> ")" <r: @R> => ExpressionBuilder::spanned_parens(expr, l, r),
}
Expression: Expression = {
<MemberHeadExpression> => <>,
<l: @L> <expr:MemberHeadExpression> <rest: ( "???." <"member"> )+> <r: @R> => ExpressionBuilder::spanned_path((expr, rest.iter().map(|i| i.to_spanned_string()).collect()), l, r),
}
// An `ArgumentExpression` is an expression that appears in an argument list. It includes all of `Expression`, and
// bare words are interpreted as strings.
ArgumentExpression: Expression = {
<Expression>,
<Bare> => Expression::bare(<>),
<BareExpression>,
}
CallArgument: Expression = {
<ArgumentExpression> => <>,
<Flag> => Expression::flag(<>),
<ArgumentExpression>,
<Flag>,
}
SingleCallArgument: Expression = {
@ -101,14 +109,22 @@ SingleExpression: Expression = {
<Binary>,
}
BareExpression: Expression = {
<l: @L> <bare: Bare> <r: @R> => ExpressionBuilder::spanned_bare(bare, l, r)
}
SpannedOperator: Spanned<Operator> = {
<l: @L> <op: Operator> <r: @R> => Spanned::from_item(op, Span::from((l, r)))
}
// === LOGICAL TOKENS === //
// A logical token may be composed of more than one raw token, but the tokens must be emitted
// from the stream in exactly one sequence. This allows us to use parser infrastructure to
// compose tokens without the risk that these logical tokens will introduce ambiguities.
Bare: BarePath = {
<head: "bare"> => BarePath::from_token(head)
Bare: Bare = {
<head: "bare"> => Bare::from_string(head.as_slice())
}
// A member is a special token that represents bare words or string literals immediate
@ -129,25 +145,25 @@ Operator: Operator = {
}
Int: i64 = {
<"num"> => i64::from_str(<>.as_slice()).unwrap()
<n: "num"> => i64::from_str(<>.as_slice()).unwrap(),
}
UnitsNum: Expression = {
<num: Int> <unit: "unit"> => Expression::leaf(Leaf::Unit(num, Unit::from_str(unit.as_slice()).unwrap()))
<l: @L> <num: Int> <unit: "unit"> <r: @R> => ExpressionBuilder::spanned_unit((num, Unit::from_str(unit.as_slice()).unwrap()), l, r),
}
String: Expression = {
<"sqstring"> => <>.as_slice()[1..(<>.as_slice().len() - 1)].to_string().into(),
<"dqstring"> => <>.as_slice()[1..(<>.as_slice().len() - 1)].to_string().into()
<l: @L> <s: "sqstring"> <r: @R> => ExpressionBuilder::spanned_string(&s.as_slice()[1..(s.as_slice().len() - 1)], l, r),
<l: @L> <s: "dqstring"> <r: @R> => ExpressionBuilder::spanned_string(&s.as_slice()[1..(s.as_slice().len() - 1)], l, r),
}
Flag: Flag = {
"-" <Bare> => Flag::Shorthand(<>.to_string()),
"--" <Bare> => Flag::Longhand(<>.to_string()),
Flag: Expression = {
<l: @L> "-" <b: Bare> <r: @R> => ExpressionBuilder::spanned_shorthand(b.to_string(), l, r),
<l: @L> "--" <b: Bare> <r: @R> => ExpressionBuilder::spanned_flag(b.to_string(), l, r),
}
Var: Expression = {
"$" <"variable"> => Variable::from_str(<>.as_slice()).into(),
<l: @L> "$" <v: "variable"> <r: @R> => ExpressionBuilder::spanned_var(v.as_slice(), l, r),
}
extern {

4701
src/parser/parser.rs
View File

File diff suppressed because it is too large Load Diff

46
src/parser/registry.rs
View File

@ -41,21 +41,31 @@ impl PositionalType {
match self {
PositionalType::Value(_) => evaluate_expr(&arg, scope),
PositionalType::Block(_) => match arg {
ast::Expression::Block(b) => Ok(Value::block(b.expr)),
ast::Expression::Binary(b) => {
if let Some(s) = b.left.as_string() {
Ok(Value::block(ast::Expression::Binary(Box::new(
ast::Binary::new(
ast::Expression::Path(Box::new(ast::Path::new(
ast::Expression::VariableReference(ast::Variable::It),
vec![s],
))),
b.operator,
b.right,
),
ast::Expression {
expr: ast::RawExpression::Block(b),
..
} => Ok(Value::block(b.expr)),
ast::Expression {
expr: ast::RawExpression::Binary(binary),
..
} => {
// TODO: Use original spans
let mut b = ast::ExpressionBuilder::new();
if let Some(s) = binary.left.as_string() {
Ok(Value::block(b.binary((
&|b| b.path((&|b| b.var("it"), vec![s.clone()])),
&|_| binary.operator.clone(),
&|_| binary.right.clone(),
))))
} else {
Ok(Value::block(ast::Expression::Binary(b)))
let mut b = ast::ExpressionBuilder::new();
let expr = b.binary((
&|_| binary.left.clone(),
&|_| binary.operator.clone(),
&|_| binary.right.clone(),
));
Ok(Value::block(expr))
}
}
other => Ok(Value::block(other)), // other =>
@ -192,13 +202,13 @@ fn extract_named(
}
fn expect_simple_expr(expr: ast::Expression) -> Result<Value, ShellError> {
match expr {
ast::Expression::Leaf(l) => Ok(match l {
match &*expr {
ast::RawExpression::Leaf(l) => Ok(match l {
ast::Leaf::Bare(s) => Value::string(s.to_string()),
ast::Leaf::String(s) => Value::string(s),
ast::Leaf::Boolean(b) => Value::boolean(b),
ast::Leaf::Int(i) => Value::int(i),
ast::Leaf::Unit(i, unit) => unit.compute(i),
ast::Leaf::Boolean(b) => Value::boolean(*b),
ast::Leaf::Int(i) => Value::int(*i),
ast::Leaf::Unit(i, unit) => unit.compute(*i),
}),
// TODO: Diagnostic