extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2024-10-05 09:52:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

More delimiters

Browse Source
This commit is contained in:
Yehuda Katz 2019-06-14 19:59:35 -07:00
parent 640484063b
commit 6e222eec2b
3 changed files with 392 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

226
src/parser/parse2/parser.rs
View File

@ -1,6 +1,8 @@
#![allow(unused)]
use crate::parser::parse2::{flag::*, operator::*, span::*, token_tree::*, tokens::*, unit::*};
use crate::parser::parse2::{
flag::*, operator::*, span::*, token_tree::*, token_tree_builder::*, tokens::*, unit::*,
};
use nom;
use nom::dbg;
use nom::types::CompleteStr;
@ -12,12 +14,13 @@ type NomSpan<'a> = LocatedSpan<CompleteStr<'a>>;
macro_rules! operator {
($name:tt : $token:tt ) => {
named!($name( NomSpan ) -> Token,
named!($name( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> t: tag!(stringify!($token))
>> r: position!()
>> (Spanned::from_nom(RawToken::Operator(Operator::from_str(t.fragment.0).unwrap()), l, r))
>> (TokenTreeBuilder::spanned_op(t.fragment.0, (l, r)))
// >> (Spanned::from_nom(RawToken::Operator(Operator::from_str(t.fragment.0).unwrap()), l, r))
)
);
};
@ -40,20 +43,20 @@ named!(pub raw_integer( NomSpan ) -> Spanned<i64>,
)
);
named!(pub integer( NomSpan ) -> Token,
named!(pub integer( NomSpan ) -> TokenNode,
do_parse!(
int: raw_integer
>> (int.map(|i| RawToken::Integer(i)))
>> (TokenTreeBuilder::spanned_int(*int, int.span))
)
);
named!(pub operator( NomSpan ) -> Token,
named!(pub operator( NomSpan ) -> TokenNode,
alt!(
gte | lte | neq | gt | lt | eq
)
);
named!(pub dq_string( NomSpan ) -> Token,
named!(pub dq_string( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> char!('"')
@ -62,11 +65,11 @@ named!(pub dq_string( NomSpan ) -> Token,
>> r1: position!()
>> char!('"')
>> r: position!()
>> (Spanned::from_nom(RawToken::String(Span::from((l1, r1))), l, r))
>> (TokenTreeBuilder::spanned_string((l1, r1), (l, r)))
)
);
named!(pub sq_string( NomSpan ) -> Token,
named!(pub sq_string( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> char!('\'')
@ -75,61 +78,61 @@ named!(pub sq_string( NomSpan ) -> Token,
>> r1: position!()
>> char!('\'')
>> r: position!()
>> (Spanned::from_nom(RawToken::String(Span::from((l1, r1))), l, r))
>> (TokenTreeBuilder::spanned_string((l1, r1), (l, r)))
)
);
named!(pub string( NomSpan ) -> Token,
named!(pub string( NomSpan ) -> TokenNode,
alt!(sq_string | dq_string)
);
named!(pub bare( NomSpan ) -> Token,
named!(pub bare( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> take_while1!(is_start_bare_char)
>> take_while!(is_bare_char)
>> r: position!()
>> (Spanned::from_nom(RawToken::Bare, l, r))
>> (TokenTreeBuilder::spanned_bare((l, r)))
)
);
named!(pub var( NomSpan ) -> Token,
named!(pub var( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> tag!("$")
>> bare: identifier
>> r: position!()
>> (Spanned::from_nom(RawToken::Variable(bare.span), l, r))
>> (TokenTreeBuilder::spanned_var(bare.span(), (l, r)))
)
);
named!(pub identifier( NomSpan ) -> Token,
named!(pub identifier( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> take_while1!(is_id_start)
>> take_while!(is_id_continue)
>> r: position!()
>> (Spanned::from_nom(RawToken::Identifier, l, r))
>> (TokenTreeBuilder::spanned_ident((l, r)))
)
);
named!(pub flag( NomSpan ) -> Token,
named!(pub flag( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> tag!("--")
>> bare: bare
>> r: position!()
>> (Spanned::from_nom(RawToken::Flag(Flag::Longhand, bare.span), l, r))
>> (TokenTreeBuilder::spanned_flag(bare.span(), (l, r)))
)
);
named!(pub shorthand( NomSpan ) -> Token,
named!(pub shorthand( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> tag!("-")
>> bare: bare
>> r: position!()
>> (Spanned::from_nom(RawToken::Flag(Flag::Shorthand, bare.span), l, r))
>> (TokenTreeBuilder::spanned_shorthand(bare.span(), (l, r)))
)
);
@ -142,34 +145,20 @@ named!(pub raw_unit( NomSpan ) -> Spanned<Unit>,
)
);
named!(pub size( NomSpan ) -> Token,
named!(pub size( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> int: raw_integer
>> unit: raw_unit
>> r: position!()
>> (Spanned::from_nom(RawToken::Size(int.item, unit.item), l, r))
>> (TokenTreeBuilder::spanned_size((*int, *unit), (l, r)))
)
);
// named!(pub unit_num( NomSpan ) -> Token,
// do_parse!(
// l: position!()
// >>
// )
// )
named!(pub leaf( NomSpan ) -> Token,
named!(pub leaf( NomSpan ) -> TokenNode,
alt!(size | integer | string | operator | flag | shorthand | var | bare)
);
named!(pub leaf_node( NomSpan ) -> TokenNode,
do_parse!(
leaf: leaf
>> (TokenNode::Token(leaf))
)
);
named!(pub delimited_paren( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
@ -179,7 +168,30 @@ named!(pub delimited_paren( NomSpan ) -> TokenNode,
char!(')')
)
>> r: position!()
>> (TokenNode::Delimited(Spanned::from_nom(DelimitedNode::new(Delimiter::Paren, items), l, r)))
>> (TokenTreeBuilder::spanned_parens(items, (l, r)))
)
);
named!(pub delimited_brace( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> items: delimited!(
char!('{'),
delimited!(space0, separated_list!(space1, node), space0),
char!('}')
)
>> r: position!()
>> (TokenTreeBuilder::spanned_brace(items, (l, r)))
)
);
named!(pub raw_call( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> head: node
>> items: opt!(preceded!(space0, separated_nonempty_list!(space1, node)))
>> r: position!()
>> (TokenTreeBuilder::spanned_call((head, items), (l, r)))
)
);
@ -190,16 +202,25 @@ named!(pub path( NomSpan ) -> TokenNode,
>> tag!(".")
>> tail: separated_list!(tag!("."), alt!(identifier | string))
>> r: position!()
>> (TokenNode::Path(Spanned::from_nom(PathNode::new(Box::new(head), tail.into_iter().map(TokenNode::Token).collect()), l, r)))
>> (TokenTreeBuilder::spanned_path((head, tail), (l, r)))
)
);
named!(pub node1( NomSpan ) -> TokenNode,
alt!(leaf_node | delimited_paren)
alt!(leaf | delimited_paren)
);
named!(pub node( NomSpan ) -> TokenNode,
alt!(path | leaf_node | delimited_paren)
alt!(path | leaf | delimited_paren | delimited_brace)
);
named!(pub pipeline( NomSpan ) -> TokenNode,
do_parse!(
l: position!()
>> list: separated_list!(delimited!(space0, tag!("|"), space0), raw_call)
>> r: position!()
>> (TokenTreeBuilder::spanned_pipeline(list, (l, r)))
)
);
fn int<T>(frag: &str, neg: Option<T>) -> i64 {
@ -273,13 +294,13 @@ mod tests {
);
assert_eq!(
apply(leaf_node, $input),
TokenNode::Token(token(RawToken::$kind $parens, $left, $right))
apply(leaf, $input),
token(RawToken::$kind $parens, $left, $right)
);
assert_eq!(
apply(node, $input),
TokenNode::Token(token(RawToken::$kind $parens, $left, $right))
token(RawToken::$kind $parens, $left, $right)
);
};
@ -512,6 +533,117 @@ mod tests {
)
}
#[test]
fn test_smoke_single_command() {
assert_eq!(
apply(raw_call, "git add ."),
build(b::call(
b::bare("git"),
vec![b::sp(), b::bare("add"), b::sp(), b::bare(".")]
))
);
assert_eq!(
apply(raw_call, "open Cargo.toml"),
build(b::call(
b::bare("open"),
vec![b::sp(), b::bare("Cargo.toml")]
))
);
assert_eq!(
apply(raw_call, "select package.version"),
build(b::call(
b::bare("select"),
vec![b::sp(), b::bare("package.version")]
))
);
assert_eq!(
apply(raw_call, "echo $it"),
build(b::call(b::bare("echo"), vec![b::sp(), b::var("it")]))
);
assert_eq!(
apply(raw_call, "open Cargo.toml --raw"),
build(b::call(
b::bare("open"),
vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::flag("raw")]
))
);
assert_eq!(
apply(raw_call, "open Cargo.toml -r"),
build(b::call(
b::bare("open"),
vec![b::sp(), b::bare("Cargo.toml"), b::sp(), b::shorthand("r")]
))
);
assert_eq!(
apply(raw_call, "config --set tabs 2"),
build(b::call(
b::bare("config"),
vec![
b::sp(),
b::flag("set"),
b::sp(),
b::bare("tabs"),
b::sp(),
b::int(2)
]
))
);
}
#[test]
fn test_smoke_pipeline() {
assert_eq!(
apply(
pipeline,
r#"git branch --merged | split-row "`n" | where $it != "* master""#
),
build(b::pipeline(vec![
b::call(
b::bare("git"),
vec![b::sp(), b::bare("branch"), b::sp(), b::flag("merged")]
),
b::call(b::bare("split-row"), vec![b::sp(), b::string("`n")]),
b::call(
b::bare("where"),
vec![
b::sp(),
b::var("it"),
b::sp(),
b::op("!="),
b::sp(),
b::string("* master")
]
)
]))
);
assert_eq!(
apply(pipeline, "ls | where { $it.size > 100 }"),
build(b::pipeline(vec![
b::call(b::bare("ls"), vec![]),
b::call(
b::bare("where"),
vec![
b::sp(),
b::braced(vec![
b::path(b::var("it"), vec![b::ident("size")]),
b::sp(),
b::op(">"),
b::sp(),
b::int(100)
])
]
)
]))
)
}
fn apply<T>(f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err<NomSpan>>, string: &str) -> T {
match f(NomSpan::new(CompleteStr(string))) {
Ok(v) => v.1,
@ -550,8 +682,8 @@ mod tests {
TokenNode::Token(Spanned::from_item(token, (left, right)))
}
fn token(token: RawToken, left: usize, right: usize) -> Token {
Spanned::from_item(token, (left, right))
fn token(token: RawToken, left: usize, right: usize) -> TokenNode {
TokenNode::Token(Spanned::from_item(token, (left, right)))
}
fn build(block: CurriedToken) -> TokenNode {

31
src/parser/parse2/token_tree.rs
View File

@ -1,20 +1,42 @@
use crate::parser::parse2::{span::*, tokens::*};
use crate::parser::parse2::{operator::*, span::*, tokens::*};
use derive_new::new;
use enum_utils::FromStr;
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub enum TokenNode {
Token(Token),
Call(Spanned<CallNode>),
Delimited(Spanned<DelimitedNode>),
Pipeline(Spanned<Vec<TokenNode>>),
Binary(Spanned<BinaryNode>),
Path(Spanned<PathNode>),
}
impl TokenNode {
pub fn span(&self) -> Span {
match self {
TokenNode::Token(t) => t.span,
TokenNode::Call(s) => s.span,
TokenNode::Delimited(s) => s.span,
TokenNode::Pipeline(s) => s.span,
TokenNode::Binary(s) => s.span,
TokenNode::Path(s) => s.span,
}
}
}
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)]
pub struct DelimitedNode {
delimiter: Delimiter,
children: Vec<TokenNode>,
}
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)]
pub struct CallNode {
head: Box<TokenNode>,
children: Vec<TokenNode>,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, FromStr)]
pub enum Delimiter {
Paren,
@ -27,3 +49,10 @@ pub struct PathNode {
head: Box<TokenNode>,
tail: Vec<TokenNode>,
}
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)]
pub struct BinaryNode {
left: Box<TokenNode>,
op: Operator,
right: Box<TokenNode>,
}

188
src/parser/parse2/token_tree_builder.rs
View File

@ -1,6 +1,9 @@
use crate::parser::parse2::flag::Flag;
use crate::parser::parse2::operator::Operator;
use crate::parser::parse2::span::{Span, Spanned};
use crate::parser::parse2::token_tree::{DelimitedNode, Delimiter, PathNode, TokenNode};
use crate::parser::parse2::token_tree::{
BinaryNode, CallNode, DelimitedNode, Delimiter, PathNode, TokenNode,
};
use crate::parser::parse2::tokens::{RawToken, Token};
use crate::parser::parse2::unit::Unit;
use derive_new::new;
@ -20,6 +23,38 @@ impl TokenTreeBuilder {
block(&mut builder)
}
pub fn pipeline(input: Vec<CurriedToken>) -> CurriedToken {
Box::new(move |b| {
let start = b.pos;
let mut out = vec![];
let mut input = input.into_iter();
let first = input
.next()
.expect("A pipeline must contain at least one element");
out.push(first(b).expect("The first element of a pipeline must not be whitespace"));
for item in input {
b.consume(" | ");
match item(b) {
None => {}
Some(v) => out.push(v),
}
}
let end = b.pos;
Some(TokenTreeBuilder::spanned_pipeline(out, (start, end)))
})
}
pub fn spanned_pipeline(input: Vec<TokenNode>, span: impl Into<Span>) -> TokenNode {
TokenNode::Pipeline(Spanned::from_item(input, span))
}
pub fn op(input: impl Into<Operator>) -> CurriedToken {
let input = input.into();
@ -164,6 +199,48 @@ impl TokenTreeBuilder {
))
}
pub fn flag(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, _) = b.consume("--");
let (inner_start, end) = b.consume(&input);
Some(TokenTreeBuilder::spanned_flag(
(inner_start, end),
(start, end),
))
})
}
pub fn spanned_flag(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Token(Spanned::from_item(
RawToken::Flag(Flag::Longhand, input.into()),
span.into(),
))
}
pub fn shorthand(input: impl Into<String>) -> CurriedToken {
let input = input.into();
Box::new(move |b| {
let (start, _) = b.consume("-");
let (inner_start, end) = b.consume(&input);
Some(TokenTreeBuilder::spanned_shorthand(
(inner_start, end),
(start, end),
))
})
}
pub fn spanned_shorthand(input: impl Into<Span>, span: impl Into<Span>) -> TokenNode {
TokenNode::Token(Spanned::from_item(
RawToken::Flag(Flag::Shorthand, input.into()),
span.into(),
))
}
pub fn ident(input: impl Into<String>) -> CurriedToken {
let input = input.into();
@ -177,6 +254,39 @@ impl TokenTreeBuilder {
TokenNode::Token(Spanned::from_item(RawToken::Identifier, span.into()))
}
pub fn call(head: CurriedToken, input: Vec<CurriedToken>) -> CurriedToken {
Box::new(move |b| {
let start = b.pos;
let head_node = head(b).expect("The head of a command must not be whitespace");
let mut tail_nodes = vec![];
for item in input {
match item(b) {
None => {}
Some(v) => tail_nodes.push(v),
};
}
let end = b.pos;
Some(TokenTreeBuilder::spanned_call(
(head_node, Some(tail_nodes)),
(start, end),
))
})
}
pub fn spanned_call(
input: (TokenNode, Option<Vec<TokenNode>>),
span: impl Into<Span>,
) -> TokenNode {
TokenNode::Call(Spanned::from_item(
CallNode::new(Box::new(input.0), input.1.unwrap_or_else(|| vec![])),
span,
))
}
pub fn parens(input: Vec<CurriedToken>) -> CurriedToken {
Box::new(move |b| {
let (start, _) = b.consume("(");
@ -190,13 +300,81 @@ impl TokenTreeBuilder {
let (_, end) = b.consume(")");
Some(TokenNode::Delimited(Spanned::from_item(
DelimitedNode::new(Delimiter::Paren, output),
(start, end),
)))
Some(TokenTreeBuilder::spanned_parens(output, (start, end)))
})
}
pub fn spanned_parens(input: impl Into<Vec<TokenNode>>, span: impl Into<Span>) -> TokenNode {
TokenNode::Delimited(Spanned::from_item(
DelimitedNode::new(Delimiter::Paren, input.into()),
span,
))
}
pub fn braced(input: Vec<CurriedToken>) -> CurriedToken {
Box::new(move |b| {
let (start, _) = b.consume("{ ");
let mut output = vec![];
for item in input {
match item(b) {
None => {}
Some(v) => output.push(v),
};
}
let (_, end) = b.consume(" }");
Some(TokenTreeBuilder::spanned_brace(output, (start, end)))
})
}
pub fn spanned_brace(input: impl Into<Vec<TokenNode>>, span: impl Into<Span>) -> TokenNode {
TokenNode::Delimited(Spanned::from_item(
DelimitedNode::new(Delimiter::Brace, input.into()),
span,
))
}
pub fn binary(
left: CurriedToken,
op: impl Into<Operator>,
right: CurriedToken,
) -> CurriedToken {
let op = op.into();
Box::new(move |b| {
let start = b.pos;
let left = left(b).expect("The left side of a binary operation must not be whitespace");
b.consume(" ");
b.consume(op.as_str());
b.consume(" ");
let right =
right(b).expect("The right side of a binary operation must not be whitespace");
let end = b.pos;
Some(TokenTreeBuilder::spanned_binary(
(left, op, right),
(start, end),
))
})
}
pub fn spanned_binary(
input: (impl Into<TokenNode>, Operator, impl Into<TokenNode>),
span: impl Into<Span>,
) -> TokenNode {
TokenNode::Binary(Spanned::from_item(
BinaryNode::new(Box::new(input.0.into()), input.1, Box::new(input.2.into())),
span,
))
}
pub fn sp() -> CurriedToken {
Box::new(|b| {
b.consume(" ");