holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #874 from andrasio/move-out-tag

Browse Source 
Move out tags when parsing and building tree nodes.
This commit is contained in:
Andrés N. Robalino 2019-10-25 22:09:39 -05:00 committed by GitHub
commit 1b3a09495d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 114 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

173
src/parser/parse/parser.rs
View File

@ -393,7 +393,7 @@ pub fn leaf(input: NomSpan) -> IResult<NomSpan, TokenNode> {
} }
#[tracable_parser] #[tracable_parser]
pub fn token_list(input: NomSpan) -> IResult<NomSpan, Tagged<Vec<TokenNode>>> { pub fn token_list(input: NomSpan) -> IResult<NomSpan, Spanned<Vec<TokenNode>>> {
let start = input.offset; let start = input.offset;
let (input, first) = node(input)?; let (input, first) = node(input)?;
@ -403,7 +403,7 @@ pub fn token_list(input: NomSpan) -> IResult<NomSpan, Tagged<Vec<TokenNode>>> {
Ok(( Ok((
input, input,
make_token_list(first, list, None).tagged((start, end, None)), make_token_list(first, list, None).spanned(Span::new(start, end)),
)) ))
} }
@ -511,14 +511,14 @@ pub fn delimited_brace(input: NomSpan) -> IResult<NomSpan, TokenNode> {
} }
#[tracable_parser] #[tracable_parser]
pub fn raw_call(input: NomSpan) -> IResult<NomSpan, Tagged<CallNode>> { pub fn raw_call(input: NomSpan) -> IResult<NomSpan, Spanned<CallNode>> {
let left = input.offset; let left = input.offset;
let (input, items) = token_list(input)?; let (input, items) = token_list(input)?;
let right = input.offset; let right = input.offset;
Ok(( Ok((
input, input,
TokenTreeBuilder::tagged_call(items.item, (left, right, input.extra)), TokenTreeBuilder::spanned_call(items.item, Span::new(left, right)),
)) ))
} }
@ -598,7 +598,7 @@ pub fn nodes(input: NomSpan) -> IResult<NomSpan, TokenNode> {
Ok(( Ok((
input, input,
TokenTreeBuilder::tagged_token_list(tokens.item, tokens.tag), TokenTreeBuilder::spanned_token_list(tokens.item, tokens.span),
)) ))
} }
@ -800,30 +800,30 @@ mod tests {
">" -> b::token_list(vec![b::op(">")]) ">" -> b::token_list(vec![b::op(">")])
} }
// assert_leaf! { equal_tokens! {
// parsers [ operator ] <nodes>
// ">=" -> 0..2 { Operator(Operator::GreaterThanOrEqual) } ">=" -> b::token_list(vec![b::op(">=")])
// } }
// assert_leaf! { equal_tokens! {
// parsers [ operator ] <nodes>
// "<" -> 0..1 { Operator(Operator::LessThan) } "<" -> b::token_list(vec![b::op("<")])
// } }
// assert_leaf! { equal_tokens! {
// parsers [ operator ] <nodes>
// "<=" -> 0..2 { Operator(Operator::LessThanOrEqual) } "<=" -> b::token_list(vec![b::op("<=")])
// } }
// assert_leaf! { equal_tokens! {
// parsers [ operator ] <nodes>
// "==" -> 0..2 { Operator(Operator::Equal) } "==" -> b::token_list(vec![b::op("==")])
// } }
// assert_leaf! { equal_tokens! {
// parsers [ operator ] <nodes>
// "!=" -> 0..2 { Operator(Operator::NotEqual) } "!=" -> b::token_list(vec![b::op("!=")])
// } }
} }
#[test] #[test]
@ -848,12 +848,14 @@ mod tests {
} }
#[test] #[test]
fn test_simple_path() { fn test_unit_sizes() {
equal_tokens! { equal_tokens! {
<nodes> <nodes>
"450MB" -> b::token_list(vec![b::bare("450MB")]) "450MB" -> b::token_list(vec![b::bare("450MB")])
} }
}
#[test]
fn test_simple_path() {
equal_tokens! { equal_tokens! {
<nodes> <nodes>
"chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::op(Operator::Dot), b::bare("exe")]) "chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::op(Operator::Dot), b::bare("exe")])
@ -877,23 +879,23 @@ mod tests {
#[test] #[test]
fn test_flag() { fn test_flag() {
// assert_leaf! { equal_tokens! {
// parsers [ flag ] <nodes>
// "--hello" -> 0..7 { Flag(Tagged::from_item(FlagKind::Longhand, span(2, 7))) } "--amigos" -> b::token_list(vec![b::flag("arepas")])
// } }
// assert_leaf! { equal_tokens! {
// parsers [ flag ] <nodes>
// "--hello-world" -> 0..13 { Flag(Tagged::from_item(FlagKind::Longhand, span(2, 13))) } "--all-amigos" -> b::token_list(vec![b::flag("all-amigos")])
// } }
} }
#[test] #[test]
fn test_shorthand() { fn test_shorthand_flag() {
// assert_leaf! { equal_tokens! {
// parsers [ shorthand ] <nodes>
// "-alt" -> 0..4 { Flag(Tagged::from_item(FlagKind::Shorthand, span(1, 4))) } "-katz" -> b::token_list(vec![b::shorthand("katz")])
// } }
} }
#[test] #[test]
@ -939,13 +941,13 @@ mod tests {
equal_tokens! { equal_tokens! {
<nodes> <nodes>
"( abc def )" -> b::token_list(vec![b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" ")])]) "( abc def )" -> b::token_list(vec![b::parens(vec![b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp()])])
} }
equal_tokens! { equal_tokens! {
<nodes> <nodes>
"( abc def 123 456GB )" -> b::token_list(vec![b::parens(vec![ "( abc def 123 456GB )" -> b::token_list(vec![b::parens(vec![
b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" "), b::int(123), b::ws(" "), b::bare("456GB"), b::ws(" ") b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp(), b::int(123), b::sp(), b::bare("456GB"), b::sp()
])]) ])])
} }
} }
@ -964,13 +966,13 @@ mod tests {
equal_tokens! { equal_tokens! {
<nodes> <nodes>
"[ abc def ]" -> b::token_list(vec![b::square(vec![b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" ")])]) "[ abc def ]" -> b::token_list(vec![b::square(vec![b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp()])])
} }
equal_tokens! { equal_tokens! {
<nodes> <nodes>
"[ abc def 123 456GB ]" -> b::token_list(vec![b::square(vec![ "[ abc def 123 456GB ]" -> b::token_list(vec![b::square(vec![
b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" "), b::int(123), b::ws(" "), b::bare("456GB"), b::ws(" ") b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp(), b::int(123), b::sp(), b::bare("456GB"), b::sp()
])]) ])])
} }
} }
@ -984,6 +986,11 @@ mod tests {
"$it.print" -> b::token_list(vec![b::var("it"), b::op("."), b::bare("print")]) "$it.print" -> b::token_list(vec![b::var("it"), b::op("."), b::bare("print")])
} }
equal_tokens! {
<nodes>
"$it.0" -> b::token_list(vec![b::var("it"), b::op("."), b::int(0)])
}
equal_tokens! { equal_tokens! {
<nodes> <nodes>
"$head.part1.part2" -> b::token_list(vec![b::var("head"), b::op("."), b::bare("part1"), b::op("."), b::bare("part2")]) "$head.part1.part2" -> b::token_list(vec![b::var("head"), b::op("."), b::bare("part1"), b::op("."), b::bare("part2")])
@ -1024,6 +1031,19 @@ mod tests {
b::op("."), b::string("world")] b::op("."), b::string("world")]
) )
} }
equal_tokens! {
<nodes>
r#"$it."are PAS".0"# -> b::token_list(
vec![
b::var("it"),
b::op("."),
b::string("are PAS"),
b::op("."),
b::int(0),
]
)
}
} }
#[test] #[test]
@ -1062,6 +1082,19 @@ mod tests {
<nodes> <nodes>
"config --set tabs 2" -> b::token_list(vec![b::bare("config"), b::sp(), b::flag("set"), b::sp(), b::bare("tabs"), b::sp(), b::int(2)]) "config --set tabs 2" -> b::token_list(vec![b::bare("config"), b::sp(), b::flag("set"), b::sp(), b::bare("tabs"), b::sp(), b::int(2)])
} }
equal_tokens! {
<nodes>
"inc --patch package.version" -> b::token_list(
vec![
b::bare("inc"),
b::sp(),
b::flag("patch"),
b::sp(),
b::bare("package"), b::op("."), b::bare("version")
]
)
}
} }
#[test] #[test]
@ -1114,41 +1147,39 @@ mod tests {
fn test_patterns() { fn test_patterns() {
equal_tokens! { equal_tokens! {
<pipeline> <pipeline>
"cp ../formats/*" -> b::pipeline(vec![vec![b::bare("cp"), b::ws(" "), b::op("."), b::op("."), b::pattern("/formats/*")]]) "cp ../formats/*" -> b::pipeline(vec![vec![b::bare("cp"), b::sp(), b::op("."), b::op("."), b::pattern("/formats/*")]])
} }
equal_tokens! { equal_tokens! {
<pipeline> <pipeline>
"cp * /dev/null" -> b::pipeline(vec![vec![b::bare("cp"), b::ws(" "), b::pattern("*"), b::ws(" "), b::bare("/dev/null")]]) "cp * /dev/null" -> b::pipeline(vec![vec![b::bare("cp"), b::sp(), b::pattern("*"), b::sp(), b::bare("/dev/null")]])
} }
} }
// #[test] #[test]
// fn test_pseudo_paths() { fn test_pseudo_paths() {
// let _ = pretty_env_logger::try_init(); let _ = pretty_env_logger::try_init();
// equal_tokens!( equal_tokens!(
// r#"sys | where cpu."max ghz" > 1"# -> <pipeline>
// b::pipeline(vec![ r#"sys | where cpu."max ghz" > 1"# -> b::pipeline(vec![
// (None, b::call(b::bare("sys"), vec![]), Some(" ")), vec![
// ( b::bare("sys"), b::sp()
// Some(" "), ],
// b::call( vec![
// b::bare("where"), b::sp(),
// vec![ b::bare("where"),
// b::sp(), b::sp(),
// b::path(b::bare("cpu"), vec![b::string("max ghz")]), b::bare("cpu"),
// b::sp(), b::op("."),
// b::op(">"), b::string("max ghz"),
// b::sp(), b::sp(),
// b::int(1) b::op(">"),
// ] b::sp(),
// ), b::int(1)
// None ]])
// ) );
// ]) }
// );
// }
// #[test] // #[test]
// fn test_smoke_pipeline() { // fn test_smoke_pipeline() {

2
src/parser/parse/token_tree.rs
View File

@ -274,7 +274,7 @@ impl TokenNode {
item: RawToken::Bare, item: RawToken::Bare,
span, span,
}) => *span, }) => *span,
other => panic!("Expected var, found {:?}", other), other => panic!("Expected bare, found {:?}", other),
} }
} }
} }

14
src/parser/parse/token_tree_builder.rs
View File

@ -18,7 +18,7 @@ pub struct TokenTreeBuilder {
} }
pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> TokenNode + 'static>; pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> TokenNode + 'static>;
pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Tagged<CallNode> + 'static>; pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Spanned<CallNode> + 'static>;
impl TokenTreeBuilder { impl TokenTreeBuilder {
pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) { pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) {
@ -89,12 +89,12 @@ impl TokenTreeBuilder {
let tokens = input.into_iter().map(|i| i(b)).collect(); let tokens = input.into_iter().map(|i| i(b)).collect();
let end = b.pos; let end = b.pos;
TokenTreeBuilder::tagged_token_list(tokens, (start, end, None)) TokenTreeBuilder::spanned_token_list(tokens, Span::new(start, end))
}) })
} }
pub fn tagged_token_list(input: Vec<TokenNode>, tag: impl Into<Tag>) -> TokenNode { pub fn spanned_token_list(input: Vec<TokenNode>, span: impl Into<Span>) -> TokenNode {
TokenNode::Nodes(input.spanned(tag.into().span)) TokenNode::Nodes(input.spanned(span.into()))
} }
pub fn op(input: impl Into<Operator>) -> CurriedToken { pub fn op(input: impl Into<Operator>) -> CurriedToken {
@ -287,11 +287,11 @@ impl TokenTreeBuilder {
let end = b.pos; let end = b.pos;
TokenTreeBuilder::tagged_call(nodes, (start, end, None)) TokenTreeBuilder::spanned_call(nodes, Span::new(start, end))
}) })
} }
pub fn tagged_call(input: Vec<TokenNode>, tag: impl Into<Tag>) -> Tagged<CallNode> { pub fn spanned_call(input: Vec<TokenNode>, span: impl Into<Span>) -> Spanned<CallNode> {
if input.len() == 0 { if input.len() == 0 {
panic!("BUG: spanned call (TODO)") panic!("BUG: spanned call (TODO)")
} }
@ -301,7 +301,7 @@ impl TokenTreeBuilder {
let head = input.next().unwrap(); let head = input.next().unwrap();
let tail = input.collect(); let tail = input.collect();
CallNode::new(Box::new(head), tail).tagged(tag.into()) CallNode::new(Box::new(head), tail).spanned(span.into())
} }
fn consume_delimiter( fn consume_delimiter(

8
src/parser/parse/tokens.rs
View File

@ -19,14 +19,14 @@ pub enum RawToken {
impl RawToken { impl RawToken {
pub fn type_name(&self) -> &'static str { pub fn type_name(&self) -> &'static str {
match self { match self {
RawToken::Number(_) => "Number", RawToken::Number(_) => "number",
RawToken::Operator(..) => "operator", RawToken::Operator(..) => "operator",
RawToken::String(_) => "String", RawToken::String(_) => "string",
RawToken::Variable(_) => "variable", RawToken::Variable(_) => "variable",
RawToken::ExternalCommand(_) => "external command", RawToken::ExternalCommand(_) => "external command",
RawToken::ExternalWord => "external word", RawToken::ExternalWord => "external word",
RawToken::GlobPattern => "glob pattern", RawToken::GlobPattern => "glob pattern",
RawToken::Bare => "String", RawToken::Bare => "string",
} }
} }
} }
@ -72,7 +72,7 @@ impl Token {
pub fn extract_number(&self) -> Option<Spanned<RawNumber>> { pub fn extract_number(&self) -> Option<Spanned<RawNumber>> {
match self.item { match self.item {
RawToken::Number(number) => Some((number).spanned(self.span)), RawToken::Number(number) => Some(number.spanned(self.span)),
_ => None, _ => None,
} }
} }