From 2706ae076d8782800849908cdaf6d2e36f95e501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20N=2E=20Robalino?= Date: Fri, 25 Oct 2019 18:31:25 -0500 Subject: [PATCH] Move out tags when parsing and building tree nodes. --- src/parser/parse/parser.rs | 173 +++++++++++++++---------- src/parser/parse/token_tree.rs | 2 +- src/parser/parse/token_tree_builder.rs | 14 +- src/parser/parse/tokens.rs | 8 +- 4 files changed, 114 insertions(+), 83 deletions(-) diff --git a/src/parser/parse/parser.rs b/src/parser/parse/parser.rs index a548102db..f7fce7c81 100644 --- a/src/parser/parse/parser.rs +++ b/src/parser/parse/parser.rs @@ -393,7 +393,7 @@ pub fn leaf(input: NomSpan) -> IResult { } #[tracable_parser] -pub fn token_list(input: NomSpan) -> IResult>> { +pub fn token_list(input: NomSpan) -> IResult>> { let start = input.offset; let (input, first) = node(input)?; @@ -403,7 +403,7 @@ pub fn token_list(input: NomSpan) -> IResult>> { Ok(( input, - make_token_list(first, list, None).tagged((start, end, None)), + make_token_list(first, list, None).spanned(Span::new(start, end)), )) } @@ -511,14 +511,14 @@ pub fn delimited_brace(input: NomSpan) -> IResult { } #[tracable_parser] -pub fn raw_call(input: NomSpan) -> IResult> { +pub fn raw_call(input: NomSpan) -> IResult> { let left = input.offset; let (input, items) = token_list(input)?; let right = input.offset; Ok(( input, - TokenTreeBuilder::tagged_call(items.item, (left, right, input.extra)), + TokenTreeBuilder::spanned_call(items.item, Span::new(left, right)), )) } @@ -598,7 +598,7 @@ pub fn nodes(input: NomSpan) -> IResult { Ok(( input, - TokenTreeBuilder::tagged_token_list(tokens.item, tokens.tag), + TokenTreeBuilder::spanned_token_list(tokens.item, tokens.span), )) } @@ -800,30 +800,30 @@ mod tests { ">" -> b::token_list(vec![b::op(">")]) } - // assert_leaf! { - // parsers [ operator ] - // ">=" -> 0..2 { Operator(Operator::GreaterThanOrEqual) } - // } + equal_tokens! { + + ">=" -> b::token_list(vec![b::op(">=")]) + } - // assert_leaf! { - // parsers [ operator ] - // "<" -> 0..1 { Operator(Operator::LessThan) } - // } + equal_tokens! { + + "<" -> b::token_list(vec![b::op("<")]) + } - // assert_leaf! { - // parsers [ operator ] - // "<=" -> 0..2 { Operator(Operator::LessThanOrEqual) } - // } + equal_tokens! { + + "<=" -> b::token_list(vec![b::op("<=")]) + } - // assert_leaf! { - // parsers [ operator ] - // "==" -> 0..2 { Operator(Operator::Equal) } - // } + equal_tokens! { + + "==" -> b::token_list(vec![b::op("==")]) + } - // assert_leaf! { - // parsers [ operator ] - // "!=" -> 0..2 { Operator(Operator::NotEqual) } - // } + equal_tokens! { + + "!=" -> b::token_list(vec![b::op("!=")]) + } } #[test] @@ -848,12 +848,14 @@ mod tests { } #[test] - fn test_simple_path() { + fn test_unit_sizes() { equal_tokens! { "450MB" -> b::token_list(vec![b::bare("450MB")]) } - + } + #[test] + fn test_simple_path() { equal_tokens! { "chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::op(Operator::Dot), b::bare("exe")]) @@ -877,23 +879,23 @@ mod tests { #[test] fn test_flag() { - // assert_leaf! { - // parsers [ flag ] - // "--hello" -> 0..7 { Flag(Tagged::from_item(FlagKind::Longhand, span(2, 7))) } - // } + equal_tokens! { + + "--amigos" -> b::token_list(vec![b::flag("arepas")]) + } - // assert_leaf! { - // parsers [ flag ] - // "--hello-world" -> 0..13 { Flag(Tagged::from_item(FlagKind::Longhand, span(2, 13))) } - // } + equal_tokens! { + + "--all-amigos" -> b::token_list(vec![b::flag("all-amigos")]) + } } #[test] - fn test_shorthand() { - // assert_leaf! { - // parsers [ shorthand ] - // "-alt" -> 0..4 { Flag(Tagged::from_item(FlagKind::Shorthand, span(1, 4))) } - // } + fn test_shorthand_flag() { + equal_tokens! { + + "-katz" -> b::token_list(vec![b::shorthand("katz")]) + } } #[test] @@ -939,13 +941,13 @@ mod tests { equal_tokens! { - "( abc def )" -> b::token_list(vec![b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" ")])]) + "( abc def )" -> b::token_list(vec![b::parens(vec![b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp()])]) } equal_tokens! { "( abc def 123 456GB )" -> b::token_list(vec![b::parens(vec![ - b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" "), b::int(123), b::ws(" "), b::bare("456GB"), b::ws(" ") + b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp(), b::int(123), b::sp(), b::bare("456GB"), b::sp() ])]) } } @@ -964,13 +966,13 @@ mod tests { equal_tokens! { - "[ abc def ]" -> b::token_list(vec![b::square(vec![b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" ")])]) + "[ abc def ]" -> b::token_list(vec![b::square(vec![b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp()])]) } equal_tokens! { "[ abc def 123 456GB ]" -> b::token_list(vec![b::square(vec![ - b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" "), b::int(123), b::ws(" "), b::bare("456GB"), b::ws(" ") + b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp(), b::int(123), b::sp(), b::bare("456GB"), b::sp() ])]) } } @@ -984,6 +986,11 @@ mod tests { "$it.print" -> b::token_list(vec![b::var("it"), b::op("."), b::bare("print")]) } + equal_tokens! { + + "$it.0" -> b::token_list(vec![b::var("it"), b::op("."), b::int(0)]) + } + equal_tokens! { "$head.part1.part2" -> b::token_list(vec![b::var("head"), b::op("."), b::bare("part1"), b::op("."), b::bare("part2")]) @@ -1024,6 +1031,19 @@ mod tests { b::op("."), b::string("world")] ) } + + equal_tokens! { + + r#"$it."are PAS".0"# -> b::token_list( + vec![ + b::var("it"), + b::op("."), + b::string("are PAS"), + b::op("."), + b::int(0), + ] + ) + } } #[test] @@ -1062,6 +1082,19 @@ mod tests { "config --set tabs 2" -> b::token_list(vec![b::bare("config"), b::sp(), b::flag("set"), b::sp(), b::bare("tabs"), b::sp(), b::int(2)]) } + + equal_tokens! { + + "inc --patch package.version" -> b::token_list( + vec![ + b::bare("inc"), + b::sp(), + b::flag("patch"), + b::sp(), + b::bare("package"), b::op("."), b::bare("version") + ] + ) + } } #[test] @@ -1114,41 +1147,39 @@ mod tests { fn test_patterns() { equal_tokens! { - "cp ../formats/*" -> b::pipeline(vec![vec![b::bare("cp"), b::ws(" "), b::op("."), b::op("."), b::pattern("/formats/*")]]) + "cp ../formats/*" -> b::pipeline(vec![vec![b::bare("cp"), b::sp(), b::op("."), b::op("."), b::pattern("/formats/*")]]) } equal_tokens! { - "cp * /dev/null" -> b::pipeline(vec![vec![b::bare("cp"), b::ws(" "), b::pattern("*"), b::ws(" "), b::bare("/dev/null")]]) + "cp * /dev/null" -> b::pipeline(vec![vec![b::bare("cp"), b::sp(), b::pattern("*"), b::sp(), b::bare("/dev/null")]]) } } - // #[test] - // fn test_pseudo_paths() { - // let _ = pretty_env_logger::try_init(); + #[test] + fn test_pseudo_paths() { + let _ = pretty_env_logger::try_init(); - // equal_tokens!( - // r#"sys | where cpu."max ghz" > 1"# -> - // b::pipeline(vec![ - // (None, b::call(b::bare("sys"), vec![]), Some(" ")), - // ( - // Some(" "), - // b::call( - // b::bare("where"), - // vec![ - // b::sp(), - // b::path(b::bare("cpu"), vec![b::string("max ghz")]), - // b::sp(), - // b::op(">"), - // b::sp(), - // b::int(1) - // ] - // ), - // None - // ) - // ]) - // ); - // } + equal_tokens!( + + r#"sys | where cpu."max ghz" > 1"# -> b::pipeline(vec![ + vec![ + b::bare("sys"), b::sp() + ], + vec![ + b::sp(), + b::bare("where"), + b::sp(), + b::bare("cpu"), + b::op("."), + b::string("max ghz"), + b::sp(), + b::op(">"), + b::sp(), + b::int(1) + ]]) + ); + } // #[test] // fn test_smoke_pipeline() { diff --git a/src/parser/parse/token_tree.rs b/src/parser/parse/token_tree.rs index c3c1df652..cb335e925 100644 --- a/src/parser/parse/token_tree.rs +++ b/src/parser/parse/token_tree.rs @@ -274,7 +274,7 @@ impl TokenNode { item: RawToken::Bare, span, }) => *span, - other => panic!("Expected var, found {:?}", other), + other => panic!("Expected bare, found {:?}", other), } } } diff --git a/src/parser/parse/token_tree_builder.rs b/src/parser/parse/token_tree_builder.rs index 891e6b9e1..7146a3c20 100644 --- a/src/parser/parse/token_tree_builder.rs +++ b/src/parser/parse/token_tree_builder.rs @@ -18,7 +18,7 @@ pub struct TokenTreeBuilder { } pub type CurriedToken = Box TokenNode + 'static>; -pub type CurriedCall = Box Tagged + 'static>; +pub type CurriedCall = Box Spanned + 'static>; impl TokenTreeBuilder { pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) { @@ -89,12 +89,12 @@ impl TokenTreeBuilder { let tokens = input.into_iter().map(|i| i(b)).collect(); let end = b.pos; - TokenTreeBuilder::tagged_token_list(tokens, (start, end, None)) + TokenTreeBuilder::spanned_token_list(tokens, Span::new(start, end)) }) } - pub fn tagged_token_list(input: Vec, tag: impl Into) -> TokenNode { - TokenNode::Nodes(input.spanned(tag.into().span)) + pub fn spanned_token_list(input: Vec, span: impl Into) -> TokenNode { + TokenNode::Nodes(input.spanned(span.into())) } pub fn op(input: impl Into) -> CurriedToken { @@ -287,11 +287,11 @@ impl TokenTreeBuilder { let end = b.pos; - TokenTreeBuilder::tagged_call(nodes, (start, end, None)) + TokenTreeBuilder::spanned_call(nodes, Span::new(start, end)) }) } - pub fn tagged_call(input: Vec, tag: impl Into) -> Tagged { + pub fn spanned_call(input: Vec, span: impl Into) -> Spanned { if input.len() == 0 { panic!("BUG: spanned call (TODO)") } @@ -301,7 +301,7 @@ impl TokenTreeBuilder { let head = input.next().unwrap(); let tail = input.collect(); - CallNode::new(Box::new(head), tail).tagged(tag.into()) + CallNode::new(Box::new(head), tail).spanned(span.into()) } fn consume_delimiter( diff --git a/src/parser/parse/tokens.rs b/src/parser/parse/tokens.rs index 94955d84d..29061ed7a 100644 --- a/src/parser/parse/tokens.rs +++ b/src/parser/parse/tokens.rs @@ -19,14 +19,14 @@ pub enum RawToken { impl RawToken { pub fn type_name(&self) -> &'static str { match self { - RawToken::Number(_) => "Number", + RawToken::Number(_) => "number", RawToken::Operator(..) => "operator", - RawToken::String(_) => "String", + RawToken::String(_) => "string", RawToken::Variable(_) => "variable", RawToken::ExternalCommand(_) => "external command", RawToken::ExternalWord => "external word", RawToken::GlobPattern => "glob pattern", - RawToken::Bare => "String", + RawToken::Bare => "string", } } } @@ -72,7 +72,7 @@ impl Token { pub fn extract_number(&self) -> Option> { match self.item { - RawToken::Number(number) => Some((number).spanned(self.span)), + RawToken::Number(number) => Some(number.spanned(self.span)), _ => None, } }