Parser refactoring for improving pipelines (#7162)

* Remove lite_parse file

* Add LiteElement to represent different pipeline elem types

* Add PipelineElement to Pipelines

* Remove lite_parse specific tests
This commit is contained in:
JT
2022-11-19 10:46:48 +13:00
committed by GitHub
parent bd30ea723e
commit 6454bf69aa
16 changed files with 1196 additions and 1099 deletions

View File

@ -1,242 +0,0 @@
use nu_parser::{lex, lite_parse, LiteBlock, ParseError};
use nu_protocol::Span;
fn lite_parse_helper(input: &[u8]) -> Result<LiteBlock, ParseError> {
let (output, err) = lex(input, 0, &[], &[], false);
if let Some(err) = err {
return Err(err);
}
let (output, err) = lite_parse(&output);
if let Some(err) = err {
return Err(err);
}
Ok(output)
}
#[test]
fn comment_before() -> Result<(), ParseError> {
// Code:
// # this is a comment
// def foo bar
let input = b"# this is a comment\ndef foo bar";
let lite_block = lite_parse_helper(input)?;
assert_eq!(lite_block.block.len(), 1);
assert_eq!(lite_block.block[0].commands.len(), 1);
assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
assert_eq!(
lite_block.block[0].commands[0].comments[0],
Span { start: 0, end: 19 }
);
Ok(())
}
#[test]
fn comment_beside() -> Result<(), ParseError> {
// Code:
// def foo bar # this is a comment
let input = b"def foo bar # this is a comment";
let lite_block = lite_parse_helper(input)?;
assert_eq!(lite_block.block.len(), 1);
assert_eq!(lite_block.block[0].commands.len(), 1);
assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
assert_eq!(
lite_block.block[0].commands[0].comments[0],
Span { start: 12, end: 31 }
);
Ok(())
}
#[test]
fn comments_stack() -> Result<(), ParseError> {
// Code:
// # this is a comment
// # another comment
// # def foo bar
let input = b"# this is a comment\n# another comment\ndef foo bar ";
let lite_block = lite_parse_helper(input)?;
assert_eq!(lite_block.block.len(), 1);
assert_eq!(lite_block.block[0].commands[0].comments.len(), 2);
assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
assert_eq!(
lite_block.block[0].commands[0].comments[0],
Span { start: 0, end: 19 }
);
assert_eq!(
lite_block.block[0].commands[0].comments[1],
Span { start: 20, end: 37 }
);
Ok(())
}
#[test]
fn separated_comments_dont_stack() -> Result<(), ParseError> {
// Code:
// # this is a comment
//
// # another comment
// # def foo bar
let input = b"# this is a comment\n\n# another comment\ndef foo bar ";
let lite_block = lite_parse_helper(input)?;
assert_eq!(lite_block.block.len(), 1);
assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
assert_eq!(lite_block.block[0].commands[0].parts.len(), 3);
assert_eq!(
lite_block.block[0].commands[0].comments[0],
Span { start: 21, end: 38 }
);
Ok(())
}
#[test]
fn multiple_pipelines() -> Result<(), ParseError> {
// Code:
// # A comment
// let a = ( 3 + (
// 4 +
// 5 ))
// let b = 1 # comment
let input = b"# comment \n let a = ( 3 + (\n 4 + \n 5 )) \n let b = 1 # comment";
let lite_block = lite_parse_helper(input)?;
assert_eq!(lite_block.block.len(), 2);
assert_eq!(lite_block.block[0].commands[0].comments.len(), 1);
assert_eq!(lite_block.block[0].commands[0].parts.len(), 4);
assert_eq!(
lite_block.block[0].commands[0].comments[0],
Span { start: 0, end: 10 }
);
assert_eq!(lite_block.block[1].commands[0].comments.len(), 1);
assert_eq!(lite_block.block[1].commands[0].parts.len(), 4);
assert_eq!(
lite_block.block[1].commands[0].comments[0],
Span { start: 52, end: 61 }
);
Ok(())
}
#[test]
fn multiple_commands() -> Result<(), ParseError> {
// Pipes add commands to the lite parser
// Code:
// let a = ls | where name == 1
// let b = 1 # comment
let input = b"let a = ls | where name == 1 \n let b = 1 # comment";
let lite_block = lite_parse_helper(input)?;
assert_eq!(lite_block.block.len(), 2);
assert_eq!(lite_block.block[0].commands.len(), 2);
assert_eq!(lite_block.block[1].commands.len(), 1);
assert_eq!(
lite_block.block[1].commands[0].comments[0],
Span { start: 41, end: 50 }
);
Ok(())
}
#[test]
fn multiple_commands_with_comment() -> Result<(), ParseError> {
// Pipes add commands to the lite parser
// The comments are attached to the commands next to them
// Code:
// let a = ls | where name == 1 # comment
// let b = 1 # comment
//let a = ls | where name == 1 # comment \n let b = 1 # comment
let input = b"let a = ls | where name == 1 # comment\n let b = 1 # comment";
let lite_block = lite_parse_helper(input)?;
assert_eq!(lite_block.block.len(), 2);
assert_eq!(lite_block.block[0].commands.len(), 2);
assert_eq!(lite_block.block[1].commands.len(), 1);
assert_eq!(
lite_block.block[0].commands[1].comments[0],
Span { start: 29, end: 38 }
);
Ok(())
}
#[test]
fn multiple_commands_with_pipes() -> Result<(), ParseError> {
// The comments inside () get encapsulated in the whole item
// Code:
// # comment 1
// # comment 2
// let a = ( ls
// | where name =~ some # another comment
// | each { |file| rm file.name } # final comment
// )
// # comment A
// let b = 0;
let input = b"# comment 1
# comment 2
let a = ( ls
| where name =~ some # another comment
| each { |file| rm file.name }) # final comment
# comment A
let b = 0
";
let lite_block = lite_parse_helper(input)?;
assert_eq!(lite_block.block.len(), 2);
assert_eq!(lite_block.block[0].commands[0].comments.len(), 3);
assert_eq!(lite_block.block[0].commands[0].parts.len(), 4);
assert_eq!(
lite_block.block[0].commands[0].parts[3],
Span {
start: 32,
end: 107
}
);
assert_eq!(
lite_block.block[0].commands[0].comments[2],
Span {
start: 108,
end: 123
}
);
assert_eq!(lite_block.block[1].commands[0].comments.len(), 1);
assert_eq!(lite_block.block[1].commands[0].parts.len(), 4);
assert_eq!(
lite_block.block[1].commands[0].comments[0],
Span {
start: 124,
end: 135
}
);
Ok(())
}

View File

@ -1,7 +1,7 @@
use nu_parser::ParseError;
use nu_parser::*;
use nu_protocol::{
ast::{Expr, Expression},
ast::{Expr, Expression, PipelineElement},
engine::{Command, EngineState, Stack, StateWorkingSet},
Signature, SyntaxShape,
};
@ -54,10 +54,10 @@ pub fn parse_int() {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Int(3),
..
}
})
))
}
@ -72,7 +72,11 @@ pub fn parse_binary_with_hex_format() {
assert!(block.len() == 1);
let expressions = &block[0];
assert!(expressions.len() == 1);
assert_eq!(expressions[0].expr, Expr::Binary(vec![0x13]))
if let PipelineElement::Expression(expr) = &expressions[0] {
assert_eq!(expr.expr, Expr::Binary(vec![0x13]))
} else {
panic!("Not an expression")
}
}
#[test]
@ -86,7 +90,11 @@ pub fn parse_binary_with_incomplete_hex_format() {
assert!(block.len() == 1);
let expressions = &block[0];
assert!(expressions.len() == 1);
assert_eq!(expressions[0].expr, Expr::Binary(vec![0x03]))
if let PipelineElement::Expression(expr) = &expressions[0] {
assert_eq!(expr.expr, Expr::Binary(vec![0x03]))
} else {
panic!("Not an expression")
}
}
#[test]
@ -100,7 +108,11 @@ pub fn parse_binary_with_binary_format() {
assert!(block.len() == 1);
let expressions = &block[0];
assert!(expressions.len() == 1);
assert_eq!(expressions[0].expr, Expr::Binary(vec![0b10101000]))
if let PipelineElement::Expression(expr) = &expressions[0] {
assert_eq!(expr.expr, Expr::Binary(vec![0b10101000]))
} else {
panic!("Not an expression")
}
}
#[test]
@ -114,7 +126,11 @@ pub fn parse_binary_with_incomplete_binary_format() {
assert!(block.len() == 1);
let expressions = &block[0];
assert!(expressions.len() == 1);
assert_eq!(expressions[0].expr, Expr::Binary(vec![0b00000010]))
if let PipelineElement::Expression(expr) = &expressions[0] {
assert_eq!(expr.expr, Expr::Binary(vec![0b00000010]))
} else {
panic!("Not an expression")
}
}
#[test]
@ -128,7 +144,11 @@ pub fn parse_binary_with_octal_format() {
assert!(block.len() == 1);
let expressions = &block[0];
assert!(expressions.len() == 1);
assert_eq!(expressions[0].expr, Expr::Binary(vec![0o250]))
if let PipelineElement::Expression(expr) = &expressions[0] {
assert_eq!(expr.expr, Expr::Binary(vec![0o250]))
} else {
panic!("Not an expression")
}
}
#[test]
@ -142,7 +162,11 @@ pub fn parse_binary_with_incomplete_octal_format() {
assert!(block.len() == 1);
let expressions = &block[0];
assert!(expressions.len() == 1);
assert_eq!(expressions[0].expr, Expr::Binary(vec![0o2]))
if let PipelineElement::Expression(expr) = &expressions[0] {
assert_eq!(expr.expr, Expr::Binary(vec![0o2]))
} else {
panic!("Not an expression")
}
}
#[test]
@ -156,7 +180,11 @@ pub fn parse_binary_with_invalid_octal_format() {
assert!(block.len() == 1);
let expressions = &block[0];
assert!(expressions.len() == 1);
assert!(!matches!(&expressions[0].expr, Expr::Binary(_)))
if let PipelineElement::Expression(expr) = &expressions[0] {
assert!(!matches!(&expr.expr, Expr::Binary(_)))
} else {
panic!("Not an expression")
}
}
#[test]
@ -172,7 +200,11 @@ pub fn parse_binary_with_multi_byte_char() {
assert!(block.len() == 1);
let expressions = &block[0];
assert!(expressions.len() == 1);
assert!(!matches!(&expressions[0].expr, Expr::Binary(_)))
if let PipelineElement::Expression(expr) = &expressions[0] {
assert!(!matches!(&expr.expr, Expr::Binary(_)))
} else {
panic!("Not an expression")
}
}
#[test]
@ -186,10 +218,11 @@ pub fn parse_string() {
assert!(block.len() == 1);
let expressions = &block[0];
assert!(expressions.len() == 1);
assert_eq!(
expressions[0].expr,
Expr::String("hello nushell".to_string())
)
if let PipelineElement::Expression(expr) = &expressions[0] {
assert_eq!(expr.expr, Expr::String("hello nushell".to_string()))
} else {
panic!("Not an expression")
}
}
#[test]
@ -209,10 +242,11 @@ pub fn parse_escaped_string() {
assert!(block.len() == 1);
let expressions = &block[0];
assert!(expressions.len() == 1);
assert_eq!(
expressions[0].expr,
Expr::String("hello nushell".to_string())
)
if let PipelineElement::Expression(expr) = &expressions[0] {
assert_eq!(expr.expr, Expr::String("hello nushell".to_string()))
} else {
panic!("Not an expression")
}
}
#[test]
@ -231,10 +265,10 @@ pub fn parse_call() {
let expressions = &block[0];
assert_eq!(expressions.len(), 1);
if let Expression {
if let PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
} = &expressions[0]
}) = &expressions[0]
{
assert_eq!(call.decl_id, 0);
}
@ -337,10 +371,10 @@ fn test_nothing_comparisson_eq() {
assert!(expressions.len() == 1);
assert!(matches!(
&expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::BinaryOp(..),
..
}
})
))
}
@ -357,10 +391,10 @@ fn test_nothing_comparisson_neq() {
assert!(expressions.len() == 1);
assert!(matches!(
&expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::BinaryOp(..),
..
}
})
))
}
@ -382,7 +416,7 @@ mod range {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Range(
Some(_),
None,
@ -393,7 +427,7 @@ mod range {
}
),
..
}
})
))
}
@ -411,7 +445,7 @@ mod range {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Range(
Some(_),
None,
@ -422,7 +456,7 @@ mod range {
}
),
..
}
})
))
}
@ -440,7 +474,7 @@ mod range {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Range(
Some(_),
None,
@ -451,7 +485,7 @@ mod range {
}
),
..
}
})
))
}
@ -469,7 +503,7 @@ mod range {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Range(
Some(_),
None,
@ -480,7 +514,7 @@ mod range {
}
),
..
}
})
))
}
@ -500,7 +534,7 @@ mod range {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Range(
Some(_),
None,
@ -511,7 +545,7 @@ mod range {
}
),
..
}
})
))
}
@ -537,7 +571,7 @@ mod range {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Range(
Some(_),
None,
@ -548,7 +582,7 @@ mod range {
}
),
..
}
})
))
}
@ -566,7 +600,7 @@ mod range {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Range(
Some(_),
None,
@ -577,7 +611,7 @@ mod range {
}
),
..
}
})
))
}
@ -595,7 +629,7 @@ mod range {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Range(
None,
None,
@ -606,7 +640,7 @@ mod range {
}
),
..
}
})
))
}
@ -624,7 +658,7 @@ mod range {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Range(
Some(_),
None,
@ -635,7 +669,7 @@ mod range {
}
),
..
}
})
))
}
@ -653,7 +687,7 @@ mod range {
assert!(expressions.len() == 1);
assert!(matches!(
expressions[0],
Expression {
PipelineElement::Expression(Expression {
expr: Expr::Range(
Some(_),
Some(_),
@ -664,7 +698,7 @@ mod range {
}
),
..
}
})
))
}
@ -997,8 +1031,11 @@ mod input_types {
let expressions = &block[0];
assert!(expressions.len() == 3);
match &expressions[0].expr {
Expr::Call(call) => {
match &expressions[0] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set
.find_decl(b"ls", &Type::Any)
.expect("Error merging delta");
@ -1007,16 +1044,22 @@ mod input_types {
_ => panic!("Expected expression Call not found"),
}
match &expressions[1].expr {
Expr::Call(call) => {
match &expressions[1] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set.find_decl(b"to-custom", &Type::Any).unwrap();
assert_eq!(call.decl_id, expected_id)
}
_ => panic!("Expected expression Call not found"),
}
match &expressions[2].expr {
Expr::Call(call) => {
match &expressions[2] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set
.find_decl(b"group-by", &Type::Custom("custom".into()))
.unwrap();
@ -1041,8 +1084,11 @@ mod input_types {
assert!(block.len() == 3);
let expressions = &block[2];
match &expressions[1].expr {
Expr::Call(call) => {
match &expressions[1] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set
.find_decl(b"agg", &Type::Custom("custom".into()))
.unwrap();
@ -1066,8 +1112,11 @@ mod input_types {
assert!(block.len() == 2);
let expressions = &block[1];
match &expressions[1].expr {
Expr::Call(call) => {
match &expressions[1] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set
.find_decl(b"agg", &Type::Custom("custom".into()))
.unwrap();
@ -1092,8 +1141,11 @@ mod input_types {
assert!(block.len() == 3);
let expressions = &block[1];
match &expressions[1].expr {
Expr::Call(call) => {
match &expressions[1] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set.find_decl(b"to-custom", &Type::Any).unwrap();
assert_eq!(call.decl_id, expected_id)
}
@ -1101,8 +1153,11 @@ mod input_types {
}
let expressions = &block[2];
match &expressions[1].expr {
Expr::Call(call) => {
match &expressions[1] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set.find_decl(b"to-custom", &Type::Any).unwrap();
assert_eq!(call.decl_id, expected_id)
}
@ -1126,16 +1181,22 @@ mod input_types {
let expressions = &block[0];
assert!(expressions.len() == 2);
match &expressions[0].expr {
Expr::Call(call) => {
match &expressions[0] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set.find_decl(b"ls", &Type::Any).unwrap();
assert_eq!(call.decl_id, expected_id)
}
_ => panic!("Expected expression Call not found"),
}
match &expressions[1].expr {
Expr::Call(call) => {
match &expressions[1] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set.find_decl(b"group-by", &Type::Any).unwrap();
assert_eq!(call.decl_id, expected_id)
}
@ -1159,8 +1220,11 @@ mod input_types {
engine_state.merge_delta(delta).unwrap();
let expressions = &block[0];
match &expressions[3].expr {
Expr::Call(call) => {
match &expressions[3] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let arg = &call.arguments[0];
match arg {
Argument::Positional(a) => match &a.expr {
@ -1171,8 +1235,11 @@ mod input_types {
let expressions = &block[0];
assert!(expressions.len() == 2);
match &expressions[1].expr {
Expr::Call(call) => {
match &expressions[1] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let working_set = StateWorkingSet::new(&engine_state);
let expected_id =
working_set.find_decl(b"min", &Type::Any).unwrap();
@ -1206,8 +1273,11 @@ mod input_types {
assert!(block.len() == 1);
let expressions = &block[0];
match &expressions[2].expr {
Expr::Call(call) => {
match &expressions[2] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set
.find_decl(b"with-column", &Type::Custom("custom".into()))
.unwrap();
@ -1216,8 +1286,11 @@ mod input_types {
_ => panic!("Expected expression Call not found"),
}
match &expressions[3].expr {
Expr::Call(call) => {
match &expressions[3] {
PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) => {
let expected_id = working_set
.find_decl(b"collect", &Type::Custom("custom".into()))
.unwrap();