holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Parser refactoring for improving pipelines (#7162)

Browse Source 
* Remove lite_parse file

* Add LiteElement to represent different pipeline elem types

* Add PipelineElement to Pipelines

* Remove lite_parse specific tests
This commit is contained in:
JT
2022-11-19 10:46:48 +13:00
committed by GitHub
parent bd30ea723e
commit 6454bf69aa
16 changed files with 1196 additions and 1099 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
crates/nu-parser/src/flatten.rs
View File

@ -1,4 +1,6 @@
use nu_protocol::ast::{Block, Expr, Expression, ImportPatternMember, PathMember, Pipeline};
use nu_protocol::ast::{
Block, Expr, Expression, ImportPatternMember, PathMember, Pipeline, PipelineElement,
};
use nu_protocol::DeclId;
use nu_protocol::{engine::StateWorkingSet, Span};
use std::fmt::{Display, Formatter, Result};
@ -495,13 +497,25 @@ pub fn flatten_expression(
}
}
pub fn flatten_pipeline_element(
working_set: &StateWorkingSet,
pipeline_element: &PipelineElement,
) -> Vec<(Span, FlatShape)> {
match pipeline_element {
PipelineElement::Expression(expr)
| PipelineElement::Redirect(expr)
| PipelineElement::And(expr)
| PipelineElement::Or(expr) => flatten_expression(working_set, expr),
}
}
pub fn flatten_pipeline(
working_set: &StateWorkingSet,
pipeline: &Pipeline,
) -> Vec<(Span, FlatShape)> {
let mut output = vec![];
for expr in &pipeline.expressions {
output.extend(flatten_expression(working_set, expr))
for expr in &pipeline.elements {
output.extend(flatten_pipeline_element(working_set, expr))
}
output
}

11
crates/nu-parser/src/lib.rs
View File

@ -3,22 +3,23 @@ mod errors;
mod flatten;
mod known_external;
mod lex;
mod lite_parse;
mod parse_keywords;
mod parser;
mod type_check;
pub use deparse::{escape_for_script_arg, escape_quote_string};
pub use errors::ParseError;
pub use flatten::{flatten_block, flatten_expression, flatten_pipeline, FlatShape};
pub use flatten::{
flatten_block, flatten_expression, flatten_pipeline, flatten_pipeline_element, FlatShape,
};
pub use known_external::KnownExternal;
pub use lex::{lex, Token, TokenContents};
pub use lite_parse::{lite_parse, LiteBlock};
pub use parse_keywords::*;
pub use parser::{
is_math_expression_like, parse, parse_block, parse_duration_bytes, parse_expression,
parse_external_call, trim_quotes, trim_quotes_str, unescape_unquote_string, Import,
is_math_expression_like, lite_parse, parse, parse_block, parse_duration_bytes,
parse_expression, parse_external_call, trim_quotes, trim_quotes_str, unescape_unquote_string,
Import, LiteBlock, LiteElement,
};
#[cfg(feature = "plugin")]

184
crates/nu-parser/src/lite_parse.rs
View File

@ -1,184 +0,0 @@
use crate::{ParseError, Token, TokenContents};
use nu_protocol::Span;
#[derive(Debug)]
pub struct LiteCommand {
pub comments: Vec<Span>,
pub parts: Vec<Span>,
}
impl Default for LiteCommand {
fn default() -> Self {
Self::new()
}
}
impl LiteCommand {
pub fn new() -> Self {
Self {
comments: vec![],
parts: vec![],
}
}
pub fn push(&mut self, span: Span) {
self.parts.push(span);
}
pub fn is_empty(&self) -> bool {
self.parts.is_empty()
}
}
#[derive(Debug)]
pub struct LitePipeline {
pub commands: Vec<LiteCommand>,
}
impl Default for LitePipeline {
fn default() -> Self {
Self::new()
}
}
impl LitePipeline {
pub fn new() -> Self {
Self { commands: vec![] }
}
pub fn push(&mut self, command: LiteCommand) {
self.commands.push(command);
}
pub fn is_empty(&self) -> bool {
self.commands.is_empty()
}
}
#[derive(Debug)]
pub struct LiteBlock {
pub block: Vec<LitePipeline>,
}
impl Default for LiteBlock {
fn default() -> Self {
Self::new()
}
}
impl LiteBlock {
pub fn new() -> Self {
Self { block: vec![] }
}
pub fn push(&mut self, pipeline: LitePipeline) {
self.block.push(pipeline);
}
pub fn is_empty(&self) -> bool {
self.block.is_empty()
}
}
pub fn lite_parse(tokens: &[Token]) -> (LiteBlock, Option<ParseError>) {
let mut block = LiteBlock::new();
let mut curr_pipeline = LitePipeline::new();
let mut curr_command = LiteCommand::new();
let mut last_token = TokenContents::Eol;
let mut curr_comment: Option<Vec<Span>> = None;
for token in tokens.iter() {
match &token.contents {
TokenContents::Item => {
// If we have a comment, go ahead and attach it
if let Some(curr_comment) = curr_comment.take() {
curr_command.comments = curr_comment;
}
curr_command.push(token.span);
last_token = TokenContents::Item;
}
TokenContents::Pipe => {
if !curr_command.is_empty() {
curr_pipeline.push(curr_command);
curr_command = LiteCommand::new();
}
last_token = TokenContents::Pipe;
}
TokenContents::Eol => {
if last_token != TokenContents::Pipe {
if !curr_command.is_empty() {
curr_pipeline.push(curr_command);
curr_command = LiteCommand::new();
}
if !curr_pipeline.is_empty() {
block.push(curr_pipeline);
curr_pipeline = LitePipeline::new();
}
}
if last_token == TokenContents::Eol {
// Clear out the comment as we're entering a new comment
curr_comment = None;
}
last_token = TokenContents::Eol;
}
TokenContents::Semicolon => {
if !curr_command.is_empty() {
curr_pipeline.push(curr_command);
curr_command = LiteCommand::new();
}
if !curr_pipeline.is_empty() {
block.push(curr_pipeline);
curr_pipeline = LitePipeline::new();
}
last_token = TokenContents::Semicolon;
}
TokenContents::Comment => {
// Comment is beside something
if last_token != TokenContents::Eol {
curr_command.comments.push(token.span);
curr_comment = None;
} else {
// Comment precedes something
if let Some(curr_comment) = &mut curr_comment {
curr_comment.push(token.span);
} else {
curr_comment = Some(vec![token.span]);
}
}
last_token = TokenContents::Comment;
}
}
}
if !curr_command.is_empty() {
curr_pipeline.push(curr_command);
}
if !curr_pipeline.is_empty() {
block.push(curr_pipeline);
}
if last_token == TokenContents::Pipe {
(
block,
Some(ParseError::UnexpectedEof(
"pipeline missing end".into(),
tokens[tokens.len() - 1].span,
)),
)
} else {
(block, None)
}
}

209
crates/nu-parser/src/parse_keywords.rs
View File

@ -2,7 +2,7 @@ use nu_path::canonicalize_with;
use nu_protocol::{
ast::{
Argument, Block, Call, Expr, Expression, ImportPattern, ImportPatternHead,
ImportPatternMember, PathMember, Pipeline,
ImportPatternMember, PathMember, Pipeline, PipelineElement,
},
engine::{StateWorkingSet, DEFAULT_OVERLAY_NAME},
span, BlockId, Exportable, Module, PositionalArg, Span, Spanned, SyntaxShape, Type,
@ -16,12 +16,11 @@ static PLUGIN_DIRS_ENV: &str = "NU_PLUGIN_DIRS";
use crate::{
known_external::KnownExternal,
lex, lite_parse,
lite_parse::LiteCommand,
lex,
parser::{
check_call, check_name, garbage, garbage_pipeline, parse, parse_internal_call,
check_call, check_name, garbage, garbage_pipeline, lite_parse, parse, parse_internal_call,
parse_multispan_value, parse_signature, parse_string, parse_var_with_opt_type, trim_quotes,
ParsedInternalCall,
LiteCommand, LiteElement, ParsedInternalCall,
},
unescape_unquote_string, ParseError,
};
@ -866,10 +865,10 @@ pub fn parse_export_in_module(
};
// Trying to warp the 'def' call into the 'export def' in a very clumsy way
if let Some(Expression {
if let Some(PipelineElement::Expression(Expression {
expr: Expr::Call(ref def_call),
..
}) = pipeline.expressions.get(0)
})) = pipeline.elements.get(0)
{
call = def_call.clone();
@ -931,10 +930,10 @@ pub fn parse_export_in_module(
};
// Trying to warp the 'def' call into the 'export def' in a very clumsy way
if let Some(Expression {
if let Some(PipelineElement::Expression(Expression {
expr: Expr::Call(ref def_call),
..
}) = pipeline.expressions.get(0)
})) = pipeline.elements.get(0)
{
call = def_call.clone();
@ -997,10 +996,10 @@ pub fn parse_export_in_module(
};
// Trying to warp the 'def' call into the 'export def' in a very clumsy way
if let Some(Expression {
if let Some(PipelineElement::Expression(Expression {
expr: Expr::Call(ref def_call),
..
}) = pipeline.expressions.get(0)
})) = pipeline.elements.get(0)
{
call = def_call.clone();
@ -1063,10 +1062,10 @@ pub fn parse_export_in_module(
};
// Trying to warp the 'alias' call into the 'export alias' in a very clumsy way
if let Some(Expression {
if let Some(PipelineElement::Expression(Expression {
expr: Expr::Call(ref alias_call),
..
}) = pipeline.expressions.get(0)
})) = pipeline.elements.get(0)
{
call = alias_call.clone();
@ -1129,10 +1128,10 @@ pub fn parse_export_in_module(
};
// Trying to warp the 'use' call into the 'export use' in a very clumsy way
if let Some(Expression {
if let Some(PipelineElement::Expression(Expression {
expr: Expr::Call(ref use_call),
..
}) = pipeline.expressions.get(0)
})) = pipeline.elements.get(0)
{
call = use_call.clone();
@ -1314,11 +1313,9 @@ pub fn parse_module_block(
for pipeline in &output.block {
if pipeline.commands.len() == 1 {
parse_def_predecl(
working_set,
&pipeline.commands[0].parts,
expand_aliases_denylist,
);
if let LiteElement::Command(command) = &pipeline.commands[0] {
parse_def_predecl(working_set, &command.parts, expand_aliases_denylist);
}
}
}
@ -1329,91 +1326,91 @@ pub fn parse_module_block(
.iter()
.map(|pipeline| {
if pipeline.commands.len() == 1 {
let name = working_set.get_span_contents(pipeline.commands[0].parts[0]);
match &pipeline.commands[0] {
LiteElement::Command(command) => {
let name = working_set.get_span_contents(command.parts[0]);
let (pipeline, err) = match name {
b"def" | b"def-env" => {
let (pipeline, err) =
parse_def(working_set, &pipeline.commands[0], expand_aliases_denylist);
let (pipeline, err) = match name {
b"def" | b"def-env" => {
let (pipeline, err) =
parse_def(working_set, command, expand_aliases_denylist);
(pipeline, err)
}
b"extern" => {
let (pipeline, err) = parse_extern(
working_set,
&pipeline.commands[0],
expand_aliases_denylist,
);
(pipeline, err)
}
b"extern" => {
let (pipeline, err) =
parse_extern(working_set, command, expand_aliases_denylist);
(pipeline, err)
}
b"alias" => {
let (pipeline, err) = parse_alias(
working_set,
&pipeline.commands[0].parts,
expand_aliases_denylist,
);
(pipeline, err)
}
b"alias" => {
let (pipeline, err) = parse_alias(
working_set,
&command.parts,
expand_aliases_denylist,
);
(pipeline, err)
}
b"use" => {
let (pipeline, _, err) = parse_use(
working_set,
&pipeline.commands[0].parts,
expand_aliases_denylist,
);
(pipeline, err)
}
b"use" => {
let (pipeline, _, err) =
parse_use(working_set, &command.parts, expand_aliases_denylist);
(pipeline, err)
}
b"export" => {
let (pipe, exportables, err) = parse_export_in_module(
working_set,
&pipeline.commands[0],
expand_aliases_denylist,
);
(pipeline, err)
}
b"export" => {
let (pipe, exportables, err) = parse_export_in_module(
working_set,
command,
expand_aliases_denylist,
);
if err.is_none() {
for exportable in exportables {
match exportable {
Exportable::Decl { name, id } => {
module.add_decl(name, id);
}
Exportable::Alias { name, id } => {
module.add_alias(name, id);
if err.is_none() {
for exportable in exportables {
match exportable {
Exportable::Decl { name, id } => {
module.add_decl(name, id);
}
Exportable::Alias { name, id } => {
module.add_alias(name, id);
}
}
}
}
(pipe, err)
}
b"export-env" => {
let (pipe, maybe_env_block, err) = parse_export_env(
working_set,
&command.parts,
expand_aliases_denylist,
);
if let Some(block_id) = maybe_env_block {
module.add_env_block(block_id);
}
(pipe, err)
}
_ => (
garbage_pipeline(&command.parts),
Some(ParseError::ExpectedKeyword(
"def or export keyword".into(),
command.parts[0],
)),
),
};
if error.is_none() {
error = err;
}
(pipe, err)
pipeline
}
b"export-env" => {
let (pipe, maybe_env_block, err) = parse_export_env(
working_set,
&pipeline.commands[0].parts,
expand_aliases_denylist,
);
if let Some(block_id) = maybe_env_block {
module.add_env_block(block_id);
}
(pipe, err)
}
_ => (
garbage_pipeline(&pipeline.commands[0].parts),
Some(ParseError::ExpectedKeyword(
"def or export keyword".into(),
pipeline.commands[0].parts[0],
)),
),
};
if error.is_none() {
error = err;
LiteElement::Or(command)
| LiteElement::And(command)
| LiteElement::Redirection(command) => (garbage_pipeline(&command.parts)),
}
pipeline
} else {
error = Some(ParseError::Expected("not a pipeline".into(), span));
garbage_pipeline(&[span])
@ -2841,14 +2838,12 @@ pub fn parse_let(
);
return (
Pipeline {
expressions: vec![Expression {
expr: Expr::Call(call),
span: nu_protocol::span(spans),
ty: output,
custom_completion: None,
}],
},
Pipeline::from_vec(vec![Expression {
expr: Expr::Call(call),
span: nu_protocol::span(spans),
ty: output,
custom_completion: None,
}]),
err,
);
}
@ -2964,14 +2959,12 @@ pub fn parse_mut(
);
return (
Pipeline {
expressions: vec![Expression {
expr: Expr::Call(call),
span: nu_protocol::span(spans),
ty: output,
custom_completion: None,
}],
},
Pipeline::from_vec(vec![Expression {
expr: Expr::Call(call),
span: nu_protocol::span(spans),
ty: output,
custom_completion: None,
}]),
err,
);
}

707
crates/nu-parser/src/parser.rs
View File

@ -1,16 +1,14 @@
use crate::{
lex, lite_parse,
lite_parse::LiteCommand,
parse_mut,
lex, parse_mut,
type_check::{math_result_type, type_compatible},
LiteBlock, ParseError, Token, TokenContents,
ParseError, Token, TokenContents,
};
use nu_protocol::{
ast::{
Argument, Assignment, Bits, Block, Boolean, Call, CellPath, Comparison, Expr, Expression,
FullCellPath, ImportPattern, ImportPatternHead, ImportPatternMember, Math, Operator,
PathMember, Pipeline, RangeInclusion, RangeOperator,
PathMember, Pipeline, PipelineElement, RangeInclusion, RangeOperator,
},
engine::StateWorkingSet,
span, BlockId, Flag, PositionalArg, Signature, Span, Spanned, SyntaxShape, Type, Unit, VarId,
@ -1055,11 +1053,16 @@ pub fn parse_call(
let (mut result, err) =
parse_builtin_commands(working_set, &lite_command, &expand_aliases_denylist);
let mut result = result.expressions.remove(0);
let result = result.elements.remove(0);
result.replace_span(working_set, expansion_span, orig_span);
// If this is the first element in a pipeline, we know it has to be an expression
if let PipelineElement::Expression(mut result) = result {
result.replace_span(working_set, expansion_span, orig_span);
return (result, err);
return (result, err);
} else {
panic!("Internal error: first element of pipeline not an expression")
}
}
}
@ -1908,9 +1911,6 @@ pub fn parse_full_cell_path(
let (output, err) = lex(source, span.start, &[b'\n', b'\r'], &[], true);
error = error.or(err);
let (output, err) = lite_parse(&output);
error = error.or(err);
// Creating a Type scope to parse the new block. This will keep track of
// the previous input type found in that block
let (output, err) =
@ -1922,9 +1922,19 @@ pub fn parse_full_cell_path(
let ty = output
.pipelines
.last()
.and_then(|Pipeline { expressions, .. }| expressions.last())
.map(|expr| match expr.expr {
Expr::BinaryOp(..) => expr.ty.clone(),
.and_then(|Pipeline { elements, .. }| elements.last())
.map(|element| match element {
PipelineElement::Expression(expr)
if matches!(
expr,
Expression {
expr: Expr::BinaryOp(..),
..
}
) =>
{
expr.ty.clone()
}
_ => working_set.type_scope.get_last_output(),
})
.unwrap_or_else(|| working_set.type_scope.get_last_output());
@ -3059,7 +3069,9 @@ pub fn parse_row_condition(
// We have an expression, so let's convert this into a block.
let mut block = Block::new();
let mut pipeline = Pipeline::new();
pipeline.expressions.push(expression);
pipeline
.elements
.push(PipelineElement::Expression(expression));
block.pipelines.push(pipeline);
@ -3709,27 +3721,29 @@ pub fn parse_list_expression(
for arg in &output.block[0].commands {
let mut spans_idx = 0;
while spans_idx < arg.parts.len() {
let (arg, err) = parse_multispan_value(
working_set,
&arg.parts,
&mut spans_idx,
element_shape,
expand_aliases_denylist,
);
error = error.or(err);
if let LiteElement::Command(command) = arg {
while spans_idx < command.parts.len() {
let (arg, err) = parse_multispan_value(
working_set,
&command.parts,
&mut spans_idx,
element_shape,
expand_aliases_denylist,
);
error = error.or(err);
if let Some(ref ctype) = contained_type {
if *ctype != arg.ty {
contained_type = Some(Type::Any);
if let Some(ref ctype) = contained_type {
if *ctype != arg.ty {
contained_type = Some(Type::Any);
}
} else {
contained_type = Some(arg.ty.clone());
}
} else {
contained_type = Some(arg.ty.clone());
args.push(arg);
spans_idx += 1;
}
args.push(arg);
spans_idx += 1;
}
}
}
@ -3799,68 +3813,84 @@ pub fn parse_table_expression(
)
}
_ => {
let mut table_headers = vec![];
match &output.block[0].commands[0] {
LiteElement::Command(command)
| LiteElement::Redirection(command)
| LiteElement::And(command)
| LiteElement::Or(command) => {
let mut table_headers = vec![];
let (headers, err) = parse_value(
working_set,
output.block[0].commands[0].parts[0],
&SyntaxShape::List(Box::new(SyntaxShape::Any)),
expand_aliases_denylist,
);
error = error.or(err);
let (headers, err) = parse_value(
working_set,
command.parts[0],
&SyntaxShape::List(Box::new(SyntaxShape::Any)),
expand_aliases_denylist,
);
error = error.or(err);
if let Expression {
expr: Expr::List(headers),
..
} = headers
{
table_headers = headers;
}
let mut rows = vec![];
for part in &output.block[1].commands[0].parts {
let (values, err) = parse_value(
working_set,
*part,
&SyntaxShape::List(Box::new(SyntaxShape::Any)),
expand_aliases_denylist,
);
error = error.or(err);
if let Expression {
expr: Expr::List(values),
span,
..
} = values
{
match values.len().cmp(&table_headers.len()) {
std::cmp::Ordering::Less => {
error = error
.or(Some(ParseError::MissingColumns(table_headers.len(), span)))
}
std::cmp::Ordering::Equal => {}
std::cmp::Ordering::Greater => {
error = error.or_else(|| {
Some(ParseError::ExtraColumns(
table_headers.len(),
values[table_headers.len()].span,
))
})
}
if let Expression {
expr: Expr::List(headers),
..
} = headers
{
table_headers = headers;
}
rows.push(values);
match &output.block[1].commands[0] {
LiteElement::Command(command)
| LiteElement::Redirection(command)
| LiteElement::And(command)
| LiteElement::Or(command) => {
let mut rows = vec![];
for part in &command.parts {
let (values, err) = parse_value(
working_set,
*part,
&SyntaxShape::List(Box::new(SyntaxShape::Any)),
expand_aliases_denylist,
);
error = error.or(err);
if let Expression {
expr: Expr::List(values),
span,
..
} = values
{
match values.len().cmp(&table_headers.len()) {
std::cmp::Ordering::Less => {
error = error.or(Some(ParseError::MissingColumns(
table_headers.len(),
span,
)))
}
std::cmp::Ordering::Equal => {}
std::cmp::Ordering::Greater => {
error = error.or_else(|| {
Some(ParseError::ExtraColumns(
table_headers.len(),
values[table_headers.len()].span,
))
})
}
}
rows.push(values);
}
}
(
Expression {
expr: Expr::Table(table_headers, rows),
span: original_span,
ty: Type::Table(vec![]), //FIXME
custom_completion: None,
},
error,
)
}
}
}
}
(
Expression {
expr: Expr::Table(table_headers, rows),
span: original_span,
ty: Type::Table(vec![]), //FIXME
custom_completion: None,
},
error,
)
}
}
}
@ -3919,9 +3949,6 @@ pub fn parse_block_expression(
_ => (None, 0),
};
let (output, err) = lite_parse(&output[amt_to_skip..]);
error = error.or(err);
// TODO: Finish this
if let SyntaxShape::Closure(Some(v)) = shape {
if let Some((sig, sig_span)) = &signature {
@ -3955,8 +3982,13 @@ pub fn parse_block_expression(
}
}
let (mut output, err) =
parse_block(working_set, &output, false, expand_aliases_denylist, false);
let (mut output, err) = parse_block(
working_set,
&output[amt_to_skip..],
false,
expand_aliases_denylist,
false,
);
error = error.or(err);
if let Some(signature) = signature {
@ -4088,9 +4120,6 @@ pub fn parse_closure_expression(
_ => (None, 0),
};
let (output, err) = lite_parse(&output[amt_to_skip..]);
error = error.or(err);
// TODO: Finish this
if let SyntaxShape::Closure(Some(v)) = shape {
if let Some((sig, sig_span)) = &signature {
@ -4124,8 +4153,13 @@ pub fn parse_closure_expression(
}
}
let (mut output, err) =
parse_block(working_set, &output, false, expand_aliases_denylist, false);
let (mut output, err) = parse_block(
working_set,
&output[amt_to_skip..],
false,
expand_aliases_denylist,
false,
);
error = error.or(err);
if let Some(signature) = signature {
@ -4926,9 +4960,7 @@ pub fn parse_expression(
if let Some(decl_id) = with_env {
let mut block = Block::default();
let ty = output.ty.clone();
block.pipelines = vec![Pipeline {
expressions: vec![output],
}];
block.pipelines = vec![Pipeline::from_vec(vec![output])];
let block_id = working_set.add_block(block);
@ -5130,11 +5162,16 @@ pub fn parse_record(
pub fn parse_block(
working_set: &mut StateWorkingSet,
lite_block: &LiteBlock,
tokens: &[Token],
scoped: bool,
expand_aliases_denylist: &[usize],
is_subexpression: bool,
) -> (Block, Option<ParseError>) {
let mut error = None;
let (lite_block, err) = lite_parse(tokens);
error = error.or(err);
trace!("parsing block: {:?}", lite_block);
if scoped {
@ -5142,18 +5179,21 @@ pub fn parse_block(
}
working_set.type_scope.enter_scope();
let mut error = None;
// Pre-declare any definition so that definitions
// that share the same block can see each other
for pipeline in &lite_block.block {
if pipeline.commands.len() == 1 {
if let Some(err) = parse_def_predecl(
working_set,
&pipeline.commands[0].parts,
expand_aliases_denylist,
) {
error = error.or(Some(err));
match &pipeline.commands[0] {
LiteElement::Command(command)
| LiteElement::Redirection(command)
| LiteElement::And(command)
| LiteElement::Or(command) => {
if let Some(err) =
parse_def_predecl(working_set, &command.parts, expand_aliases_denylist)
{
error = error.or(Some(err));
}
}
}
}
}
@ -5167,88 +5207,146 @@ pub fn parse_block(
let mut output = pipeline
.commands
.iter()
.map(|command| {
let (expr, err) =
parse_expression(working_set, &command.parts, expand_aliases_denylist);
.map(|command| match command {
LiteElement::Command(command) => {
let (expr, err) = parse_expression(
working_set,
&command.parts,
expand_aliases_denylist,
);
working_set.type_scope.add_type(expr.ty.clone());
working_set.type_scope.add_type(expr.ty.clone());
if error.is_none() {
error = err;
}
PipelineElement::Expression(expr)
}
LiteElement::Redirection(command) => {
let (expr, err) = parse_expression(
working_set,
&command.parts,
expand_aliases_denylist,
);
working_set.type_scope.add_type(expr.ty.clone());
if error.is_none() {
error = err;
}
PipelineElement::Redirect(expr)
}
LiteElement::And(command) => {
let (expr, err) = parse_expression(
working_set,
&command.parts,
expand_aliases_denylist,
);
working_set.type_scope.add_type(expr.ty.clone());
if error.is_none() {
error = err;
}
PipelineElement::And(expr)
}
LiteElement::Or(command) => {
let (expr, err) = parse_expression(
working_set,
&command.parts,
expand_aliases_denylist,
);
working_set.type_scope.add_type(expr.ty.clone());
if error.is_none() {
error = err;
}
PipelineElement::Or(expr)
}
})
.collect::<Vec<PipelineElement>>();
if is_subexpression {
for element in output.iter_mut().skip(1) {
if element.has_in_variable(working_set) {
*element = wrap_element_with_collect(working_set, element);
}
}
} else {
for element in output.iter_mut() {
if element.has_in_variable(working_set) {
*element = wrap_element_with_collect(working_set, element);
}
}
}
Pipeline { elements: output }
} else {
match &pipeline.commands[0] {
LiteElement::Command(command)
| LiteElement::Redirection(command)
| LiteElement::And(command)
| LiteElement::Or(command) => {
let (mut pipeline, err) =
parse_builtin_commands(working_set, command, expand_aliases_denylist);
if idx == 0 {
if let Some(let_decl_id) = working_set.find_decl(b"let", &Type::Any) {
if let Some(let_env_decl_id) =
working_set.find_decl(b"let-env", &Type::Any)
{
for element in pipeline.elements.iter_mut() {
if let PipelineElement::Expression(Expression {
expr: Expr::Call(call),
..
}) = element
{
if call.decl_id == let_decl_id
|| call.decl_id == let_env_decl_id
{
// Do an expansion
if let Some(Expression {
expr: Expr::Keyword(_, _, expr),
..
}) = call.positional_iter_mut().nth(1)
{
if expr.has_in_variable(working_set) {
*expr = Box::new(wrap_expr_with_collect(
working_set,
expr,
));
}
}
continue;
} else if element.has_in_variable(working_set)
&& !is_subexpression
{
*element =
wrap_element_with_collect(working_set, element);
}
} else if element.has_in_variable(working_set)
&& !is_subexpression
{
*element =
wrap_element_with_collect(working_set, element);
}
}
}
}
}
if error.is_none() {
error = err;
}
expr
})
.collect::<Vec<Expression>>();
if is_subexpression {
for expr in output.iter_mut().skip(1) {
if expr.has_in_variable(working_set) {
*expr = wrap_expr_with_collect(working_set, expr);
}
}
} else {
for expr in output.iter_mut() {
if expr.has_in_variable(working_set) {
*expr = wrap_expr_with_collect(working_set, expr);
}
pipeline
}
}
Pipeline {
expressions: output,
}
} else {
let (mut pipeline, err) = parse_builtin_commands(
working_set,
&pipeline.commands[0],
expand_aliases_denylist,
);
if idx == 0 {
if let Some(let_decl_id) = working_set.find_decl(b"let", &Type::Any) {
if let Some(let_env_decl_id) = working_set.find_decl(b"let-env", &Type::Any)
{
for expr in pipeline.expressions.iter_mut() {
if let Expression {
expr: Expr::Call(call),
..
} = expr
{
if call.decl_id == let_decl_id
|| call.decl_id == let_env_decl_id
{
// Do an expansion
if let Some(Expression {
expr: Expr::Keyword(_, _, expr),
..
}) = call.positional_iter_mut().nth(1)
{
if expr.has_in_variable(working_set) {
*expr = Box::new(wrap_expr_with_collect(
working_set,
expr,
));
}
}
continue;
} else if expr.has_in_variable(working_set) && !is_subexpression
{
*expr = wrap_expr_with_collect(working_set, expr);
}
} else if expr.has_in_variable(working_set) && !is_subexpression {
*expr = wrap_expr_with_collect(working_set, expr);
}
}
}
}
}
if error.is_none() {
error = err;
}
pipeline
}
})
.into();
@ -5306,14 +5404,32 @@ fn discover_captures_in_pipeline(
seen_blocks: &mut HashMap<BlockId, Vec<(VarId, Span)>>,
) -> Result<Vec<(VarId, Span)>, ParseError> {
let mut output = vec![];
for expr in &pipeline.expressions {
let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?;
for element in &pipeline.elements {
let result =
discover_captures_in_pipeline_element(working_set, element, seen, seen_blocks)?;
output.extend(&result);
}
Ok(output)
}
// Closes over captured variables
pub fn discover_captures_in_pipeline_element(
working_set: &StateWorkingSet,
element: &PipelineElement,
seen: &mut Vec<VarId>,
seen_blocks: &mut HashMap<BlockId, Vec<(VarId, Span)>>,
) -> Result<Vec<(VarId, Span)>, ParseError> {
match element {
PipelineElement::Expression(expression)
| PipelineElement::Redirect(expression)
| PipelineElement::And(expression)
| PipelineElement::Or(expression) => {
discover_captures_in_expr(working_set, expression, seen, seen_blocks)
}
}
}
// Closes over captured variables
pub fn discover_captures_in_expr(
working_set: &StateWorkingSet,
@ -5553,6 +5669,26 @@ pub fn discover_captures_in_expr(
Ok(output)
}
fn wrap_element_with_collect(
working_set: &mut StateWorkingSet,
element: &PipelineElement,
) -> PipelineElement {
match element {
PipelineElement::Expression(expression) => {
PipelineElement::Expression(wrap_expr_with_collect(working_set, expression))
}
PipelineElement::Redirect(expression) => {
PipelineElement::Redirect(wrap_expr_with_collect(working_set, expression))
}
PipelineElement::And(expression) => {
PipelineElement::And(wrap_expr_with_collect(working_set, expression))
}
PipelineElement::Or(expression) => {
PipelineElement::Or(wrap_expr_with_collect(working_set, expression))
}
}
}
fn wrap_expr_with_collect(working_set: &mut StateWorkingSet, expr: &Expression) -> Expression {
let span = expr.span;
@ -5573,9 +5709,7 @@ fn wrap_expr_with_collect(working_set: &mut StateWorkingSet, expr: &Expression)
expr.replace_in_variable(working_set, var_id);
let block = Block {
pipelines: vec![Pipeline {
expressions: vec![expr],
}],
pipelines: vec![Pipeline::from_vec(vec![expr])],
signature: Box::new(signature),
..Default::default()
};
@ -5618,6 +5752,196 @@ fn wrap_expr_with_collect(working_set: &mut StateWorkingSet, expr: &Expression)
}
}
#[derive(Debug)]
pub struct LiteCommand {
pub comments: Vec<Span>,
pub parts: Vec<Span>,
}
impl Default for LiteCommand {
fn default() -> Self {
Self::new()
}
}
impl LiteCommand {
pub fn new() -> Self {
Self {
comments: vec![],
parts: vec![],
}
}
pub fn push(&mut self, span: Span) {
self.parts.push(span);
}
pub fn is_empty(&self) -> bool {
self.parts.is_empty()
}
}
#[derive(Debug)]
pub enum LiteElement {
Command(LiteCommand),
Redirection(LiteCommand),
And(LiteCommand),
Or(LiteCommand),
}
#[derive(Debug)]
pub struct LitePipeline {
pub commands: Vec<LiteElement>,
}
impl Default for LitePipeline {
fn default() -> Self {
Self::new()
}
}
impl LitePipeline {
pub fn new() -> Self {
Self { commands: vec![] }
}
pub fn push(&mut self, element: LiteElement) {
self.commands.push(element);
}
pub fn is_empty(&self) -> bool {
self.commands.is_empty()
}
}
#[derive(Debug)]
pub struct LiteBlock {
pub block: Vec<LitePipeline>,
}
impl Default for LiteBlock {
fn default() -> Self {
Self::new()
}
}
impl LiteBlock {
pub fn new() -> Self {
Self { block: vec![] }
}
pub fn push(&mut self, pipeline: LitePipeline) {
self.block.push(pipeline);
}
pub fn is_empty(&self) -> bool {
self.block.is_empty()
}
}
pub fn lite_parse(tokens: &[Token]) -> (LiteBlock, Option<ParseError>) {
let mut block = LiteBlock::new();
let mut curr_pipeline = LitePipeline::new();
let mut curr_command = LiteCommand::new();
let mut last_token = TokenContents::Eol;
let mut curr_comment: Option<Vec<Span>> = None;
for token in tokens.iter() {
match &token.contents {
TokenContents::Item => {
// If we have a comment, go ahead and attach it
if let Some(curr_comment) = curr_comment.take() {
curr_command.comments = curr_comment;
}
curr_command.push(token.span);
last_token = TokenContents::Item;
}
TokenContents::Pipe => {
if !curr_command.is_empty() {
curr_pipeline.push(LiteElement::Command(curr_command));
curr_command = LiteCommand::new();
}
last_token = TokenContents::Pipe;
}
TokenContents::Eol => {
if last_token != TokenContents::Pipe {
if !curr_command.is_empty() {
curr_pipeline.push(LiteElement::Command(curr_command));
curr_command = LiteCommand::new();
}
if !curr_pipeline.is_empty() {
block.push(curr_pipeline);
curr_pipeline = LitePipeline::new();
}
}
if last_token == TokenContents::Eol {
// Clear out the comment as we're entering a new comment
curr_comment = None;
}
last_token = TokenContents::Eol;
}
TokenContents::Semicolon => {
if !curr_command.is_empty() {
curr_pipeline.push(LiteElement::Command(curr_command));
curr_command = LiteCommand::new();
}
if !curr_pipeline.is_empty() {
block.push(curr_pipeline);
curr_pipeline = LitePipeline::new();
}
last_token = TokenContents::Semicolon;
}
TokenContents::Comment => {
// Comment is beside something
if last_token != TokenContents::Eol {
curr_command.comments.push(token.span);
curr_comment = None;
} else {
// Comment precedes something
if let Some(curr_comment) = &mut curr_comment {
curr_comment.push(token.span);
} else {
curr_comment = Some(vec![token.span]);
}
}
last_token = TokenContents::Comment;
}
}
}
if !curr_command.is_empty() {
curr_pipeline.push(LiteElement::Command(curr_command));
}
if !curr_pipeline.is_empty() {
block.push(curr_pipeline);
}
if last_token == TokenContents::Pipe {
(
block,
Some(ParseError::UnexpectedEof(
"pipeline missing end".into(),
tokens[tokens.len() - 1].span,
)),
)
} else {
(block, None)
}
}
// Parses a vector of u8 to create an AST Block. If a file name is given, then
// the name is stored in the working set. When parsing a source without a file
// name, the source of bytes is stored as "source"
@ -5644,9 +5968,6 @@ pub fn parse(
let (output, err) = lex(contents, span_offset, &[], &[], false);
error = error.or(err);
let (output, err) = lite_parse(&output);
error = error.or(err);
let (mut output, err) =
parse_block(working_set, &output, scoped, expand_aliases_denylist, false);
error = error.or(err);