nushell/crates/nu-parser/src/parser.rs

6659 lines
230 KiB
Rust
Raw Normal View History

2021-09-02 10:25:22 +02:00
use crate::{
eval::{eval_constant, value_as_string},
allow lists to have type annotations (#8529) this pr refines #8270 and closes #8109 # description examples: the original syntax is okay ```nu def okay [nums: list] {} # the type of list will be list<any> ``` empty annotations are allowed in any variation the last two may be caught by a future formatter, but do not affect `nu` code currently ```nu def okay [nums: list<>] {} # okay def okay [nums: list< >] {} # weird but also okay def okay [nums: list< >] {} # also weird but okay ``` types are allowed (See [notes](#notes) below) ```nu def okay [nums: list<int>] {} # `test [a b c]` will throw an error def okay [nums: list< int > {} # any amount of space within the angle brackets is okay def err [nums: list <int>] {} # this is not okay, `nums` and `<int>` will be parsed as # two separate params, ``` nested annotations are allowed in many variations ```nu def okay [items: list<list<int>>] {} def okay [items: list<list>] {} ``` any unterminated annotation is caught ```nu Error: nu::parser::unexpected_eof × Unexpected end of code. ╭─[source:1:1] 1 │ def err [nums: list<int] {} · ▲ · ╰── expected closing > ╰──── ``` unknown types are flagged ```nu Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<str>] {} · ─┬─ · ╰── unknown type ╰──── Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<int, string>] {} · ─────┬───── · ╰── unknown type ╰──── ``` # notes the error message for mismatched types in not as intuitive ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ┬ · ╰── expected int ╰──── ``` it should be something like this ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ──┬── · ╰── expected list<int> ╰──── ``` this is currently not implemented
2023-03-24 12:54:06 +01:00
lex::{lex, lex_signature},
2022-12-22 12:41:44 +01:00
lite_parser::{lite_parse, LiteCommand, LiteElement},
parse_mut,
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
parse_patterns::{parse_match_pattern, parse_pattern},
2021-09-02 10:25:22 +02:00
type_check::{math_result_type, type_compatible},
ParseError, Token, TokenContents,
2021-08-17 01:00:00 +02:00
};
2021-07-02 03:42:25 +02:00
2021-09-02 03:29:43 +02:00
use nu_protocol::{
ast::{
Argument, Assignment, Bits, Block, Boolean, Call, CellPath, Comparison, Expr, Expression,
FullCellPath, ImportPattern, ImportPatternHead, ImportPatternMember, MatchPattern, Math,
Operator, PathMember, Pattern, Pipeline, PipelineElement, RangeInclusion, RangeOperator,
},
2021-09-02 20:21:37 +02:00
engine::StateWorkingSet,
span, BlockId, Flag, PositionalArg, Signature, Span, Spanned, SyntaxShape, Type, Unit, VarId,
ENV_VARIABLE_ID, IN_VARIABLE_ID,
2021-09-02 03:29:43 +02:00
};
2021-07-23 23:19:30 +02:00
2021-09-26 20:39:19 +02:00
use crate::parse_keywords::{
is_unaliasable_parser_keyword, parse_alias, parse_def, parse_def_predecl,
2023-03-10 22:20:31 +01:00
parse_export_in_block, parse_extern, parse_for, parse_hide, parse_keyword, parse_let_or_const,
parse_module, parse_old_alias, parse_overlay_hide, parse_overlay_new, parse_overlay_use,
parse_source, parse_use, parse_where, parse_where_expr,
2021-09-02 03:29:43 +02:00
};
2021-07-23 23:19:30 +02:00
2022-06-17 20:11:48 +02:00
use itertools::Itertools;
2022-01-01 22:42:50 +01:00
use log::trace;
2022-03-01 00:31:53 +01:00
use std::{
collections::{HashMap, HashSet},
num::ParseIntError,
};
2021-11-02 21:56:00 +01:00
#[cfg(feature = "plugin")]
use crate::parse_keywords::parse_register;
2021-11-02 21:56:00 +01:00
2021-09-26 20:39:19 +02:00
pub fn garbage(span: Span) -> Expression {
2021-07-01 02:01:04 +02:00
Expression::garbage(span)
}
pub fn garbage_pipeline(spans: &[Span]) -> Pipeline {
Pipeline::from_vec(vec![garbage(span(spans))])
2021-09-10 09:28:43 +02:00
}
2021-07-01 03:31:02 +02:00
fn is_identifier_byte(b: u8) -> bool {
b != b'.'
&& b != b'['
&& b != b'('
&& b != b'{'
&& b != b'+'
&& b != b'-'
&& b != b'*'
&& b != b'^'
&& b != b'/'
&& b != b'='
&& b != b'!'
&& b != b'<'
&& b != b'>'
&& b != b'&'
&& b != b'|'
2021-07-01 03:31:02 +02:00
}
pub fn is_math_expression_like(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
) -> bool {
let bytes = working_set.get_span_contents(span);
if bytes.is_empty() {
return false;
}
if bytes == b"true"
|| bytes == b"false"
|| bytes == b"null"
|| bytes == b"not"
|| bytes == b"if"
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
|| bytes == b"match"
{
return true;
}
let b = bytes[0];
if b == b'(' || b == b'{' || b == b'[' || b == b'$' || b == b'"' || b == b'\'' || b == b'-' {
return true;
}
if parse_number(bytes, span).1.is_none() {
return true;
}
if parse_filesize(working_set, span).1.is_none() {
return true;
}
if parse_duration(working_set, span).1.is_none() {
return true;
}
if parse_datetime(working_set, span).1.is_none() {
return true;
}
if parse_binary(working_set, span).1.is_none() {
return true;
}
if parse_range(working_set, span, expand_aliases_denylist)
.1
.is_none()
{
return true;
}
false
}
2021-07-01 03:31:02 +02:00
fn is_identifier(bytes: &[u8]) -> bool {
bytes.iter().all(|x| is_identifier_byte(*x))
}
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
pub fn is_variable(bytes: &[u8]) -> bool {
2021-07-01 03:31:02 +02:00
if bytes.len() > 1 && bytes[0] == b'$' {
is_identifier(&bytes[1..])
} else {
is_identifier(bytes)
}
}
2021-11-14 20:40:26 +01:00
pub fn trim_quotes(bytes: &[u8]) -> &[u8] {
if (bytes.starts_with(b"\"") && bytes.ends_with(b"\"") && bytes.len() > 1)
|| (bytes.starts_with(b"\'") && bytes.ends_with(b"\'") && bytes.len() > 1)
|| (bytes.starts_with(b"`") && bytes.ends_with(b"`") && bytes.len() > 1)
2021-11-14 20:40:26 +01:00
{
&bytes[1..(bytes.len() - 1)]
} else {
bytes
}
}
pub fn trim_quotes_str(s: &str) -> &str {
if (s.starts_with('"') && s.ends_with('"') && s.len() > 1)
|| (s.starts_with('\'') && s.ends_with('\'') && s.len() > 1)
|| (s.starts_with('`') && s.ends_with('`') && s.len() > 1)
{
&s[1..(s.len() - 1)]
} else {
s
}
}
pub fn check_call(command: Span, sig: &Signature, call: &Call) -> Option<ParseError> {
2021-10-13 19:53:27 +02:00
// Allow the call to pass if they pass in the help flag
if call.named_iter().any(|(n, _, _)| n.item == "help") {
2021-10-13 19:53:27 +02:00
return None;
}
if call.positional_len() < sig.required_positional.len() {
// Comparing the types of all signature positional arguments against the parsed
// expressions found in the call. If one type is not found then it could be assumed
// that that positional argument is missing from the parsed call
for argument in &sig.required_positional {
let found = call.positional_iter().fold(false, |ac, expr| {
if argument.shape.to_type() == expr.ty || argument.shape == SyntaxShape::Any {
true
} else {
ac
}
});
if !found {
if let Some(last) = call.positional_iter().last() {
2022-01-04 00:14:33 +01:00
return Some(ParseError::MissingPositional(
argument.name.clone(),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(last.span.end, last.span.end),
2022-01-04 00:14:33 +01:00
sig.call_signature(),
));
} else {
return Some(ParseError::MissingPositional(
argument.name.clone(),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(command.end, command.end),
2022-01-04 00:14:33 +01:00
sig.call_signature(),
));
}
}
}
let missing = &sig.required_positional[call.positional_len()];
if let Some(last) = call.positional_iter().last() {
2022-01-04 00:14:33 +01:00
Some(ParseError::MissingPositional(
missing.name.clone(),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(last.span.end, last.span.end),
2022-01-04 00:14:33 +01:00
sig.call_signature(),
))
} else {
Some(ParseError::MissingPositional(
missing.name.clone(),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(command.end, command.end),
2022-01-04 00:14:33 +01:00
sig.call_signature(),
))
}
2021-07-02 04:22:54 +02:00
} else {
for req_flag in sig.named.iter().filter(|x| x.required) {
if call.named_iter().all(|(n, _, _)| n.item != req_flag.long) {
2021-07-02 04:22:54 +02:00
return Some(ParseError::MissingRequiredFlag(
req_flag.long.clone(),
command,
));
}
}
None
}
}
2021-09-26 20:39:19 +02:00
pub fn check_name<'a>(
2021-09-02 10:25:22 +02:00
working_set: &mut StateWorkingSet,
2021-09-13 21:59:11 +02:00
spans: &'a [Span],
) -> Option<(&'a Span, ParseError)> {
let command_len = if !spans.is_empty() {
if working_set.get_span_contents(spans[0]) == b"export" {
2
} else {
1
}
} else {
return None;
};
2021-09-13 21:59:11 +02:00
if spans.len() == 1 {
None
} else if spans.len() < command_len + 3 {
if working_set.get_span_contents(spans[command_len]) == b"=" {
let name =
String::from_utf8_lossy(working_set.get_span_contents(span(&spans[..command_len])));
2021-09-13 21:59:11 +02:00
Some((
&spans[command_len],
2021-09-13 21:59:11 +02:00
ParseError::AssignmentMismatch(
format!("{name} missing name"),
2021-09-13 21:59:11 +02:00
"missing name".into(),
spans[command_len],
2021-09-13 21:59:11 +02:00
),
))
} else {
None
}
} else if working_set.get_span_contents(spans[command_len + 1]) != b"=" {
let name =
String::from_utf8_lossy(working_set.get_span_contents(span(&spans[..command_len])));
2021-09-13 21:59:11 +02:00
Some((
&spans[command_len + 1],
2021-09-13 21:59:11 +02:00
ParseError::AssignmentMismatch(
format!("{name} missing sign"),
2021-09-13 21:59:11 +02:00
"missing equal sign".into(),
spans[command_len + 1],
2021-09-13 21:59:11 +02:00
),
2021-09-10 09:28:43 +02:00
))
} else {
None
}
}
Re-implement aliases (#8123) # Description This PR adds an alternative alias implementation. Old aliases still work but you need to use `old-alias` instead of `alias`. Instead of replacing spans in the original code and re-parsing, which proved to be extremely error-prone and a constant source of panics, the new implementation creates a new command that references the old command. Consider the new alias defined as `alias ll = ls -l`. The parser creates a new command called `ll` and remembers that it is actually a `ls` command called with the `-l` flag. Then, when the parser sees the `ll` command, it will translate it to `ls -l` and passes to it any parameters that were passed to the call to `ll`. It works quite similar to how known externals defined with `extern` are implemented. The new alias implementation should work the same way as the old aliases, including exporting from modules, referencing both known and unknown externals. It seems to preserve custom completions and pipeline metadata. It is quite robust in most cases but there are some rough edges (see later). Fixes https://github.com/nushell/nushell/issues/7648, https://github.com/nushell/nushell/issues/8026, https://github.com/nushell/nushell/issues/7512, https://github.com/nushell/nushell/issues/5780, https://github.com/nushell/nushell/issues/7754 No effect: https://github.com/nushell/nushell/issues/8122 (we might revisit the completions code after this PR) Should use custom command instead: https://github.com/nushell/nushell/issues/6048 # User-Facing Changes Since aliases are now basically commands, it has some new implications: 1. `alias spam = "spam"` (requires command call) * **workaround**: use `alias spam = echo "spam"` 2. `def foo [] { 'foo' }; alias foo = ls -l` (foo defined more than once) * **workaround**: use different name (commands also have this limitation) 4. `alias ls = (ls | sort-by type name -i)` * **workaround**: Use custom command. _The common issue with this is that it is currently not easy to pass flags through custom commands and command referencing itself will lead to stack overflow. Both of these issues are meant to be addressed._ 5. TODO: Help messages, `which` command, `$nu.scope.aliases`, etc. * Should we treat the aliases as commands or should they be separated from regular commands? 6. Needs better error message and syntax highlight for recursed alias (`alias f = f`) 7. Can't create alias with the same name as existing command (`alias ls = ls -a`) * Might be possible to add support for it (not 100% sure) 8. Standalone `alias` doesn't list aliases anymore 9. Can't alias parser keywords (e.g., stuff like `alias ou = overlay use` won't work) * TODO: Needs a better error message when attempting to do so # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-02-27 08:44:05 +01:00
fn parse_external_arg(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
) -> (Expression, Option<ParseError>) {
let contents = working_set.get_span_contents(span);
let mut error = None;
if contents.starts_with(b"$") || contents.starts_with(b"(") {
let (arg, err) = parse_dollar_expr(working_set, span, expand_aliases_denylist);
error = error.or(err);
(arg, error)
} else if contents.starts_with(b"[") {
let (arg, err) = parse_list_expression(
working_set,
span,
&SyntaxShape::Any,
expand_aliases_denylist,
);
error = error.or(err);
(arg, error)
} else {
// Eval stage trims the quotes, so we don't have to do the same thing when parsing.
let contents = if contents.starts_with(b"\"") {
let (contents, err) = unescape_string(contents, span);
error = error.or(err);
String::from_utf8_lossy(&contents).to_string()
} else {
String::from_utf8_lossy(contents).to_string()
};
(
Expression {
expr: Expr::String(contents),
span,
ty: Type::String,
custom_completion: None,
},
error,
)
}
}
2021-09-02 10:25:22 +02:00
pub fn parse_external_call(
2021-10-08 23:51:47 +02:00
working_set: &mut StateWorkingSet,
2021-09-02 10:25:22 +02:00
spans: &[Span],
expand_aliases_denylist: &[usize],
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression: bool,
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
trace!("parse external");
2021-09-02 10:25:22 +02:00
let mut args = vec![];
let head_contents = working_set.get_span_contents(spans[0]);
let head_span = if head_contents.starts_with(b"^") {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(spans[0].start + 1, spans[0].end)
} else {
spans[0]
};
2022-03-03 20:05:55 +01:00
let head_contents = working_set.get_span_contents(head_span).to_vec();
2021-10-08 23:51:47 +02:00
let mut error = None;
let head = if head_contents.starts_with(b"$") || head_contents.starts_with(b"(") {
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
// the expression is inside external_call, so it's a subexpression
let (arg, err) = parse_expression(working_set, &[head_span], expand_aliases_denylist, true);
error = error.or(err);
Box::new(arg)
} else {
let (contents, err) = unescape_unquote_string(&head_contents, head_span);
error = error.or(err);
Box::new(Expression {
expr: Expr::String(contents),
span: head_span,
ty: Type::String,
custom_completion: None,
})
};
2021-09-02 10:25:22 +02:00
for span in &spans[1..] {
Re-implement aliases (#8123) # Description This PR adds an alternative alias implementation. Old aliases still work but you need to use `old-alias` instead of `alias`. Instead of replacing spans in the original code and re-parsing, which proved to be extremely error-prone and a constant source of panics, the new implementation creates a new command that references the old command. Consider the new alias defined as `alias ll = ls -l`. The parser creates a new command called `ll` and remembers that it is actually a `ls` command called with the `-l` flag. Then, when the parser sees the `ll` command, it will translate it to `ls -l` and passes to it any parameters that were passed to the call to `ll`. It works quite similar to how known externals defined with `extern` are implemented. The new alias implementation should work the same way as the old aliases, including exporting from modules, referencing both known and unknown externals. It seems to preserve custom completions and pipeline metadata. It is quite robust in most cases but there are some rough edges (see later). Fixes https://github.com/nushell/nushell/issues/7648, https://github.com/nushell/nushell/issues/8026, https://github.com/nushell/nushell/issues/7512, https://github.com/nushell/nushell/issues/5780, https://github.com/nushell/nushell/issues/7754 No effect: https://github.com/nushell/nushell/issues/8122 (we might revisit the completions code after this PR) Should use custom command instead: https://github.com/nushell/nushell/issues/6048 # User-Facing Changes Since aliases are now basically commands, it has some new implications: 1. `alias spam = "spam"` (requires command call) * **workaround**: use `alias spam = echo "spam"` 2. `def foo [] { 'foo' }; alias foo = ls -l` (foo defined more than once) * **workaround**: use different name (commands also have this limitation) 4. `alias ls = (ls | sort-by type name -i)` * **workaround**: Use custom command. _The common issue with this is that it is currently not easy to pass flags through custom commands and command referencing itself will lead to stack overflow. Both of these issues are meant to be addressed._ 5. TODO: Help messages, `which` command, `$nu.scope.aliases`, etc. * Should we treat the aliases as commands or should they be separated from regular commands? 6. Needs better error message and syntax highlight for recursed alias (`alias f = f`) 7. Can't create alias with the same name as existing command (`alias ls = ls -a`) * Might be possible to add support for it (not 100% sure) 8. Standalone `alias` doesn't list aliases anymore 9. Can't alias parser keywords (e.g., stuff like `alias ou = overlay use` won't work) * TODO: Needs a better error message when attempting to do so # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-02-27 08:44:05 +01:00
let (arg, err) = parse_external_arg(working_set, *span, expand_aliases_denylist);
error = error.or(err);
args.push(arg);
2021-07-01 02:01:04 +02:00
}
2021-09-02 10:25:22 +02:00
(
Expression {
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
expr: Expr::ExternalCall(head, args, is_subexpression),
2021-09-02 10:25:22 +02:00
span: span(spans),
ty: Type::Any,
custom_completion: None,
2021-09-02 10:25:22 +02:00
},
2021-10-08 23:51:47 +02:00
error,
2021-09-02 10:25:22 +02:00
)
}
2021-07-01 02:01:04 +02:00
2021-09-02 10:25:22 +02:00
fn parse_long_flag(
working_set: &mut StateWorkingSet,
spans: &[Span],
spans_idx: &mut usize,
sig: &Signature,
expand_aliases_denylist: &[usize],
) -> (
Option<Spanned<String>>,
Option<Expression>,
Option<ParseError>,
) {
2021-09-02 10:25:22 +02:00
let arg_span = spans[*spans_idx];
let arg_contents = working_set.get_span_contents(arg_span);
if arg_contents.starts_with(b"--") {
2021-10-12 19:44:23 +02:00
// FIXME: only use the first flag you find?
2021-09-02 10:25:22 +02:00
let split: Vec<_> = arg_contents.split(|x| *x == b'=').collect();
let long_name = String::from_utf8(split[0].into());
if let Ok(long_name) = long_name {
2021-10-13 19:53:27 +02:00
let long_name = long_name[2..].to_string();
2021-09-02 10:25:22 +02:00
if let Some(flag) = sig.get_long_flag(&long_name) {
if let Some(arg_shape) = &flag.arg {
if split.len() > 1 {
// and we also have the argument
let long_name_len = long_name.len();
2021-09-02 10:25:22 +02:00
let mut span = arg_span;
span.start += long_name_len + 3; //offset by long flag and '='
let (arg, err) =
parse_value(working_set, span, arg_shape, expand_aliases_denylist);
2021-09-02 10:25:22 +02:00
(
Some(Spanned {
item: long_name,
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
span: Span::new(arg_span.start, arg_span.start + long_name_len + 2),
}),
Some(arg),
err,
)
2021-09-02 10:25:22 +02:00
} else if let Some(arg) = spans.get(*spans_idx + 1) {
let (arg, err) =
parse_value(working_set, *arg, arg_shape, expand_aliases_denylist);
2021-09-02 10:25:22 +02:00
*spans_idx += 1;
(
Some(Spanned {
item: long_name,
span: arg_span,
}),
Some(arg),
err,
)
2021-07-08 22:29:00 +02:00
} else {
2021-09-02 10:25:22 +02:00
(
Some(Spanned {
item: long_name,
span: arg_span,
}),
2021-09-02 10:25:22 +02:00
None,
2022-01-04 00:14:33 +01:00
Some(ParseError::MissingFlagParam(
arg_shape.to_string(),
arg_span,
)),
2021-09-02 10:25:22 +02:00
)
2021-07-08 22:29:00 +02:00
}
} else {
2021-09-02 10:25:22 +02:00
// A flag with no argument
(
Some(Spanned {
item: long_name,
span: arg_span,
}),
None,
None,
)
2021-07-08 22:29:00 +02:00
}
} else {
2021-09-02 10:25:22 +02:00
(
Some(Spanned {
item: long_name.clone(),
span: arg_span,
}),
2021-09-02 10:25:22 +02:00
None,
2021-09-21 06:03:06 +02:00
Some(ParseError::UnknownFlag(
sig.name.clone(),
long_name.clone(),
arg_span,
sig.clone().formatted_flags(),
2021-09-21 06:03:06 +02:00
)),
2021-09-02 10:25:22 +02:00
)
2021-07-08 22:29:00 +02:00
}
} else {
(
Some(Spanned {
item: "--".into(),
span: arg_span,
}),
None,
Some(ParseError::NonUtf8(arg_span)),
)
2021-07-08 22:29:00 +02:00
}
2021-09-02 10:25:22 +02:00
} else {
(None, None, None)
2021-07-08 22:29:00 +02:00
}
2021-09-02 10:25:22 +02:00
}
2021-07-08 22:29:00 +02:00
2021-09-02 10:25:22 +02:00
fn parse_short_flags(
working_set: &mut StateWorkingSet,
spans: &[Span],
spans_idx: &mut usize,
positional_idx: usize,
sig: &Signature,
) -> (Option<Vec<Flag>>, Option<ParseError>) {
let mut error = None;
let arg_span = spans[*spans_idx];
let arg_contents = working_set.get_span_contents(arg_span);
if arg_contents.starts_with(b"-") && arg_contents.len() > 1 {
let short_flags = &arg_contents[1..];
let mut found_short_flags = vec![];
let mut unmatched_short_flags = vec![];
for short_flag in short_flags.iter().enumerate() {
let short_flag_char = char::from(*short_flag.1);
let orig = arg_span;
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let short_flag_span = Span::new(
orig.start + 1 + short_flag.0,
orig.start + 1 + short_flag.0 + 1,
);
2021-09-02 10:25:22 +02:00
if let Some(flag) = sig.get_short_flag(short_flag_char) {
// If we require an arg and are in a batch of short flags, error
if !found_short_flags.is_empty() && flag.arg.is_some() {
error = error.or(Some(ParseError::ShortFlagBatchCantTakeArg(short_flag_span)))
2021-07-08 22:29:00 +02:00
}
2021-09-02 10:25:22 +02:00
found_short_flags.push(flag);
} else {
unmatched_short_flags.push(short_flag_span);
2021-07-08 22:29:00 +02:00
}
2021-09-02 10:25:22 +02:00
}
2021-07-08 22:29:00 +02:00
2021-09-02 10:25:22 +02:00
if found_short_flags.is_empty() {
// check to see if we have a negative number
if let Some(positional) = sig.get_positional(positional_idx) {
if positional.shape == SyntaxShape::Int || positional.shape == SyntaxShape::Number {
if String::from_utf8_lossy(arg_contents).parse::<f64>().is_ok() {
return (None, None);
2021-07-08 22:29:00 +02:00
} else if let Some(first) = unmatched_short_flags.first() {
2021-09-21 06:03:06 +02:00
let contents = working_set.get_span_contents(*first);
error = error.or_else(|| {
Some(ParseError::UnknownFlag(
sig.name.clone(),
format!("-{}", String::from_utf8_lossy(contents)),
2021-09-21 06:03:06 +02:00
*first,
sig.clone().formatted_flags(),
2021-09-21 06:03:06 +02:00
))
});
2021-07-08 22:29:00 +02:00
}
} else if let Some(first) = unmatched_short_flags.first() {
2021-09-21 06:03:06 +02:00
let contents = working_set.get_span_contents(*first);
error = error.or_else(|| {
Some(ParseError::UnknownFlag(
sig.name.clone(),
format!("-{}", String::from_utf8_lossy(contents)),
2021-09-21 06:03:06 +02:00
*first,
sig.clone().formatted_flags(),
2021-09-21 06:03:06 +02:00
))
});
2021-07-08 22:29:00 +02:00
}
2021-09-02 10:25:22 +02:00
} else if let Some(first) = unmatched_short_flags.first() {
2021-09-21 06:03:06 +02:00
let contents = working_set.get_span_contents(*first);
error = error.or_else(|| {
Some(ParseError::UnknownFlag(
sig.name.clone(),
format!("-{}", String::from_utf8_lossy(contents)),
2021-09-21 06:03:06 +02:00
*first,
sig.clone().formatted_flags(),
2021-09-21 06:03:06 +02:00
))
});
2021-09-02 10:25:22 +02:00
}
} else if !unmatched_short_flags.is_empty() {
if let Some(first) = unmatched_short_flags.first() {
2021-09-21 06:03:06 +02:00
let contents = working_set.get_span_contents(*first);
error = error.or_else(|| {
Some(ParseError::UnknownFlag(
sig.name.clone(),
format!("-{}", String::from_utf8_lossy(contents)),
2021-09-21 06:03:06 +02:00
*first,
sig.clone().formatted_flags(),
2021-09-21 06:03:06 +02:00
))
});
2021-07-08 22:29:00 +02:00
}
}
2021-09-02 10:25:22 +02:00
(Some(found_short_flags), error)
} else {
(None, None)
2021-07-08 22:29:00 +02:00
}
2021-09-02 10:25:22 +02:00
}
2021-07-08 22:29:00 +02:00
2021-09-02 10:25:22 +02:00
fn first_kw_idx(
working_set: &StateWorkingSet,
signature: &Signature,
spans: &[Span],
spans_idx: usize,
positional_idx: usize,
) -> (Option<usize>, usize) {
for idx in (positional_idx + 1)..signature.num_positionals() {
if let Some(PositionalArg {
shape: SyntaxShape::Keyword(kw, ..),
..
}) = signature.get_positional(idx)
{
#[allow(clippy::needless_range_loop)]
for span_idx in spans_idx..spans.len() {
let contents = working_set.get_span_contents(spans[span_idx]);
2021-08-26 23:48:27 +02:00
2021-09-02 10:25:22 +02:00
if contents == kw {
return (Some(idx), span_idx);
2021-08-26 23:48:27 +02:00
}
}
}
}
2021-09-02 10:25:22 +02:00
(None, spans.len())
}
2021-08-26 23:48:27 +02:00
2021-09-02 10:25:22 +02:00
fn calculate_end_span(
working_set: &StateWorkingSet,
signature: &Signature,
spans: &[Span],
spans_idx: usize,
positional_idx: usize,
) -> usize {
if signature.rest_positional.is_some() {
spans.len()
} else {
let (kw_pos, kw_idx) =
first_kw_idx(working_set, signature, spans, spans_idx, positional_idx);
2021-08-26 23:48:27 +02:00
2021-09-02 10:25:22 +02:00
if let Some(kw_pos) = kw_pos {
// We found a keyword. Keywords, once found, create a guidepost to
// show us where the positionals will lay into the arguments. Because they're
// keywords, they get to set this by being present
2021-08-26 23:48:27 +02:00
2021-09-02 10:25:22 +02:00
let positionals_between = kw_pos - positional_idx - 1;
if positionals_between > (kw_idx - spans_idx) {
kw_idx
2021-08-26 23:48:27 +02:00
} else {
2021-09-02 10:25:22 +02:00
kw_idx - positionals_between
}
} else {
// Make space for the remaining require positionals, if we can
if signature.num_positionals_after(positional_idx) == 0 {
spans.len()
} else if positional_idx < signature.required_positional.len()
2021-09-02 10:25:22 +02:00
&& spans.len() > (signature.required_positional.len() - positional_idx)
{
spans.len() - (signature.required_positional.len() - positional_idx - 1)
} else {
2021-09-04 09:59:38 +02:00
spans_idx + 1
2021-07-24 07:57:17 +02:00
}
}
}
2021-09-02 10:25:22 +02:00
}
2021-07-24 07:57:17 +02:00
pub fn parse_multispan_value(
2021-09-02 10:25:22 +02:00
working_set: &mut StateWorkingSet,
spans: &[Span],
spans_idx: &mut usize,
shape: &SyntaxShape,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
let mut error = None;
match shape {
SyntaxShape::VarWithOptType => {
2022-01-01 22:42:50 +01:00
trace!("parsing: var with opt type");
let (arg, err) = parse_var_with_opt_type(working_set, spans, spans_idx, false);
2021-09-02 10:25:22 +02:00
error = error.or(err);
2021-07-08 23:16:25 +02:00
2021-09-02 10:25:22 +02:00
(arg, error)
}
SyntaxShape::RowCondition => {
2022-01-01 22:42:50 +01:00
trace!("parsing: row condition");
let (arg, err) =
parse_row_condition(working_set, &spans[*spans_idx..], expand_aliases_denylist);
2021-09-02 10:25:22 +02:00
error = error.or(err);
*spans_idx = spans.len() - 1;
2021-07-16 08:24:46 +02:00
2021-09-02 10:25:22 +02:00
(arg, error)
}
SyntaxShape::MathExpression => {
trace!("parsing: math expression");
let (arg, err) = parse_math_expression(
working_set,
&spans[*spans_idx..],
None,
expand_aliases_denylist,
);
error = error.or(err);
*spans_idx = spans.len() - 1;
(arg, error)
}
SyntaxShape::OneOf(shapes) => {
// handle for `if` command.
let block_then_exp = shapes.as_slice() == [SyntaxShape::Block, SyntaxShape::Expression];
let mut err = None;
for shape in shapes.iter() {
let (s, option_err) = parse_multispan_value(
working_set,
spans,
spans_idx,
shape,
expand_aliases_denylist,
);
match option_err {
None => return (s, None),
e => {
// `if` is parsing block first and then expression.
// when we're writing something like `else if $a`, parsing as a
// block will result to error(because it's not a block)
//
// If parse as a expression also failed, user is more likely concerned
// about expression failure rather than "expect block failure"".
if block_then_exp {
match &err {
Some(ParseError::Expected(expected, _)) => {
if expected.starts_with("block") {
err = e
}
}
_ => err = err.or(e),
}
} else {
err = err.or(e)
}
}
}
}
let span = spans[*spans_idx];
if err.is_some() {
(Expression::garbage(span), err)
} else {
(
Expression::garbage(span),
Some(ParseError::Expected(
format!("one of a list of accepted shapes: {shapes:?}"),
span,
)),
)
}
}
2021-09-02 10:25:22 +02:00
SyntaxShape::Expression => {
2022-01-01 22:42:50 +01:00
trace!("parsing: expression");
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
// is it subexpression?
// Not sure, but let's make it not, so the behavior is the same as previous version of nushell.
let (arg, err) = parse_expression(
working_set,
&spans[*spans_idx..],
expand_aliases_denylist,
false,
);
2021-09-02 10:25:22 +02:00
error = error.or(err);
*spans_idx = spans.len() - 1;
2021-07-08 23:16:25 +02:00
2021-09-02 10:25:22 +02:00
(arg, error)
}
SyntaxShape::Keyword(keyword, arg) => {
2022-01-01 22:42:50 +01:00
trace!(
"parsing: keyword({}) {:?}",
String::from_utf8_lossy(keyword),
arg
);
2021-09-02 10:25:22 +02:00
let arg_span = spans[*spans_idx];
2021-07-08 23:16:25 +02:00
2021-09-02 10:25:22 +02:00
let arg_contents = working_set.get_span_contents(arg_span);
2021-07-17 07:28:25 +02:00
2021-09-02 10:25:22 +02:00
if arg_contents != keyword {
// When keywords mismatch, this is a strong indicator of something going wrong.
// We won't often override the current error, but as this is a strong indicator
// go ahead and override the current error and tell the user about the missing
// keyword/literal.
error = Some(ParseError::ExpectedKeyword(
String::from_utf8_lossy(keyword).into(),
arg_span,
))
}
2021-07-17 07:28:25 +02:00
2021-09-02 10:25:22 +02:00
*spans_idx += 1;
if *spans_idx >= spans.len() {
error = error.or_else(|| {
Some(ParseError::KeywordMissingArgument(
2022-01-04 00:14:33 +01:00
arg.to_string(),
2021-07-30 00:56:51 +02:00
String::from_utf8_lossy(keyword).into(),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(spans[*spans_idx - 1].end, spans[*spans_idx - 1].end),
2021-07-08 23:16:25 +02:00
))
2021-09-02 10:25:22 +02:00
});
return (
2021-07-08 23:16:25 +02:00
Expression {
2021-09-02 10:25:22 +02:00
expr: Expr::Keyword(
keyword.clone(),
spans[*spans_idx - 1],
Box::new(Expression::garbage(arg_span)),
),
2021-07-08 23:16:25 +02:00
span: arg_span,
ty: Type::Any,
custom_completion: None,
2021-07-08 23:16:25 +02:00
},
error,
2021-09-02 10:25:22 +02:00
);
2021-07-08 23:16:25 +02:00
}
2021-09-02 10:25:22 +02:00
let keyword_span = spans[*spans_idx - 1];
let (expr, err) =
parse_multispan_value(working_set, spans, spans_idx, arg, expand_aliases_denylist);
2021-09-02 10:25:22 +02:00
error = error.or(err);
let ty = expr.ty.clone();
2021-07-17 07:28:25 +02:00
2021-09-02 10:25:22 +02:00
(
Expression {
expr: Expr::Keyword(keyword.clone(), keyword_span, Box::new(expr)),
span: arg_span,
ty,
custom_completion: None,
2021-09-02 10:25:22 +02:00
},
error,
)
}
_ => {
// All other cases are single-span values
let arg_span = spans[*spans_idx];
2021-07-08 23:16:25 +02:00
let (arg, err) = parse_value(working_set, arg_span, shape, expand_aliases_denylist);
2021-09-02 10:25:22 +02:00
error = error.or(err);
(arg, error)
2021-07-08 23:16:25 +02:00
}
}
2021-09-02 10:25:22 +02:00
}
2021-07-08 23:16:25 +02:00
pub struct ParsedInternalCall {
pub call: Box<Call>,
pub output: Type,
pub error: Option<ParseError>,
}
2021-09-02 10:25:22 +02:00
pub fn parse_internal_call(
working_set: &mut StateWorkingSet,
command_span: Span,
spans: &[Span],
decl_id: usize,
expand_aliases_denylist: &[usize],
) -> ParsedInternalCall {
2022-01-01 22:42:50 +01:00
trace!("parsing: internal call (decl id: {})", decl_id);
2021-09-02 10:25:22 +02:00
let mut error = None;
2021-07-02 00:40:08 +02:00
let mut call = Call::new(command_span);
2021-09-02 10:25:22 +02:00
call.decl_id = decl_id;
call.head = command_span;
2021-07-08 08:19:38 +02:00
let decl = working_set.get_decl(decl_id);
let signature = decl.signature();
let output = signature.output_type.clone();
2021-09-02 10:25:22 +02:00
// The index into the positional parameter in the definition
let mut positional_idx = 0;
2021-07-08 08:19:38 +02:00
2021-09-02 10:25:22 +02:00
// The index into the spans of argument data given to parse
// Starting at the first argument
let mut spans_idx = 0;
2021-07-02 00:40:08 +02:00
Re-implement aliases (#8123) # Description This PR adds an alternative alias implementation. Old aliases still work but you need to use `old-alias` instead of `alias`. Instead of replacing spans in the original code and re-parsing, which proved to be extremely error-prone and a constant source of panics, the new implementation creates a new command that references the old command. Consider the new alias defined as `alias ll = ls -l`. The parser creates a new command called `ll` and remembers that it is actually a `ls` command called with the `-l` flag. Then, when the parser sees the `ll` command, it will translate it to `ls -l` and passes to it any parameters that were passed to the call to `ll`. It works quite similar to how known externals defined with `extern` are implemented. The new alias implementation should work the same way as the old aliases, including exporting from modules, referencing both known and unknown externals. It seems to preserve custom completions and pipeline metadata. It is quite robust in most cases but there are some rough edges (see later). Fixes https://github.com/nushell/nushell/issues/7648, https://github.com/nushell/nushell/issues/8026, https://github.com/nushell/nushell/issues/7512, https://github.com/nushell/nushell/issues/5780, https://github.com/nushell/nushell/issues/7754 No effect: https://github.com/nushell/nushell/issues/8122 (we might revisit the completions code after this PR) Should use custom command instead: https://github.com/nushell/nushell/issues/6048 # User-Facing Changes Since aliases are now basically commands, it has some new implications: 1. `alias spam = "spam"` (requires command call) * **workaround**: use `alias spam = echo "spam"` 2. `def foo [] { 'foo' }; alias foo = ls -l` (foo defined more than once) * **workaround**: use different name (commands also have this limitation) 4. `alias ls = (ls | sort-by type name -i)` * **workaround**: Use custom command. _The common issue with this is that it is currently not easy to pass flags through custom commands and command referencing itself will lead to stack overflow. Both of these issues are meant to be addressed._ 5. TODO: Help messages, `which` command, `$nu.scope.aliases`, etc. * Should we treat the aliases as commands or should they be separated from regular commands? 6. Needs better error message and syntax highlight for recursed alias (`alias f = f`) 7. Can't create alias with the same name as existing command (`alias ls = ls -a`) * Might be possible to add support for it (not 100% sure) 8. Standalone `alias` doesn't list aliases anymore 9. Can't alias parser keywords (e.g., stuff like `alias ou = overlay use` won't work) * TODO: Needs a better error message when attempting to do so # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-02-27 08:44:05 +01:00
if let Some(alias) = decl.as_alias() {
if let Expression {
expr: Expr::Call(wrapped_call),
..
} = &alias.wrapped_call
{
// Replace this command's call with the aliased call, but keep the alias name
call = *wrapped_call.clone();
call.head = command_span;
// Skip positionals passed to aliased call
positional_idx = call.positional_len();
} else {
return ParsedInternalCall {
call: Box::new(call),
output: Type::Any,
error: Some(ParseError::UnknownState(
"Alias does not point to internal call.".to_string(),
command_span,
)),
};
}
}
working_set.type_scope.add_type(output.clone());
if signature.creates_scope {
working_set.enter_scope();
}
2021-09-02 10:25:22 +02:00
while spans_idx < spans.len() {
let arg_span = spans[spans_idx];
2021-07-08 23:16:25 +02:00
2021-09-02 10:25:22 +02:00
// Check if we're on a long flag, if so, parse
let (long_name, arg, err) = parse_long_flag(
working_set,
spans,
&mut spans_idx,
&signature,
expand_aliases_denylist,
);
2021-09-02 10:25:22 +02:00
if let Some(long_name) = long_name {
// We found a long flag, like --bar
if matches!(err, Some(ParseError::UnknownFlag(_, _, _, _)))
&& signature.allows_unknown_args
{
let (arg, arg_err) = parse_value(
working_set,
arg_span,
&SyntaxShape::Any,
expand_aliases_denylist,
);
error = error.or(arg_err);
call.add_unknown(arg);
} else {
error = error.or(err);
call.add_named((long_name, None, arg));
}
2021-09-02 10:25:22 +02:00
spans_idx += 1;
continue;
}
2021-07-02 00:40:08 +02:00
2021-09-02 10:25:22 +02:00
// Check if we're on a short flag or group of short flags, if so, parse
let (short_flags, err) = parse_short_flags(
working_set,
spans,
&mut spans_idx,
positional_idx,
&signature,
);
2021-07-08 22:29:00 +02:00
2022-07-17 14:46:40 +02:00
if let Some(mut short_flags) = short_flags {
if short_flags.is_empty() {
// workaround for completions (PR #6067)
2022-07-17 14:46:40 +02:00
short_flags.push(Flag {
long: "".to_string(),
short: Some('a'),
arg: None,
required: false,
desc: "".to_string(),
var_id: None,
default_value: None,
})
}
2021-09-02 10:25:22 +02:00
if matches!(err, Some(ParseError::UnknownFlag(_, _, _, _)))
&& signature.allows_unknown_args
{
let (arg, arg_err) = parse_value(
working_set,
arg_span,
&SyntaxShape::Any,
expand_aliases_denylist,
);
call.add_unknown(arg);
error = error.or(arg_err);
} else {
error = error.or(err);
for flag in short_flags {
if let Some(arg_shape) = flag.arg {
if let Some(arg) = spans.get(spans_idx + 1) {
let (arg, err) =
parse_value(working_set, *arg, &arg_shape, expand_aliases_denylist);
error = error.or(err);
if flag.long.is_empty() {
if let Some(short) = flag.short {
call.add_named((
Spanned {
item: String::new(),
span: spans[spans_idx],
},
Some(Spanned {
item: short.to_string(),
span: spans[spans_idx],
}),
Some(arg),
));
}
} else {
call.add_named((
Spanned {
item: flag.long.clone(),
span: spans[spans_idx],
},
None,
Some(arg),
));
}
spans_idx += 1;
} else {
error = error.or_else(|| {
Some(ParseError::MissingFlagParam(
arg_shape.to_string(),
arg_span,
))
})
}
} else if flag.long.is_empty() {
if let Some(short) = flag.short {
call.add_named((
Spanned {
item: String::new(),
span: spans[spans_idx],
},
Some(Spanned {
item: short.to_string(),
span: spans[spans_idx],
}),
None,
));
}
2021-07-08 08:19:38 +02:00
} else {
call.add_named((
Spanned {
item: flag.long.clone(),
span: spans[spans_idx],
},
None,
None,
));
}
2021-07-08 08:19:38 +02:00
}
2021-07-08 22:29:00 +02:00
}
2021-09-02 10:25:22 +02:00
spans_idx += 1;
continue;
}
2021-07-08 22:29:00 +02:00
2021-09-02 10:25:22 +02:00
// Parse a positional arg if there is one
if let Some(positional) = signature.get_positional(positional_idx) {
let end = calculate_end_span(working_set, &signature, spans, spans_idx, positional_idx);
2021-07-17 01:22:01 +02:00
let end = if spans.len() > spans_idx && end == spans_idx {
end + 1
} else {
end
};
if spans[..end].is_empty() || spans_idx == end {
error = error.or_else(|| {
Some(ParseError::MissingPositional(
positional.name.clone(),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(spans[spans_idx].end, spans[spans_idx].end),
2022-01-04 00:14:33 +01:00
signature.call_signature(),
))
});
positional_idx += 1;
continue;
}
2021-09-02 10:25:22 +02:00
let (arg, err) = parse_multispan_value(
working_set,
&spans[..end],
&mut spans_idx,
&positional.shape,
expand_aliases_denylist,
2021-09-02 10:25:22 +02:00
);
error = error.or(err);
2021-07-23 23:46:55 +02:00
2021-09-02 10:25:22 +02:00
let arg = if !type_compatible(&positional.shape.to_type(), &arg.ty) {
error = error.or_else(|| {
Some(ParseError::TypeMismatch(
positional.shape.to_type(),
arg.ty,
arg.span,
))
});
Expression::garbage(arg.span)
2021-07-08 08:19:38 +02:00
} else {
2021-09-02 10:25:22 +02:00
arg
};
call.add_positional(arg);
2021-09-02 10:25:22 +02:00
positional_idx += 1;
} else if signature.allows_unknown_args {
let (arg, arg_err) = parse_value(
working_set,
arg_span,
&SyntaxShape::Any,
expand_aliases_denylist,
);
call.add_unknown(arg);
error = error.or(arg_err);
2021-09-02 10:25:22 +02:00
} else {
call.add_positional(Expression::garbage(arg_span));
2022-01-04 00:14:33 +01:00
error = error.or_else(|| {
Some(ParseError::ExtraPositional(
signature.call_signature(),
arg_span,
))
})
2021-07-08 08:19:38 +02:00
}
2021-07-02 00:40:08 +02:00
2021-07-08 08:19:38 +02:00
error = error.or(err);
2021-09-02 10:25:22 +02:00
spans_idx += 1;
2021-07-08 08:19:38 +02:00
}
2021-09-02 10:25:22 +02:00
let err = check_call(command_span, &signature, &call);
error = error.or(err);
2021-07-31 07:20:40 +02:00
2021-10-09 18:10:46 +02:00
if signature.creates_scope {
working_set.exit_scope();
}
ParsedInternalCall {
call: Box::new(call),
output,
error,
}
2021-09-02 10:25:22 +02:00
}
2021-07-31 07:20:40 +02:00
2021-09-02 10:25:22 +02:00
pub fn parse_call(
working_set: &mut StateWorkingSet,
spans: &[Span],
2021-12-19 08:46:13 +01:00
head: Span,
expand_aliases_denylist: &[usize],
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression: bool,
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
2022-01-01 22:42:50 +01:00
trace!("parsing: call");
2021-10-29 22:50:28 +02:00
if spans.is_empty() {
return (
2021-12-19 08:46:13 +01:00
garbage(head),
Some(ParseError::UnknownState(
"Encountered command with zero spans".into(),
span(spans),
)),
2021-10-29 22:50:28 +02:00
);
}
2021-09-02 10:25:22 +02:00
let mut pos = 0;
let cmd_start = pos;
2021-10-29 22:50:28 +02:00
let mut name_spans = vec![];
2022-01-10 03:52:01 +01:00
let mut name = vec![];
2021-08-17 01:00:00 +02:00
2021-10-29 22:50:28 +02:00
for word_span in spans[cmd_start..].iter() {
// Find the longest group of words that could form a command
// if is_math_expression_like(working_set, *word_span, expand_aliases_denylist) {
// let bytes = working_set.get_span_contents(*word_span);
// if bytes != b"true" && bytes != b"false" && bytes != b"null" && bytes != b"not" {
// break;
// }
// }
2021-08-09 02:19:07 +02:00
2021-10-29 22:50:28 +02:00
name_spans.push(*word_span);
2021-09-02 10:25:22 +02:00
2022-01-10 03:52:01 +01:00
let name_part = working_set.get_span_contents(*word_span);
if name.is_empty() {
name.extend(name_part);
} else {
name.push(b' ');
name.extend(name_part);
}
2021-09-02 10:25:22 +02:00
// If the word is an alias, expand it and re-parse the expression
if let Some(alias_id) = working_set.find_alias(&name) {
if !expand_aliases_denylist.contains(&alias_id) {
2022-01-01 22:42:50 +01:00
trace!("expanding alias");
let expansion = working_set.get_alias(alias_id);
2022-02-15 03:09:21 +01:00
let expansion_span = span(expansion);
let orig_span = span(&[spans[cmd_start], spans[pos]]);
2021-10-29 22:50:28 +02:00
let mut new_spans: Vec<Span> = vec![];
2022-02-15 03:09:21 +01:00
new_spans.extend(&spans[0..cmd_start]);
2021-10-29 22:50:28 +02:00
new_spans.extend(expansion);
2022-03-03 20:05:55 +01:00
// TODO: This seems like it should be `pos + 1`. `pos` starts as 0
2021-10-29 22:50:28 +02:00
if spans.len() > pos {
new_spans.extend(&spans[(pos + 1)..]);
}
2021-08-17 01:00:00 +02:00
let mut expand_aliases_denylist = expand_aliases_denylist.to_vec();
expand_aliases_denylist.push(alias_id);
let lite_command = LiteCommand {
comments: vec![],
parts: new_spans.clone(),
};
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
let (mut result, err) = parse_builtin_commands(
working_set,
&lite_command,
&expand_aliases_denylist,
is_subexpression,
);
let result = result.elements.remove(0);
2021-08-17 01:00:00 +02:00
// If this is the first element in a pipeline, we know it has to be an expression
if let PipelineElement::Expression(_, mut result) = result {
result.replace_span(working_set, expansion_span, orig_span);
2021-08-17 01:00:00 +02:00
return (result, err);
} else {
panic!("Internal error: first element of pipeline not an expression")
}
2021-09-02 10:25:22 +02:00
}
2021-10-29 22:50:28 +02:00
}
2021-08-17 01:00:00 +02:00
2021-10-29 22:50:28 +02:00
pos += 1;
}
let input = working_set.type_scope.get_previous();
let mut maybe_decl_id = working_set.find_decl(&name, input);
2021-10-29 22:50:28 +02:00
while maybe_decl_id.is_none() {
// Find the longest command match
if name_spans.len() <= 1 {
// Keep the first word even if it does not match -- could be external command
break;
2021-06-30 03:42:56 +02:00
}
2021-09-11 14:07:19 +02:00
2021-10-29 22:50:28 +02:00
name_spans.pop();
pos -= 1;
2022-01-10 03:52:01 +01:00
let mut name = vec![];
for name_span in &name_spans {
let name_part = working_set.get_span_contents(*name_span);
if name.is_empty() {
name.extend(name_part);
} else {
name.push(b' ');
name.extend(name_part);
}
}
maybe_decl_id = working_set.find_decl(&name, input);
2021-10-29 22:50:28 +02:00
}
if let Some(decl_id) = maybe_decl_id {
2021-09-11 14:07:19 +02:00
// Before the internal parsing we check if there is no let or alias declarations
// that are missing their name, e.g.: let = 1 or alias = 2
if spans.len() > 1 {
let test_equal = working_set.get_span_contents(spans[1]);
2021-09-11 14:16:40 +02:00
if test_equal == [b'='] {
2022-01-01 22:42:50 +01:00
trace!("incomplete statement");
2021-09-11 14:07:19 +02:00
return (
2021-12-19 08:46:13 +01:00
garbage(span(spans)),
2021-09-11 14:07:19 +02:00
Some(ParseError::UnknownState(
2021-09-12 17:36:16 +02:00
"Incomplete statement".into(),
2021-09-11 14:07:19 +02:00
span(spans),
)),
);
}
}
Re-implement aliases (#8123) # Description This PR adds an alternative alias implementation. Old aliases still work but you need to use `old-alias` instead of `alias`. Instead of replacing spans in the original code and re-parsing, which proved to be extremely error-prone and a constant source of panics, the new implementation creates a new command that references the old command. Consider the new alias defined as `alias ll = ls -l`. The parser creates a new command called `ll` and remembers that it is actually a `ls` command called with the `-l` flag. Then, when the parser sees the `ll` command, it will translate it to `ls -l` and passes to it any parameters that were passed to the call to `ll`. It works quite similar to how known externals defined with `extern` are implemented. The new alias implementation should work the same way as the old aliases, including exporting from modules, referencing both known and unknown externals. It seems to preserve custom completions and pipeline metadata. It is quite robust in most cases but there are some rough edges (see later). Fixes https://github.com/nushell/nushell/issues/7648, https://github.com/nushell/nushell/issues/8026, https://github.com/nushell/nushell/issues/7512, https://github.com/nushell/nushell/issues/5780, https://github.com/nushell/nushell/issues/7754 No effect: https://github.com/nushell/nushell/issues/8122 (we might revisit the completions code after this PR) Should use custom command instead: https://github.com/nushell/nushell/issues/6048 # User-Facing Changes Since aliases are now basically commands, it has some new implications: 1. `alias spam = "spam"` (requires command call) * **workaround**: use `alias spam = echo "spam"` 2. `def foo [] { 'foo' }; alias foo = ls -l` (foo defined more than once) * **workaround**: use different name (commands also have this limitation) 4. `alias ls = (ls | sort-by type name -i)` * **workaround**: Use custom command. _The common issue with this is that it is currently not easy to pass flags through custom commands and command referencing itself will lead to stack overflow. Both of these issues are meant to be addressed._ 5. TODO: Help messages, `which` command, `$nu.scope.aliases`, etc. * Should we treat the aliases as commands or should they be separated from regular commands? 6. Needs better error message and syntax highlight for recursed alias (`alias f = f`) 7. Can't create alias with the same name as existing command (`alias ls = ls -a`) * Might be possible to add support for it (not 100% sure) 8. Standalone `alias` doesn't list aliases anymore 9. Can't alias parser keywords (e.g., stuff like `alias ou = overlay use` won't work) * TODO: Needs a better error message when attempting to do so # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-02-27 08:44:05 +01:00
// TODO: Try to remove the clone
let decl = working_set.get_decl(decl_id).clone();
2022-01-01 22:42:50 +01:00
Re-implement aliases (#8123) # Description This PR adds an alternative alias implementation. Old aliases still work but you need to use `old-alias` instead of `alias`. Instead of replacing spans in the original code and re-parsing, which proved to be extremely error-prone and a constant source of panics, the new implementation creates a new command that references the old command. Consider the new alias defined as `alias ll = ls -l`. The parser creates a new command called `ll` and remembers that it is actually a `ls` command called with the `-l` flag. Then, when the parser sees the `ll` command, it will translate it to `ls -l` and passes to it any parameters that were passed to the call to `ll`. It works quite similar to how known externals defined with `extern` are implemented. The new alias implementation should work the same way as the old aliases, including exporting from modules, referencing both known and unknown externals. It seems to preserve custom completions and pipeline metadata. It is quite robust in most cases but there are some rough edges (see later). Fixes https://github.com/nushell/nushell/issues/7648, https://github.com/nushell/nushell/issues/8026, https://github.com/nushell/nushell/issues/7512, https://github.com/nushell/nushell/issues/5780, https://github.com/nushell/nushell/issues/7754 No effect: https://github.com/nushell/nushell/issues/8122 (we might revisit the completions code after this PR) Should use custom command instead: https://github.com/nushell/nushell/issues/6048 # User-Facing Changes Since aliases are now basically commands, it has some new implications: 1. `alias spam = "spam"` (requires command call) * **workaround**: use `alias spam = echo "spam"` 2. `def foo [] { 'foo' }; alias foo = ls -l` (foo defined more than once) * **workaround**: use different name (commands also have this limitation) 4. `alias ls = (ls | sort-by type name -i)` * **workaround**: Use custom command. _The common issue with this is that it is currently not easy to pass flags through custom commands and command referencing itself will lead to stack overflow. Both of these issues are meant to be addressed._ 5. TODO: Help messages, `which` command, `$nu.scope.aliases`, etc. * Should we treat the aliases as commands or should they be separated from regular commands? 6. Needs better error message and syntax highlight for recursed alias (`alias f = f`) 7. Can't create alias with the same name as existing command (`alias ls = ls -a`) * Might be possible to add support for it (not 100% sure) 8. Standalone `alias` doesn't list aliases anymore 9. Can't alias parser keywords (e.g., stuff like `alias ou = overlay use` won't work) * TODO: Needs a better error message when attempting to do so # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-02-27 08:44:05 +01:00
let parsed_call = if let Some(alias) = decl.as_alias() {
if let Expression {
expr: Expr::ExternalCall(head, args, is_subexpression),
span: _,
ty,
custom_completion,
} = &alias.wrapped_call
{
trace!("parsing: alias of external call");
let mut error = None;
let mut final_args = args.clone();
for arg_span in spans.iter().skip(1) {
let (arg, err) =
parse_external_arg(working_set, *arg_span, expand_aliases_denylist);
error = error.or(err);
final_args.push(arg);
}
let mut head = head.clone();
head.span = spans[0]; // replacing the spans preserves syntax highlighting
return (
Expression {
expr: Expr::ExternalCall(head, final_args, *is_subexpression),
span: span(spans),
ty: ty.clone(),
custom_completion: *custom_completion,
},
error,
);
} else {
trace!("parsing: alias of internal call");
parse_internal_call(
working_set,
span(&spans[cmd_start..pos]),
&spans[pos..],
decl_id,
expand_aliases_denylist,
)
}
} else {
trace!("parsing: internal call");
parse_internal_call(
working_set,
span(&spans[cmd_start..pos]),
&spans[pos..],
decl_id,
expand_aliases_denylist,
)
};
(
Expression {
expr: Expr::Call(parsed_call.call),
span: span(spans),
ty: parsed_call.output,
custom_completion: None,
},
parsed_call.error,
)
2021-09-02 10:25:22 +02:00
} else {
// We might be parsing left-unbounded range ("..10")
let bytes = working_set.get_span_contents(spans[0]);
2022-01-01 22:42:50 +01:00
trace!("parsing: range {:?} ", bytes);
if let (Some(b'.'), Some(b'.')) = (bytes.first(), bytes.get(1)) {
2022-01-01 22:42:50 +01:00
trace!("-- found leading range indicator");
let (range_expr, range_err) =
parse_range(working_set, spans[0], expand_aliases_denylist);
if range_err.is_none() {
2022-01-01 22:42:50 +01:00
trace!("-- successfully parsed range");
return (range_expr, range_err);
}
}
2022-01-01 22:42:50 +01:00
trace!("parsing: external call");
2021-10-29 22:50:28 +02:00
// Otherwise, try external command
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
parse_external_call(
working_set,
spans,
expand_aliases_denylist,
is_subexpression,
)
2021-09-02 10:25:22 +02:00
}
}
2022-03-01 00:31:53 +01:00
pub fn parse_binary(
working_set: &mut StateWorkingSet,
span: Span,
) -> (Expression, Option<ParseError>) {
let (hex_value, err) = parse_binary_with_base(working_set, span, 16, 2, b"0x[", b"]");
if err.is_some() {
let (octal_value, err) = parse_binary_with_base(working_set, span, 8, 3, b"0o[", b"]");
if err.is_some() {
return parse_binary_with_base(working_set, span, 2, 8, b"0b[", b"]");
}
return (octal_value, err);
2022-03-01 00:31:53 +01:00
}
(hex_value, err)
}
2022-03-01 00:31:53 +01:00
fn parse_binary_with_base(
working_set: &mut StateWorkingSet,
span: Span,
base: u32,
min_digits_per_byte: usize,
prefix: &[u8],
suffix: &[u8],
) -> (Expression, Option<ParseError>) {
2022-03-01 00:31:53 +01:00
let token = working_set.get_span_contents(span);
if let Some(token) = token.strip_prefix(prefix) {
if let Some(token) = token.strip_suffix(suffix) {
let (lexed, err) = lex(
token,
span.start + prefix.len(),
&[b',', b'\r', b'\n'],
&[],
true,
);
2022-03-01 00:31:53 +01:00
let mut binary_value = vec![];
2022-03-01 00:31:53 +01:00
for token in lexed {
match token.contents {
TokenContents::Item => {
let contents = working_set.get_span_contents(token.span);
binary_value.extend_from_slice(contents);
2022-03-01 00:31:53 +01:00
}
Better errors when bash-like operators are used (#7241) # Description Adds improved errors for when a user uses a bashism that nu doesn't support. fixes #7237 Examples: ``` Error: nu::parser::shell_andand (link) × The '&&' operator is not supported in Nushell ╭─[entry #1:1:1] 1 │ ls && ls · ─┬ · ╰── instead of '&&', use ';' or 'and' ╰──── help: use ';' instead of the shell '&&', or 'and' instead of the boolean '&&' ``` ``` Error: nu::parser::shell_oror (link) × The '||' operator is not supported in Nushell ╭─[entry #8:1:1] 1 │ ls || ls · ─┬ · ╰── instead of '||', use 'try' or 'or' ╰──── help: use 'try' instead of the shell '||', or 'or' instead of the boolean '||' ``` ``` Error: nu::parser::shell_err (link) × The '2>' shell operation is 'err>' in Nushell. ╭─[entry #9:1:1] 1 │ foo 2> bar.txt · ─┬ · ╰── use 'err>' instead of '2>' in Nushell ╰──── ``` ``` Error: nu::parser::shell_outerr (link) × The '2>&1' shell operation is 'out+err>' in Nushell. ╭─[entry #10:1:1] 1 │ foo 2>&1 bar.txt · ──┬─ · ╰── use 'out+err>' instead of '2>&1' in Nushell ╰──── help: Nushell redirection will write all of stdout before stderr. ``` # User-Facing Changes **BREAKING CHANGES** This removes the `&&` and `||` operators. We previously supported by `&&`/`and` and `||`/`or`. With this change, only `and` and `or` are valid boolean operators. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-12-08 00:02:11 +01:00
TokenContents::Pipe
| TokenContents::PipePipe
| TokenContents::OutGreaterThan
| TokenContents::ErrGreaterThan
| TokenContents::OutErrGreaterThan => {
2022-03-01 00:31:53 +01:00
return (
garbage(span),
Some(ParseError::Expected("binary".into(), span)),
);
}
Better errors when bash-like operators are used (#7241) # Description Adds improved errors for when a user uses a bashism that nu doesn't support. fixes #7237 Examples: ``` Error: nu::parser::shell_andand (link) × The '&&' operator is not supported in Nushell ╭─[entry #1:1:1] 1 │ ls && ls · ─┬ · ╰── instead of '&&', use ';' or 'and' ╰──── help: use ';' instead of the shell '&&', or 'and' instead of the boolean '&&' ``` ``` Error: nu::parser::shell_oror (link) × The '||' operator is not supported in Nushell ╭─[entry #8:1:1] 1 │ ls || ls · ─┬ · ╰── instead of '||', use 'try' or 'or' ╰──── help: use 'try' instead of the shell '||', or 'or' instead of the boolean '||' ``` ``` Error: nu::parser::shell_err (link) × The '2>' shell operation is 'err>' in Nushell. ╭─[entry #9:1:1] 1 │ foo 2> bar.txt · ─┬ · ╰── use 'err>' instead of '2>' in Nushell ╰──── ``` ``` Error: nu::parser::shell_outerr (link) × The '2>&1' shell operation is 'out+err>' in Nushell. ╭─[entry #10:1:1] 1 │ foo 2>&1 bar.txt · ──┬─ · ╰── use 'out+err>' instead of '2>&1' in Nushell ╰──── help: Nushell redirection will write all of stdout before stderr. ``` # User-Facing Changes **BREAKING CHANGES** This removes the `&&` and `||` operators. We previously supported by `&&`/`and` and `||`/`or`. With this change, only `and` and `or` are valid boolean operators. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-12-08 00:02:11 +01:00
TokenContents::Comment | TokenContents::Semicolon | TokenContents::Eol => {}
2022-03-01 00:31:53 +01:00
}
}
let required_padding = (min_digits_per_byte - binary_value.len() % min_digits_per_byte)
% min_digits_per_byte;
if required_padding != 0 {
binary_value = {
let mut tail = binary_value;
let mut binary_value: Vec<u8> = vec![b'0'; required_padding];
binary_value.append(&mut tail);
binary_value
};
2022-03-01 00:31:53 +01:00
}
let str = String::from_utf8_lossy(&binary_value).to_string();
2022-03-01 00:31:53 +01:00
match decode_with_base(&str, base, min_digits_per_byte) {
2022-03-01 00:31:53 +01:00
Ok(v) => {
return (
Expression {
expr: Expr::Binary(v),
span,
ty: Type::Binary,
custom_completion: None,
},
err,
)
}
Err(x) => {
return (
garbage(span),
Some(ParseError::IncorrectValue(
"not a binary value".into(),
span,
x.to_string(),
)),
)
}
}
}
}
(
garbage(span),
Some(ParseError::Expected("binary".into(), span)),
)
}
fn decode_with_base(s: &str, base: u32, digits_per_byte: usize) -> Result<Vec<u8>, ParseIntError> {
2022-06-17 20:11:48 +02:00
s.chars()
.chunks(digits_per_byte)
.into_iter()
.map(|chunk| {
let str: String = chunk.collect();
u8::from_str_radix(&str, base)
})
.collect()
}
fn strip_underscores(token: &[u8]) -> String {
String::from_utf8_lossy(token)
.chars()
.filter(|c| *c != '_')
.collect()
}
2021-09-07 00:02:24 +02:00
pub fn parse_int(token: &[u8], span: Span) -> (Expression, Option<ParseError>) {
fn extract_int(token: &str, span: Span, radix: u32) -> (Expression, Option<ParseError>) {
if let Ok(num) = i64::from_str_radix(token, radix) {
2021-08-08 22:21:21 +02:00
(
Expression {
expr: Expr::Int(num),
2021-08-08 22:21:21 +02:00
span,
2021-09-02 10:25:22 +02:00
ty: Type::Int,
custom_completion: None,
2021-08-08 22:21:21 +02:00
},
None,
)
} else {
(
garbage(span),
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
Some(ParseError::InvalidLiteral(
format!("invalid digits for radix {}", radix),
2021-09-02 10:25:22 +02:00
"int".into(),
span,
)),
2021-08-08 22:21:21 +02:00
)
}
}
let token = strip_underscores(token);
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
if token.is_empty() {
return (
garbage(span),
Some(ParseError::Expected("int".into(), span)),
);
}
if let Some(num) = token.strip_prefix("0b") {
extract_int(num, span, 2)
} else if let Some(num) = token.strip_prefix("0o") {
extract_int(num, span, 8)
} else if let Some(num) = token.strip_prefix("0x") {
extract_int(num, span, 16)
} else if let Ok(num) = token.parse::<i64>() {
2021-09-02 10:25:22 +02:00
(
Expression {
expr: Expr::Int(num),
2021-09-02 10:25:22 +02:00
span,
ty: Type::Int,
custom_completion: None,
2021-09-02 10:25:22 +02:00
},
None,
)
} else {
(
garbage(span),
Some(ParseError::Expected("int".into(), span)),
)
2021-07-01 02:01:04 +02:00
}
2021-09-02 10:25:22 +02:00
}
2021-07-01 02:01:04 +02:00
2021-09-07 00:02:24 +02:00
pub fn parse_float(token: &[u8], span: Span) -> (Expression, Option<ParseError>) {
let token = strip_underscores(token);
if let Ok(x) = token.parse::<f64>() {
2021-09-02 10:25:22 +02:00
(
Expression {
expr: Expr::Float(x),
span,
ty: Type::Float,
custom_completion: None,
2021-09-02 10:25:22 +02:00
},
None,
)
} else {
(
garbage(span),
Some(ParseError::Expected("float".into(), span)),
)
}
}
2021-07-30 05:26:06 +02:00
2021-09-07 00:02:24 +02:00
pub fn parse_number(token: &[u8], span: Span) -> (Expression, Option<ParseError>) {
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
match parse_int(token, span) {
(x, None) => {
return (x, None);
}
(_, Some(ParseError::Expected(_, _))) => {}
(x, e) => return (x, e),
2021-07-30 05:26:06 +02:00
}
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
if let (x, None) = parse_float(token, span) {
return (x, None);
}
(
garbage(span),
Some(ParseError::Expected("number".into(), span)),
)
2021-09-02 10:25:22 +02:00
}
2021-07-30 05:26:06 +02:00
pub fn parse_range(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
) -> (Expression, Option<ParseError>) {
2022-01-01 22:42:50 +01:00
trace!("parsing: range");
// Range follows the following syntax: [<from>][<next_operator><next>]<range_operator>[<to>]
// where <next_operator> is ".."
// and <range_operator> is ".." or "..<"
// and one of the <from> or <to> bounds must be present (just '..' is not allowed since it
// looks like parent directory)
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
//bugbug range cannot be [..] because that looks like parent directory
let contents = working_set.get_span_contents(span);
let token = if let Ok(s) = String::from_utf8(contents.into()) {
s
} else {
return (garbage(span), Some(ParseError::NonUtf8(span)));
};
if !token.contains("..") {
return (
garbage(span),
Some(ParseError::Expected(
"at least one range bound set".into(),
span,
)),
);
}
// First, figure out what exact operators are used and determine their positions
let dotdot_pos: Vec<_> = token.match_indices("..").map(|(pos, _)| pos).collect();
let (next_op_pos, range_op_pos) =
match dotdot_pos.len() {
1 => (None, dotdot_pos[0]),
2 => (Some(dotdot_pos[0]), dotdot_pos[1]),
_ => return (
garbage(span),
Some(ParseError::Expected(
"one range operator ('..' or '..<') and optionally one next operator ('..')"
.into(),
span,
)),
),
};
let (inclusion, range_op_str, range_op_span) = if let Some(pos) = token.find("..<") {
if pos == range_op_pos {
let op_str = "..<";
let op_span = Span::new(
span.start + range_op_pos,
span.start + range_op_pos + op_str.len(),
);
(RangeInclusion::RightExclusive, "..<", op_span)
} else {
return (
garbage(span),
Some(ParseError::Expected(
"inclusive operator preceding second range bound".into(),
span,
)),
);
}
} else {
let op_str = "..";
let op_span = Span::new(
span.start + range_op_pos,
span.start + range_op_pos + op_str.len(),
);
(RangeInclusion::Inclusive, "..", op_span)
};
// Now, based on the operator positions, figure out where the bounds & next are located and
// parse them
2021-10-12 19:44:23 +02:00
// TODO: Actually parse the next number in the range
let from = if token.starts_with("..") {
// token starts with either next operator, or range operator -- we don't care which one
None
} else {
let from_span = Span::new(span.start, span.start + dotdot_pos[0]);
match parse_value(
working_set,
from_span,
&SyntaxShape::Number,
expand_aliases_denylist,
) {
(expression, None) => Some(Box::new(expression)),
_ => {
return (
garbage(span),
Some(ParseError::Expected("number".into(), span)),
)
}
}
};
let to = if token.ends_with(range_op_str) {
None
} else {
let to_span = Span::new(range_op_span.end, span.end);
match parse_value(
working_set,
to_span,
&SyntaxShape::Number,
expand_aliases_denylist,
) {
(expression, None) => Some(Box::new(expression)),
_ => {
return (
garbage(span),
Some(ParseError::Expected("number".into(), span)),
)
}
}
};
2022-01-01 22:42:50 +01:00
trace!("-- from: {:?} to: {:?}", from, to);
if let (None, None) = (&from, &to) {
return (
garbage(span),
Some(ParseError::Expected(
"at least one range bound set".into(),
span,
)),
);
}
let (next, next_op_span) = if let Some(pos) = next_op_pos {
let next_op_span = Span::new(span.start + pos, span.start + pos + "..".len());
let next_span = Span::new(next_op_span.end, range_op_span.start);
match parse_value(
working_set,
next_span,
&SyntaxShape::Number,
expand_aliases_denylist,
) {
(expression, None) => (Some(Box::new(expression)), next_op_span),
_ => {
return (
garbage(span),
Some(ParseError::Expected("number".into(), span)),
)
}
}
} else {
2021-12-19 08:46:13 +01:00
(None, span)
};
let range_op = RangeOperator {
inclusion,
span: range_op_span,
next_op_span,
};
(
Expression {
expr: Expr::Range(from, next, to, range_op),
span,
ty: Type::Range,
custom_completion: None,
},
None,
)
}
2021-09-02 10:25:22 +02:00
pub(crate) fn parse_dollar_expr(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
trace!("parsing: dollar expression");
2021-09-02 10:25:22 +02:00
let contents = working_set.get_span_contents(span);
2021-07-30 05:26:06 +02:00
if contents.starts_with(b"$\"") || contents.starts_with(b"$'") {
parse_string_interpolation(working_set, span, expand_aliases_denylist)
} else if contents.starts_with(b"$.") {
parse_simple_cell_path(
working_set,
Span::new(span.start + 2, span.end),
expand_aliases_denylist,
)
} else if let (expr, None) = parse_range(working_set, span, expand_aliases_denylist) {
2021-09-05 00:40:15 +02:00
(expr, None)
2021-09-02 10:25:22 +02:00
} else {
parse_full_cell_path(working_set, None, span, expand_aliases_denylist)
2021-09-02 10:25:22 +02:00
}
}
2021-07-30 05:26:06 +02:00
pub fn parse_paren_expr(
working_set: &mut StateWorkingSet,
span: Span,
shape: &SyntaxShape,
expand_aliases_denylist: &[usize],
) -> (Expression, Option<ParseError>) {
if let (expr, None) = parse_range(working_set, span, expand_aliases_denylist) {
(expr, None)
} else if matches!(shape, SyntaxShape::Signature) {
return parse_signature(working_set, span, expand_aliases_denylist);
} else {
parse_full_cell_path(working_set, None, span, expand_aliases_denylist)
}
}
Require that values that look like numbers parse as numberlike (#8635) # Description Require that any value that looks like it might be a number (starts with a digit, or a '-' + digit, or a '+' + digits, or a special form float like `-inf`, `inf`, or `NaN`) must now be treated as a number-like value. Number-like syntax can only parse into number-like values. Number-like values include: durations, ints, floats, ranges, filesizes, binary data, etc. # User-Facing Changes BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE Just making sure we see this for release notes 😅 This breaks any and all numberlike values that were treated as strings before. Example, we used to allow `3,` as a bare word. Anything like this would now require quotes or backticks to be treated as a string or bare word, respectively. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-28 08:31:38 +02:00
pub fn parse_numberlike_expr(
working_set: &mut StateWorkingSet,
span: Span,
shape: &SyntaxShape,
expand_aliases_denylist: &[usize],
) -> (Expression, Option<ParseError>) {
let bytes = working_set.get_span_contents(span);
match shape {
SyntaxShape::Binary => parse_binary(working_set, span),
SyntaxShape::Number => parse_number(bytes, span),
SyntaxShape::Decimal => parse_float(bytes, span),
SyntaxShape::Int => parse_int(bytes, span),
SyntaxShape::Duration => parse_duration(working_set, span),
SyntaxShape::DateTime => parse_datetime(working_set, span),
SyntaxShape::Filesize => parse_filesize(working_set, span),
SyntaxShape::Range => parse_range(working_set, span, expand_aliases_denylist),
SyntaxShape::CellPath => parse_simple_cell_path(working_set, span, expand_aliases_denylist),
SyntaxShape::String => (
garbage(span),
give better error when a numberlike is used for strings (#8706) # Description Before: ``` × Type mismatch during operation. ╭─[source:1:1] 1 │ def 7zup [] {} · ──┬─ · ╰── expected string ╰──── ``` Now: ``` × Type mismatch during operation. ╭─[source:1:1] 1 │ def 7zup [] {} · ──┬─ · ╰── expected string, found number-like value (hint: use quotes or backticks) ╰──── ``` # User-Facing Changes _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- crates/nu-utils/standard_library/tests.nu` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-04-01 21:48:45 +02:00
Some(ParseError::Mismatch(
"string".into(),
"number-like value (hint: use quotes or backticks)".into(),
span,
)),
),
Require that values that look like numbers parse as numberlike (#8635) # Description Require that any value that looks like it might be a number (starts with a digit, or a '-' + digit, or a '+' + digits, or a special form float like `-inf`, `inf`, or `NaN`) must now be treated as a number-like value. Number-like syntax can only parse into number-like values. Number-like values include: durations, ints, floats, ranges, filesizes, binary data, etc. # User-Facing Changes BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE Just making sure we see this for release notes 😅 This breaks any and all numberlike values that were treated as strings before. Example, we used to allow `3,` as a bare word. Anything like this would now require quotes or backticks to be treated as a string or bare word, respectively. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-28 08:31:38 +02:00
SyntaxShape::Any => {
if bytes == b"0b" {
// FIXME: having to work around this filesize that also looks like a binary value
parse_filesize(working_set, span)
} else if bytes.starts_with(b"0x[")
|| bytes.starts_with(b"0b[")
|| bytes.starts_with(b"0o[")
{
parse_binary(working_set, span)
} else if bytes.starts_with(b"0x")
|| bytes.starts_with(b"0b")
|| bytes.starts_with(b"0o")
{
parse_int(bytes, span)
} else {
for shape in &[
SyntaxShape::Range,
SyntaxShape::Int,
SyntaxShape::Binary,
SyntaxShape::Filesize,
SyntaxShape::Duration,
SyntaxShape::DateTime, //FIXME requires 3 failed conversion attempts before failing
SyntaxShape::Number,
] {
let (result, err) =
parse_value(working_set, span, shape, expand_aliases_denylist);
if err.is_none() {
return (result, err);
}
}
(
garbage(span),
Some(ParseError::Expected(
"number-like value (int, float, date, etc)".into(),
span,
)),
)
}
}
_ => (
garbage(span),
Some(ParseError::Expected(
"number-like value (int, float, date, etc)".into(),
span,
)),
),
}
}
pub fn parse_brace_expr(
working_set: &mut StateWorkingSet,
span: Span,
shape: &SyntaxShape,
expand_aliases_denylist: &[usize],
) -> (Expression, Option<ParseError>) {
// Try to detect what kind of value we're about to parse
// FIXME: In the future, we should work over the token stream so we only have to do this once
// before parsing begins
// FIXME: we're still using the shape because we rely on it to know how to handle syntax where
// the parse is ambiguous. We'll need to update the parts of the grammar where this is ambiguous
// and then revisit the parsing.
if span.end <= (span.start + 1) {
return (
Expression::garbage(span),
Some(ParseError::Expected(
format!("non-block value: {shape}"),
span,
)),
);
}
let bytes = working_set.get_span_contents(Span::new(span.start + 1, span.end - 1));
let (tokens, _) = lex(bytes, span.start + 1, &[b'\r', b'\n', b'\t'], &[b':'], true);
let second_token = tokens
.get(0)
.map(|token| working_set.get_span_contents(token.span));
let second_token_contents = tokens.get(0).map(|token| token.contents);
let third_token = tokens
.get(1)
.map(|token| working_set.get_span_contents(token.span));
if matches!(second_token, None) {
// If we're empty, that means an empty record or closure
Restrict closure expression to be something like `{|| ...}` (#8290) # Description As title, closes: #7921 closes: #8273 # User-Facing Changes when define a closure without pipe, nushell will raise error for now: ``` ❯ let x = {ss ss} Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #2:1:1] 1 │ let x = {ss ss} · ───┬─── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `any`, `each`, `all`, `where` command accepts closure, it forces user input closure like `{||`, or parse error will returned. ``` ❯ {major:2, minor:1, patch:4} | values | each { into string } Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #4:1:1] 1 │ {major:2, minor:1, patch:4} | values | each { into string } · ───────┬─────── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `with-env`, `do`, `def`, `try` are special, they still remain the same, although it says that it accepts a closure, but they don't need to be written like `{||`, it's more likely a block but can capture variable outside of scope: ``` ❯ def test [input] { echo [0 1 2] | do { do { echo $input } } }; test aaa aaa ``` Just realize that It's a big breaking change, we need to update config and scripts... # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-17 13:36:28 +01:00
if matches!(shape, SyntaxShape::Closure(None)) {
parse_closure_expression(working_set, shape, span, expand_aliases_denylist, false)
} else if matches!(shape, SyntaxShape::Closure(Some(_))) {
parse_closure_expression(working_set, shape, span, expand_aliases_denylist, true)
} else if matches!(shape, SyntaxShape::Block) {
parse_block_expression(working_set, span, expand_aliases_denylist)
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
} else if matches!(shape, SyntaxShape::MatchBlock) {
parse_match_block_expression(working_set, span, expand_aliases_denylist)
} else {
parse_record(working_set, span, expand_aliases_denylist)
}
} else if matches!(second_token_contents, Some(TokenContents::Pipe))
|| matches!(second_token_contents, Some(TokenContents::PipePipe))
{
Restrict closure expression to be something like `{|| ...}` (#8290) # Description As title, closes: #7921 closes: #8273 # User-Facing Changes when define a closure without pipe, nushell will raise error for now: ``` ❯ let x = {ss ss} Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #2:1:1] 1 │ let x = {ss ss} · ───┬─── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `any`, `each`, `all`, `where` command accepts closure, it forces user input closure like `{||`, or parse error will returned. ``` ❯ {major:2, minor:1, patch:4} | values | each { into string } Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #4:1:1] 1 │ {major:2, minor:1, patch:4} | values | each { into string } · ───────┬─────── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `with-env`, `do`, `def`, `try` are special, they still remain the same, although it says that it accepts a closure, but they don't need to be written like `{||`, it's more likely a block but can capture variable outside of scope: ``` ❯ def test [input] { echo [0 1 2] | do { do { echo $input } } }; test aaa aaa ``` Just realize that It's a big breaking change, we need to update config and scripts... # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-17 13:36:28 +01:00
parse_closure_expression(working_set, shape, span, expand_aliases_denylist, true)
} else if matches!(third_token, Some(b":")) {
parse_full_cell_path(working_set, None, span, expand_aliases_denylist)
Restrict closure expression to be something like `{|| ...}` (#8290) # Description As title, closes: #7921 closes: #8273 # User-Facing Changes when define a closure without pipe, nushell will raise error for now: ``` ❯ let x = {ss ss} Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #2:1:1] 1 │ let x = {ss ss} · ───┬─── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `any`, `each`, `all`, `where` command accepts closure, it forces user input closure like `{||`, or parse error will returned. ``` ❯ {major:2, minor:1, patch:4} | values | each { into string } Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #4:1:1] 1 │ {major:2, minor:1, patch:4} | values | each { into string } · ───────┬─────── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `with-env`, `do`, `def`, `try` are special, they still remain the same, although it says that it accepts a closure, but they don't need to be written like `{||`, it's more likely a block but can capture variable outside of scope: ``` ❯ def test [input] { echo [0 1 2] | do { do { echo $input } } }; test aaa aaa ``` Just realize that It's a big breaking change, we need to update config and scripts... # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-17 13:36:28 +01:00
} else if matches!(shape, SyntaxShape::Closure(None)) {
parse_closure_expression(working_set, shape, span, expand_aliases_denylist, false)
} else if matches!(shape, SyntaxShape::Closure(Some(_))) || matches!(shape, SyntaxShape::Any) {
parse_closure_expression(working_set, shape, span, expand_aliases_denylist, true)
} else if matches!(shape, SyntaxShape::Block) {
parse_block_expression(working_set, span, expand_aliases_denylist)
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
} else if matches!(shape, SyntaxShape::MatchBlock) {
parse_match_block_expression(working_set, span, expand_aliases_denylist)
} else {
(
Expression::garbage(span),
Some(ParseError::Expected(
format!("non-block value: {shape}"),
span,
)),
)
}
}
2021-09-02 10:25:22 +02:00
pub fn parse_string_interpolation(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
#[derive(PartialEq, Eq, Debug)]
enum InterpolationMode {
String,
Expression,
}
let mut error = None;
2021-07-30 05:26:06 +02:00
2021-09-02 10:25:22 +02:00
let contents = working_set.get_span_contents(span);
2021-07-30 05:26:06 +02:00
let mut double_quote = false;
let (start, end) = if contents.starts_with(b"$\"") {
double_quote = true;
let end = if contents.ends_with(b"\"") && contents.len() > 2 {
span.end - 1
} else {
span.end
};
(span.start + 2, end)
} else if contents.starts_with(b"$'") {
let end = if contents.ends_with(b"'") && contents.len() > 2 {
span.end - 1
} else {
span.end
};
(span.start + 2, end)
2021-09-02 10:25:22 +02:00
} else {
(span.start, span.end)
2021-09-02 10:25:22 +02:00
};
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let inner_span = Span::new(start, end);
2021-09-02 10:25:22 +02:00
let contents = working_set.get_span_contents(inner_span).to_vec();
let mut output = vec![];
let mut mode = InterpolationMode::String;
let mut token_start = start;
let mut delimiter_stack = vec![];
2021-09-02 10:25:22 +02:00
let mut consecutive_backslashes: usize = 0;
2021-09-02 10:25:22 +02:00
let mut b = start;
while b != end {
let current_byte = contents[b - start];
if mode == InterpolationMode::String {
let preceding_consecutive_backslashes = consecutive_backslashes;
let is_backslash = current_byte == b'\\';
consecutive_backslashes = if is_backslash {
preceding_consecutive_backslashes + 1
} else {
0
};
if current_byte == b'(' && (!double_quote || preceding_consecutive_backslashes % 2 == 0)
{
mode = InterpolationMode::Expression;
if token_start < b {
let span = Span::new(token_start, b);
let str_contents = working_set.get_span_contents(span);
let str_contents = if double_quote {
let (str_contents, err) = unescape_string(str_contents, span);
error = error.or(err);
str_contents
} else {
str_contents.to_vec()
};
output.push(Expression {
expr: Expr::String(String::from_utf8_lossy(&str_contents).to_string()),
span,
ty: Type::String,
custom_completion: None,
});
token_start = b;
}
}
}
if mode == InterpolationMode::Expression {
let byte = current_byte;
if let Some(b'\'') = delimiter_stack.last() {
if byte == b'\'' {
delimiter_stack.pop();
}
} else if let Some(b'"') = delimiter_stack.last() {
if byte == b'"' {
delimiter_stack.pop();
}
} else if let Some(b'`') = delimiter_stack.last() {
if byte == b'`' {
delimiter_stack.pop();
}
} else if byte == b'\'' {
delimiter_stack.push(b'\'')
} else if byte == b'"' {
delimiter_stack.push(b'"');
} else if byte == b'`' {
delimiter_stack.push(b'`')
} else if byte == b'(' {
delimiter_stack.push(b')');
} else if byte == b')' {
if let Some(b')') = delimiter_stack.last() {
delimiter_stack.pop();
}
if delimiter_stack.is_empty() {
mode = InterpolationMode::String;
2021-09-02 10:25:22 +02:00
if token_start < b {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let span = Span::new(token_start, b + 1);
2021-07-30 05:26:06 +02:00
let (expr, err) =
parse_full_cell_path(working_set, None, span, expand_aliases_denylist);
error = error.or(err);
output.push(expr);
}
2021-09-02 10:25:22 +02:00
token_start = b + 1;
continue;
2021-07-30 05:26:06 +02:00
}
}
}
2021-09-02 10:25:22 +02:00
b += 1;
}
2021-07-30 05:26:06 +02:00
2021-09-02 10:25:22 +02:00
match mode {
InterpolationMode::String => {
if token_start < end {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let span = Span::new(token_start, end);
2021-09-02 10:25:22 +02:00
let str_contents = working_set.get_span_contents(span);
let str_contents = if double_quote {
let (str_contents, err) = unescape_string(str_contents, span);
error = error.or(err);
str_contents
} else {
str_contents.to_vec()
};
2021-09-02 10:25:22 +02:00
output.push(Expression {
expr: Expr::String(String::from_utf8_lossy(&str_contents).to_string()),
2021-07-30 05:26:06 +02:00
span,
ty: Type::String,
custom_completion: None,
2021-09-02 10:25:22 +02:00
});
}
}
InterpolationMode::Expression => {
if token_start < end {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let span = Span::new(token_start, end);
2021-09-02 10:25:22 +02:00
let (expr, err) =
parse_full_cell_path(working_set, None, span, expand_aliases_denylist);
2021-09-02 10:25:22 +02:00
error = error.or(err);
output.push(expr);
}
2021-07-30 05:26:06 +02:00
}
2021-07-24 07:57:17 +02:00
}
2021-07-02 09:15:30 +02:00
(
Expression {
expr: Expr::StringInterpolation(output),
span,
ty: Type::String,
custom_completion: None,
},
error,
)
2021-09-02 10:25:22 +02:00
}
2021-07-24 07:57:17 +02:00
2021-09-02 10:25:22 +02:00
pub fn parse_variable_expr(
working_set: &mut StateWorkingSet,
span: Span,
) -> (Expression, Option<ParseError>) {
let contents = working_set.get_span_contents(span);
2022-03-19 20:12:10 +01:00
if contents == b"$nothing" {
return (
Expression {
expr: Expr::Nothing,
span,
ty: Type::Nothing,
custom_completion: None,
},
None,
);
2021-10-29 20:15:17 +02:00
} else if contents == b"$nu" {
return (
Expression {
2021-11-02 04:08:05 +01:00
expr: Expr::Var(nu_protocol::NU_VARIABLE_ID),
span,
ty: Type::Any,
2021-11-02 04:08:05 +01:00
custom_completion: None,
},
None,
);
} else if contents == b"$in" {
return (
Expression {
expr: Expr::Var(nu_protocol::IN_VARIABLE_ID),
span,
ty: Type::Any,
custom_completion: None,
},
None,
);
} else if contents == b"$env" {
return (
Expression {
expr: Expr::Var(nu_protocol::ENV_VARIABLE_ID),
span,
ty: Type::Any,
custom_completion: None,
},
None,
);
2021-09-02 10:25:22 +02:00
}
let (id, err) = parse_variable(working_set, span);
if err.is_none() {
if let Some(id) = id {
(
2021-07-02 09:15:30 +02:00
Expression {
2021-09-02 10:25:22 +02:00
expr: Expr::Var(id),
2021-07-02 09:15:30 +02:00
span,
2022-03-09 10:42:19 +01:00
ty: working_set.get_variable(id).ty.clone(),
custom_completion: None,
2021-07-02 09:15:30 +02:00
},
None,
2021-09-02 10:25:22 +02:00
)
} else {
(garbage(span), Some(ParseError::VariableNotFound(span)))
2021-07-02 09:15:30 +02:00
}
2021-09-02 10:25:22 +02:00
} else {
(garbage(span), err)
}
}
2021-07-02 09:15:30 +02:00
2021-10-02 04:59:11 +02:00
pub fn parse_cell_path(
working_set: &mut StateWorkingSet,
tokens: impl Iterator<Item = Token>,
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 04:50:58 +01:00
expect_dot: bool,
expand_aliases_denylist: &[usize],
2021-10-02 04:59:11 +02:00
) -> (Vec<PathMember>, Option<ParseError>) {
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 04:50:58 +01:00
enum TokenType {
Dot, // .
QuestionOrDot, // ? or .
PathMember, // an int or string, like `1` or `foo`
}
// Parsing a cell path is essentially a state machine, and this is the state
let mut expected_token = if expect_dot {
TokenType::Dot
} else {
TokenType::PathMember
};
2021-10-02 04:59:11 +02:00
let mut error = None;
let mut tail = vec![];
for path_element in tokens {
let bytes = working_set.get_span_contents(path_element.span);
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 04:50:58 +01:00
match expected_token {
TokenType::Dot => {
if bytes.len() != 1 || bytes[0] != b'.' {
return (
tail,
Some(ParseError::Expected('.'.into(), path_element.span)),
);
}
expected_token = TokenType::PathMember;
2021-10-02 04:59:11 +02:00
}
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 04:50:58 +01:00
TokenType::QuestionOrDot => {
if bytes.len() == 1 && bytes[0] == b'.' {
expected_token = TokenType::PathMember;
} else if bytes.len() == 1 && bytes[0] == b'?' {
if let Some(last) = tail.last_mut() {
match last {
PathMember::String {
ref mut optional, ..
} => *optional = true,
PathMember::Int {
ref mut optional, ..
} => *optional = true,
}
}
expected_token = TokenType::Dot;
} else {
return (
tail,
Some(ParseError::Expected(". or ?".into(), path_element.span)),
);
}
}
TokenType::PathMember => {
match parse_int(bytes, path_element.span) {
(
2021-10-02 04:59:11 +02:00
Expression {
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 04:50:58 +01:00
expr: Expr::Int(val),
2021-10-02 04:59:11 +02:00
span,
..
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 04:50:58 +01:00
},
None,
) => tail.push(PathMember::Int {
val: val as usize,
span,
optional: false,
}),
_ => {
let (result, err) =
parse_string(working_set, path_element.span, expand_aliases_denylist);
error = error.or(err);
match result {
Expression {
expr: Expr::String(string),
span,
..
} => {
tail.push(PathMember::String {
val: string,
span,
optional: false,
});
}
_ => {
return (
tail,
Some(ParseError::Expected("string".into(), path_element.span)),
);
}
2021-10-02 04:59:11 +02:00
}
}
}
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 04:50:58 +01:00
expected_token = TokenType::QuestionOrDot;
2021-10-02 04:59:11 +02:00
}
}
}
(tail, error)
}
pub fn parse_simple_cell_path(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
) -> (Expression, Option<ParseError>) {
let source = working_set.get_span_contents(span);
let mut error = None;
let (tokens, err) = lex(source, span.start, &[b'\n', b'\r'], &[b'.', b'?'], true);
error = error.or(err);
let tokens = tokens.into_iter().peekable();
let (cell_path, err) = parse_cell_path(working_set, tokens, false, expand_aliases_denylist);
error = error.or(err);
(
Expression {
expr: Expr::CellPath(CellPath { members: cell_path }),
span,
ty: Type::CellPath,
custom_completion: None,
},
error,
)
}
2021-09-26 20:39:19 +02:00
pub fn parse_full_cell_path(
2021-09-02 10:25:22 +02:00
working_set: &mut StateWorkingSet,
2021-09-09 23:47:20 +02:00
implicit_head: Option<VarId>,
2021-09-02 10:25:22 +02:00
span: Span,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
trace!("parsing: full cell path");
2021-09-26 20:39:19 +02:00
let full_cell_span = span;
2021-09-07 00:02:24 +02:00
let source = working_set.get_span_contents(span);
2021-09-02 10:25:22 +02:00
let mut error = None;
2021-07-08 08:19:38 +02:00
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 04:50:58 +01:00
let (tokens, err) = lex(source, span.start, &[b'\n', b'\r'], &[b'.', b'?'], true);
2021-09-07 00:02:24 +02:00
error = error.or(err);
2021-09-02 10:25:22 +02:00
2021-09-09 23:47:20 +02:00
let mut tokens = tokens.into_iter().peekable();
if let Some(head) = tokens.peek() {
2021-09-07 00:02:24 +02:00
let bytes = working_set.get_span_contents(head.span);
2021-10-02 04:59:11 +02:00
let (head, expect_dot) = if bytes.starts_with(b"(") {
2022-01-01 22:42:50 +01:00
trace!("parsing: paren-head of full cell path");
let head_span = head.span;
2021-09-07 00:02:24 +02:00
let mut start = head.span.start;
let mut end = head.span.end;
2021-07-08 08:19:38 +02:00
2021-09-07 00:02:24 +02:00
if bytes.starts_with(b"(") {
start += 1;
}
if bytes.ends_with(b")") {
end -= 1;
} else {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
error =
error.or_else(|| Some(ParseError::Unclosed(")".into(), Span::new(end, end))));
2021-09-07 00:02:24 +02:00
}
2021-07-02 09:15:30 +02:00
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let span = Span::new(start, end);
2021-07-02 09:15:30 +02:00
2021-09-07 00:02:24 +02:00
let source = working_set.get_span_contents(span);
2021-07-02 09:15:30 +02:00
2021-11-21 19:13:09 +01:00
let (output, err) = lex(source, span.start, &[b'\n', b'\r'], &[], true);
2021-09-07 00:02:24 +02:00
error = error.or(err);
2021-07-02 09:15:30 +02:00
// Creating a Type scope to parse the new block. This will keep track of
// the previous input type found in that block
let (output, err) =
parse_block(working_set, &output, true, expand_aliases_denylist, true);
working_set
.type_scope
.add_type(working_set.type_scope.get_last_output());
let ty = output
.pipelines
.last()
.and_then(|Pipeline { elements, .. }| elements.last())
.map(|element| match element {
PipelineElement::Expression(_, expr)
if matches!(
expr,
Expression {
expr: Expr::BinaryOp(..),
..
}
) =>
{
expr.ty.clone()
}
_ => working_set.type_scope.get_last_output(),
})
.unwrap_or_else(|| working_set.type_scope.get_last_output());
2021-09-07 00:02:24 +02:00
error = error.or(err);
2021-07-02 09:15:30 +02:00
2021-09-07 00:02:24 +02:00
let block_id = working_set.add_block(output);
2021-09-09 23:47:20 +02:00
tokens.next();
2021-09-07 00:02:24 +02:00
2021-09-09 23:47:20 +02:00
(
Expression {
expr: Expr::Subexpression(block_id),
span: head_span,
ty,
custom_completion: None,
2021-09-09 23:47:20 +02:00
},
true,
)
} else if bytes.starts_with(b"[") {
2022-01-01 22:42:50 +01:00
trace!("parsing: table head of full cell path");
let (output, err) =
parse_table_expression(working_set, head.span, expand_aliases_denylist);
error = error.or(err);
tokens.next();
2021-11-11 00:14:00 +01:00
(output, true)
} else if bytes.starts_with(b"{") {
2022-01-01 22:42:50 +01:00
trace!("parsing: record head of full cell path");
let (output, err) = parse_record(working_set, head.span, expand_aliases_denylist);
2021-11-11 00:14:00 +01:00
error = error.or(err);
tokens.next();
(output, true)
2021-09-07 00:02:24 +02:00
} else if bytes.starts_with(b"$") {
2022-01-01 22:42:50 +01:00
trace!("parsing: $variable head of full cell path");
2021-09-07 00:02:24 +02:00
let (out, err) = parse_variable_expr(working_set, head.span);
error = error.or(err);
2021-09-09 23:47:20 +02:00
tokens.next();
(out, true)
} else if let Some(var_id) = implicit_head {
trace!("parsing: implicit head of full cell path");
2021-09-09 23:47:20 +02:00
(
Expression {
expr: Expr::Var(var_id),
span: head.span,
ty: Type::Any,
custom_completion: None,
2021-09-09 23:47:20 +02:00
},
false,
)
2021-09-07 00:02:24 +02:00
} else {
return (
garbage(span),
Some(ParseError::Mismatch(
"variable or subexpression".into(),
String::from_utf8_lossy(bytes).to_string(),
span,
)),
);
};
Optional members in cell paths: Attempt 2 (#8379) This is a follow up from https://github.com/nushell/nushell/pull/7540. Please provide feedback if you have the time! ## Summary This PR lets you use `?` to indicate that a member in a cell path is optional and Nushell should return `null` if that member cannot be accessed. Unlike the previous PR, `?` is now a _postfix_ modifier for cell path members. A cell path of `.foo?.bar` means that `foo` is optional and `bar` is not. `?` does _not_ suppress all errors; it is intended to help in situations where data has "holes", i.e. the data types are correct but something is missing. Type mismatches (like trying to do a string path access on a date) will still fail. ### Record Examples ```bash { foo: 123 }.foo # returns 123 { foo: 123 }.bar # errors { foo: 123 }.bar? # returns null { foo: 123 } | get bar # errors { foo: 123 } | get bar? # returns null { foo: 123 }.bar.baz # errors { foo: 123 }.bar?.baz # errors because `baz` is not present on the result from `bar?` { foo: 123 }.bar.baz? # errors { foo: 123 }.bar?.baz? # returns null ``` ### List Examples ``` 〉[{foo: 1} {foo: 2} {}].foo Error: nu::shell::column_not_found × Cannot find column ╭─[entry #30:1:1] 1 │ [{foo: 1} {foo: 2} {}].foo · ─┬ ─┬─ · │ ╰── cannot find column 'foo' · ╰── value originates here ╰──── 〉[{foo: 1} {foo: 2} {}].foo? ╭───┬───╮ │ 0 │ 1 │ │ 1 │ 2 │ │ 2 │ │ ╰───┴───╯ 〉[{foo: 1} {foo: 2} {}].foo?.2 | describe nothing 〉[a b c].4? | describe nothing 〉[{foo: 1} {foo: 2} {}] | where foo? == 1 ╭───┬─────╮ │ # │ foo │ ├───┼─────┤ │ 0 │ 1 │ ╰───┴─────╯ ``` # Breaking changes 1. Column names with `?` in them now need to be quoted. 2. The `-i`/`--ignore-errors` flag has been removed from `get` and `select` 1. After this PR, most `get` error handling can be done with `?` and/or `try`/`catch`. 4. Cell path accesses like this no longer work without a `?`: ```bash 〉[{a:1 b:2} {a:3}].b.0 2 ``` We had some clever code that was able to recognize that since we only want row `0`, it's OK if other rows are missing column `b`. I removed that because it's tricky to maintain, and now that query needs to be written like: ```bash 〉[{a:1 b:2} {a:3}].b?.0 2 ``` I think the regression is acceptable for now. I plan to do more work in the future to enable streaming of cell path accesses, and when that happens I'll be able to make `.b.0` work again.
2023-03-16 04:50:58 +01:00
let (tail, err) = parse_cell_path(working_set, tokens, expect_dot, expand_aliases_denylist);
2021-10-02 04:59:11 +02:00
error = error.or(err);
2021-09-07 00:02:24 +02:00
(
Expression {
// FIXME: Get the type of the data at the tail using follow_cell_path() (or something)
ty: if !tail.is_empty() {
// Until the aforementioned fix is implemented, this is necessary to allow mutable list upserts
// such as $a.1 = 2 to work correctly.
Type::Any
} else {
head.ty.clone()
},
expr: Expr::FullCellPath(Box::new(FullCellPath { head, tail })),
span: full_cell_span,
custom_completion: None,
},
error,
)
2021-09-07 00:02:24 +02:00
} else {
(garbage(span), error)
}
2021-09-02 10:25:22 +02:00
}
2021-07-02 09:15:30 +02:00
pub fn parse_directory(
working_set: &mut StateWorkingSet,
span: Span,
) -> (Expression, Option<ParseError>) {
let bytes = working_set.get_span_contents(span);
let (token, err) = unescape_unquote_string(bytes, span);
trace!("parsing: directory");
if err.is_none() {
trace!("-- found {}", token);
(
Expression {
expr: Expr::Directory(token),
span,
ty: Type::String,
custom_completion: None,
},
None,
)
} else {
(
garbage(span),
Some(ParseError::Expected("directory".into(), span)),
)
}
}
2021-10-04 21:21:31 +02:00
pub fn parse_filepath(
working_set: &mut StateWorkingSet,
span: Span,
) -> (Expression, Option<ParseError>) {
let bytes = working_set.get_span_contents(span);
let (token, err) = unescape_unquote_string(bytes, span);
2022-01-01 22:42:50 +01:00
trace!("parsing: filepath");
2021-10-04 21:21:31 +02:00
if err.is_none() {
trace!("-- found {}", token);
2021-10-04 21:21:31 +02:00
(
Expression {
expr: Expr::Filepath(token),
2021-10-04 21:21:31 +02:00
span,
ty: Type::String,
custom_completion: None,
},
None,
)
} else {
(
garbage(span),
2021-11-18 20:32:27 +01:00
Some(ParseError::Expected("filepath".into(), span)),
2021-10-04 21:21:31 +02:00
)
}
}
/// Parse a datetime type, eg '2022-02-02'
pub fn parse_datetime(
working_set: &mut StateWorkingSet,
span: Span,
) -> (Expression, Option<ParseError>) {
trace!("parsing: datetime");
let bytes = working_set.get_span_contents(span);
if bytes.is_empty() || !bytes[0].is_ascii_digit() {
return (
garbage(span),
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
Some(ParseError::Expected("datetime".into(), span)),
);
}
let token = String::from_utf8_lossy(bytes).to_string();
if let Ok(datetime) = chrono::DateTime::parse_from_rfc3339(&token) {
return (
Expression {
expr: Expr::DateTime(datetime),
span,
ty: Type::Date,
custom_completion: None,
},
None,
);
}
// Just the date
let just_date = token.clone() + "T00:00:00+00:00";
if let Ok(datetime) = chrono::DateTime::parse_from_rfc3339(&just_date) {
return (
Expression {
expr: Expr::DateTime(datetime),
span,
ty: Type::Date,
custom_completion: None,
},
None,
);
}
// Date and time, assume UTC
let datetime = token + "+00:00";
if let Ok(datetime) = chrono::DateTime::parse_from_rfc3339(&datetime) {
return (
Expression {
expr: Expr::DateTime(datetime),
span,
ty: Type::Date,
custom_completion: None,
},
None,
);
}
(
garbage(span),
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
Some(ParseError::Expected("datetime".into(), span)),
)
}
2021-10-05 04:27:39 +02:00
/// Parse a duration type, eg '10day'
pub fn parse_duration(
working_set: &StateWorkingSet,
2021-10-05 04:27:39 +02:00
span: Span,
) -> (Expression, Option<ParseError>) {
2022-01-01 22:42:50 +01:00
trace!("parsing: duration");
let bytes = working_set.get_span_contents(span);
match parse_duration_bytes(bytes, span) {
Some(expression) => (expression, None),
None => (
garbage(span),
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
Some(ParseError::Expected(
"duration with valid units".into(),
span,
)),
),
}
}
// Borrowed from libm at https://github.com/rust-lang/libm/blob/master/src/math/modf.rs
pub fn modf(x: f64) -> (f64, f64) {
let rv2: f64;
let mut u = x.to_bits();
let e = ((u >> 52 & 0x7ff) as i32) - 0x3ff;
/* no fractional part */
if e >= 52 {
rv2 = x;
if e == 0x400 && (u << 12) != 0 {
/* nan */
return (x, rv2);
2021-10-05 04:27:39 +02:00
}
u &= 1 << 63;
return (f64::from_bits(u), rv2);
2021-10-05 04:27:39 +02:00
}
/* no integral part*/
if e < 0 {
u &= 1 << 63;
rv2 = f64::from_bits(u);
return (x, rv2);
}
let mask = ((!0) >> 12) >> e;
if (u & mask) == 0 {
rv2 = x;
u &= 1 << 63;
return (f64::from_bits(u), rv2);
}
u &= !mask;
rv2 = f64::from_bits(u);
(x - rv2, rv2)
}
2021-10-05 04:27:39 +02:00
pub fn parse_duration_bytes(num_with_unit_bytes: &[u8], span: Span) -> Option<Expression> {
if num_with_unit_bytes.is_empty()
|| (!num_with_unit_bytes[0].is_ascii_digit() && num_with_unit_bytes[0] != b'-')
{
return None;
}
let num_with_unit = String::from_utf8_lossy(num_with_unit_bytes).to_string();
2021-10-05 04:27:39 +02:00
let unit_groups = [
Allow parsing of mu (µ) character for durations (issue #8614) (#8647) # Description This is to resolve the issue [8614](https://github.com/nushell/nushell/issues/8614). It allows the parsing of the mu (µ) character for durations, so you can type `10µs`, and it correctly outputs, whilst maintaining the current `us` parsing as well. It also forces `durations` to be entered in lower case. ![image](https://user-images.githubusercontent.com/44570273/228217360-57ebc902-cec5-4683-910e-0b18fbe160b1.png) (The bottom one `1sec | into duration --convert us` looks like an existing bug, where converting to `us` outputs `us` rather than `µs`) # User-Facing Changes Allows the user to parse durations in µs Forces `durations` to be entered in lower case rather than any case, and will error if not in lower case. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-03-30 17:35:35 +02:00
(Unit::Nanosecond, "ns", None),
(Unit::Microsecond, "us", Some((Unit::Nanosecond, 1000))),
(
// µ Micro Sign
Unit::Microsecond,
"\u{00B5}s",
Some((Unit::Nanosecond, 1000)),
),
(
// μ Greek small letter Mu
Unit::Microsecond,
"\u{03BC}s",
Some((Unit::Nanosecond, 1000)),
),
(Unit::Millisecond, "ms", Some((Unit::Microsecond, 1000))),
(Unit::Second, "sec", Some((Unit::Millisecond, 1000))),
(Unit::Minute, "min", Some((Unit::Second, 60))),
(Unit::Hour, "hr", Some((Unit::Minute, 60))),
(Unit::Day, "day", Some((Unit::Minute, 1440))),
(Unit::Week, "wk", Some((Unit::Day, 7))),
2021-10-05 04:27:39 +02:00
];
Allow parsing of mu (µ) character for durations (issue #8614) (#8647) # Description This is to resolve the issue [8614](https://github.com/nushell/nushell/issues/8614). It allows the parsing of the mu (µ) character for durations, so you can type `10µs`, and it correctly outputs, whilst maintaining the current `us` parsing as well. It also forces `durations` to be entered in lower case. ![image](https://user-images.githubusercontent.com/44570273/228217360-57ebc902-cec5-4683-910e-0b18fbe160b1.png) (The bottom one `1sec | into duration --convert us` looks like an existing bug, where converting to `us` outputs `us` rather than `µs`) # User-Facing Changes Allows the user to parse durations in µs Forces `durations` to be entered in lower case rather than any case, and will error if not in lower case. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-03-30 17:35:35 +02:00
if let Some(unit) = unit_groups.iter().find(|&x| num_with_unit.ends_with(x.1)) {
let mut lhs = num_with_unit;
Allow parsing of mu (µ) character for durations (issue #8614) (#8647) # Description This is to resolve the issue [8614](https://github.com/nushell/nushell/issues/8614). It allows the parsing of the mu (µ) character for durations, so you can type `10µs`, and it correctly outputs, whilst maintaining the current `us` parsing as well. It also forces `durations` to be entered in lower case. ![image](https://user-images.githubusercontent.com/44570273/228217360-57ebc902-cec5-4683-910e-0b18fbe160b1.png) (The bottom one `1sec | into duration --convert us` looks like an existing bug, where converting to `us` outputs `us` rather than `µs`) # User-Facing Changes Allows the user to parse durations in µs Forces `durations` to be entered in lower case rather than any case, and will error if not in lower case. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-03-30 17:35:35 +02:00
for _ in 0..unit.1.chars().count() {
2021-10-05 04:27:39 +02:00
lhs.pop();
}
let (decimal_part, number_part) = modf(match lhs.parse::<f64>() {
Ok(x) => x,
Err(_) => return None,
});
let (num, unit_to_use) = match unit.2 {
Some(unit_to_convert_to) => (
Some(
((number_part * unit_to_convert_to.1 as f64)
+ (decimal_part * unit_to_convert_to.1 as f64)) as i64,
),
unit_to_convert_to.0,
),
None => (Some(number_part as i64), unit.0),
2021-10-05 04:27:39 +02:00
};
if let Some(x) = num {
2022-01-01 22:42:50 +01:00
trace!("-- found {} {:?}", x, unit_to_use);
2021-10-05 04:27:39 +02:00
let lhs_span = Span::new(span.start, span.start + lhs.len());
let unit_span = Span::new(span.start + lhs.len(), span.end);
return Some(Expression {
expr: Expr::ValueWithUnit(
Box::new(Expression {
expr: Expr::Int(x),
span: lhs_span,
ty: Type::Number,
custom_completion: None,
}),
Spanned {
item: unit_to_use,
span: unit_span,
},
),
span,
ty: Type::Duration,
custom_completion: None,
});
2021-10-05 04:27:39 +02:00
}
}
None
2021-10-05 04:27:39 +02:00
}
/// Parse a unit type, eg '10kb'
pub fn parse_filesize(
working_set: &StateWorkingSet,
2021-10-05 04:27:39 +02:00
span: Span,
) -> (Expression, Option<ParseError>) {
trace!("parsing: filesize");
2021-10-05 04:27:39 +02:00
let bytes = working_set.get_span_contents(span);
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
//todo: parse_filesize_bytes should distinguish between not-that-type and syntax error in units
match parse_filesize_bytes(bytes, span) {
Some(expression) => (expression, None),
None => (
garbage(span),
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
Some(ParseError::Expected(
"filesize with valid units".into(),
span,
)),
),
}
}
pub fn parse_filesize_bytes(num_with_unit_bytes: &[u8], span: Span) -> Option<Expression> {
if num_with_unit_bytes.is_empty()
|| (!num_with_unit_bytes[0].is_ascii_digit() && num_with_unit_bytes[0] != b'-')
{
return None;
}
let num_with_unit = String::from_utf8_lossy(num_with_unit_bytes).to_string();
let uppercase_num_with_unit = num_with_unit.to_uppercase();
2021-10-05 04:27:39 +02:00
let unit_groups = [
(Unit::Kilobyte, "KB", Some((Unit::Byte, 1000))),
(Unit::Megabyte, "MB", Some((Unit::Kilobyte, 1000))),
(Unit::Gigabyte, "GB", Some((Unit::Megabyte, 1000))),
(Unit::Terabyte, "TB", Some((Unit::Gigabyte, 1000))),
(Unit::Petabyte, "PB", Some((Unit::Terabyte, 1000))),
(Unit::Exabyte, "EB", Some((Unit::Petabyte, 1000))),
(Unit::Zettabyte, "ZB", Some((Unit::Exabyte, 1000))),
2021-10-05 04:27:39 +02:00
(Unit::Kibibyte, "KIB", Some((Unit::Byte, 1024))),
(Unit::Mebibyte, "MIB", Some((Unit::Kibibyte, 1024))),
(Unit::Gibibyte, "GIB", Some((Unit::Mebibyte, 1024))),
(Unit::Tebibyte, "TIB", Some((Unit::Gibibyte, 1024))),
(Unit::Pebibyte, "PIB", Some((Unit::Tebibyte, 1024))),
(Unit::Exbibyte, "EIB", Some((Unit::Pebibyte, 1024))),
(Unit::Zebibyte, "ZIB", Some((Unit::Exbibyte, 1024))),
2021-10-05 04:27:39 +02:00
(Unit::Byte, "B", None),
];
if let Some(unit) = unit_groups
.iter()
.find(|&x| uppercase_num_with_unit.ends_with(x.1))
{
let mut lhs = num_with_unit;
2021-10-05 04:27:39 +02:00
for _ in 0..unit.1.len() {
lhs.pop();
}
let (decimal_part, number_part) = modf(match lhs.parse::<f64>() {
Ok(x) => x,
Err(_) => return None,
});
let (num, unit_to_use) = match unit.2 {
Some(unit_to_convert_to) => (
Some(
((number_part * unit_to_convert_to.1 as f64)
+ (decimal_part * unit_to_convert_to.1 as f64)) as i64,
),
unit_to_convert_to.0,
),
None => (Some(number_part as i64), unit.0),
2021-10-05 04:27:39 +02:00
};
if let Some(x) = num {
2022-01-01 22:42:50 +01:00
trace!("-- found {} {:?}", x, unit_to_use);
2021-10-05 04:27:39 +02:00
let lhs_span = Span::new(span.start, span.start + lhs.len());
let unit_span = Span::new(span.start + lhs.len(), span.end);
return Some(Expression {
expr: Expr::ValueWithUnit(
Box::new(Expression {
expr: Expr::Int(x),
span: lhs_span,
ty: Type::Number,
custom_completion: None,
}),
Spanned {
item: unit_to_use,
span: unit_span,
},
),
span,
ty: Type::Filesize,
custom_completion: None,
});
2021-10-05 04:27:39 +02:00
}
}
None
2021-10-05 04:27:39 +02:00
}
2021-10-04 21:21:31 +02:00
pub fn parse_glob_pattern(
working_set: &mut StateWorkingSet,
span: Span,
) -> (Expression, Option<ParseError>) {
let bytes = working_set.get_span_contents(span);
let (token, err) = unescape_unquote_string(bytes, span);
trace!("parsing: glob pattern");
2021-10-04 21:21:31 +02:00
if err.is_none() {
2022-01-01 22:42:50 +01:00
trace!("-- found {}", token);
2021-10-04 21:21:31 +02:00
(
Expression {
expr: Expr::GlobPattern(token),
2021-10-04 21:21:31 +02:00
span,
ty: Type::String,
custom_completion: None,
},
None,
)
} else {
(
garbage(span),
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
Some(ParseError::Expected("glob pattern string".into(), span)),
2021-10-04 21:21:31 +02:00
)
}
}
pub fn unescape_string(bytes: &[u8], span: Span) -> (Vec<u8>, Option<ParseError>) {
let mut output = Vec::new();
let mut idx = 0;
let mut err = None;
'us_loop: while idx < bytes.len() {
if bytes[idx] == b'\\' {
// We're in an escape
idx += 1;
match bytes.get(idx) {
Some(b'"') => {
output.push(b'"');
idx += 1;
}
Some(b'\'') => {
output.push(b'\'');
idx += 1;
}
Some(b'\\') => {
output.push(b'\\');
idx += 1;
}
Some(b'/') => {
output.push(b'/');
idx += 1;
}
Some(b'(') => {
output.push(b'(');
idx += 1;
}
Some(b')') => {
output.push(b')');
idx += 1;
}
Some(b'{') => {
output.push(b'{');
idx += 1;
}
Some(b'}') => {
output.push(b'}');
idx += 1;
}
Some(b'$') => {
output.push(b'$');
idx += 1;
}
Some(b'^') => {
output.push(b'^');
idx += 1;
}
Some(b'#') => {
output.push(b'#');
idx += 1;
}
Some(b'|') => {
output.push(b'|');
idx += 1;
}
Some(b'~') => {
output.push(b'~');
idx += 1;
}
Some(b'a') => {
output.push(0x7);
idx += 1;
}
Some(b'b') => {
output.push(0x8);
idx += 1;
}
Some(b'e') => {
output.push(0x1b);
idx += 1;
}
Some(b'f') => {
output.push(0xc);
idx += 1;
}
Some(b'n') => {
output.push(b'\n');
idx += 1;
}
Some(b'r') => {
output.push(b'\r');
idx += 1;
}
Some(b't') => {
output.push(b'\t');
idx += 1;
}
Some(b'u') => {
let mut digits = String::with_capacity(10);
let mut cur_idx = idx + 1; // index of first beyond current end of token
if let Some(b'{') = bytes.get(idx + 1) {
cur_idx = idx + 2;
loop {
match bytes.get(cur_idx) {
Some(b'}') => {
cur_idx += 1;
break;
}
Some(c) => {
digits.push(*c as char);
cur_idx += 1;
}
_ => {
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
err = Some(ParseError::InvalidLiteral(
"missing '}' for unicode escape '\\u{X...}'".into(),
"string".into(),
Span::new(span.start + idx, span.end),
));
break 'us_loop;
}
}
}
}
if (1..=6).contains(&digits.len()) {
let int = u32::from_str_radix(&digits, 16);
if let Ok(int) = int {
if int <= 0x10ffff {
let result = char::from_u32(int);
if let Some(result) = result {
let mut buffer = vec![0; 4];
let result = result.encode_utf8(&mut buffer);
for elem in result.bytes() {
output.push(elem);
}
idx = cur_idx;
continue 'us_loop;
}
}
}
}
// fall through -- escape not accepted above, must be error.
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
err = Some(ParseError::InvalidLiteral(
2023-03-10 22:20:31 +01:00
"invalid unicode escape '\\u{X...}', must be 1-6 hex digits, max value 10FFFF".into(),
"string".into(),
Span::new(span.start + idx, span.end),
));
break 'us_loop;
}
_ => {
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
err = Some(ParseError::InvalidLiteral(
"unrecognized escape after '\\'".into(),
"string".into(),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(span.start + idx, span.end),
));
break 'us_loop;
}
}
} else {
output.push(bytes[idx]);
idx += 1;
}
}
(output, err)
}
pub fn unescape_unquote_string(bytes: &[u8], span: Span) -> (String, Option<ParseError>) {
if bytes.starts_with(b"\"") {
// Needs unescaping
let bytes = trim_quotes(bytes);
let (bytes, err) = unescape_string(bytes, span);
if let Ok(token) = String::from_utf8(bytes) {
(token, err)
} else {
(
String::new(),
Some(ParseError::Expected("string".into(), span)),
)
}
} else {
let bytes = trim_quotes(bytes);
if let Ok(token) = String::from_utf8(bytes.into()) {
(token, None)
} else {
(
String::new(),
Some(ParseError::Expected("string".into(), span)),
)
}
}
}
2021-09-02 10:25:22 +02:00
pub fn parse_string(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
2022-01-01 22:42:50 +01:00
trace!("parsing: string");
2021-09-02 10:25:22 +02:00
let bytes = working_set.get_span_contents(span);
2022-01-19 15:58:12 +01:00
if bytes.is_empty() {
return (
Expression::garbage(span),
Some(ParseError::Expected("String".into(), span)),
);
}
// Check for bare word interpolation
if bytes[0] != b'\'' && bytes[0] != b'"' && bytes[0] != b'`' && bytes.contains(&b'(') {
return parse_string_interpolation(working_set, span, expand_aliases_denylist);
}
let (s, err) = unescape_unquote_string(bytes, span);
2021-07-16 22:26:40 +02:00
(
Expression {
expr: Expr::String(s),
span,
ty: Type::String,
custom_completion: None,
},
err,
)
2021-09-02 10:25:22 +02:00
}
2021-07-02 09:15:30 +02:00
2021-11-04 03:32:35 +01:00
pub fn parse_string_strict(
working_set: &mut StateWorkingSet,
span: Span,
) -> (Expression, Option<ParseError>) {
2022-01-01 22:42:50 +01:00
trace!("parsing: string, with required delimiters");
2021-11-04 03:32:35 +01:00
let bytes = working_set.get_span_contents(span);
2022-01-19 15:58:12 +01:00
// Check for unbalanced quotes:
{
let bytes = if bytes.starts_with(b"$") {
&bytes[1..]
} else {
bytes
};
if bytes.starts_with(b"\"") && (bytes.len() == 1 || !bytes.ends_with(b"\"")) {
return (garbage(span), Some(ParseError::Unclosed("\"".into(), span)));
}
if bytes.starts_with(b"\'") && (bytes.len() == 1 || !bytes.ends_with(b"\'")) {
return (garbage(span), Some(ParseError::Unclosed("\'".into(), span)));
}
2022-01-19 15:58:12 +01:00
}
2021-11-04 03:32:35 +01:00
let (bytes, quoted) = if (bytes.starts_with(b"\"") && bytes.ends_with(b"\"") && bytes.len() > 1)
|| (bytes.starts_with(b"\'") && bytes.ends_with(b"\'") && bytes.len() > 1)
{
(&bytes[1..(bytes.len() - 1)], true)
} else if (bytes.starts_with(b"$\"") && bytes.ends_with(b"\"") && bytes.len() > 2)
|| (bytes.starts_with(b"$\'") && bytes.ends_with(b"\'") && bytes.len() > 2)
{
(&bytes[2..(bytes.len() - 1)], true)
2021-11-04 03:32:35 +01:00
} else {
(bytes, false)
};
if let Ok(token) = String::from_utf8(bytes.into()) {
2022-01-01 22:42:50 +01:00
trace!("-- found {}", token);
2021-11-04 03:32:35 +01:00
if quoted {
(
Expression {
expr: Expr::String(token),
span,
ty: Type::String,
custom_completion: None,
},
None,
)
} else if token.contains(' ') {
(
garbage(span),
Some(ParseError::Expected("string".into(), span)),
)
} else {
(
Expression {
expr: Expr::String(token),
span,
ty: Type::String,
custom_completion: None,
},
None,
)
}
} else {
(
garbage(span),
Some(ParseError::Expected("string".into(), span)),
)
}
}
2021-10-12 19:44:23 +02:00
//TODO: Handle error case for unknown shapes
2021-09-02 10:25:22 +02:00
pub fn parse_shape_name(
working_set: &StateWorkingSet,
2021-09-02 10:25:22 +02:00
bytes: &[u8],
span: Span,
) -> (SyntaxShape, Option<ParseError>) {
allow lists to have type annotations (#8529) this pr refines #8270 and closes #8109 # description examples: the original syntax is okay ```nu def okay [nums: list] {} # the type of list will be list<any> ``` empty annotations are allowed in any variation the last two may be caught by a future formatter, but do not affect `nu` code currently ```nu def okay [nums: list<>] {} # okay def okay [nums: list< >] {} # weird but also okay def okay [nums: list< >] {} # also weird but okay ``` types are allowed (See [notes](#notes) below) ```nu def okay [nums: list<int>] {} # `test [a b c]` will throw an error def okay [nums: list< int > {} # any amount of space within the angle brackets is okay def err [nums: list <int>] {} # this is not okay, `nums` and `<int>` will be parsed as # two separate params, ``` nested annotations are allowed in many variations ```nu def okay [items: list<list<int>>] {} def okay [items: list<list>] {} ``` any unterminated annotation is caught ```nu Error: nu::parser::unexpected_eof × Unexpected end of code. ╭─[source:1:1] 1 │ def err [nums: list<int] {} · ▲ · ╰── expected closing > ╰──── ``` unknown types are flagged ```nu Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<str>] {} · ─┬─ · ╰── unknown type ╰──── Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<int, string>] {} · ─────┬───── · ╰── unknown type ╰──── ``` # notes the error message for mismatched types in not as intuitive ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ┬ · ╰── expected int ╰──── ``` it should be something like this ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ──┬── · ╰── expected list<int> ╰──── ``` this is currently not implemented
2023-03-24 12:54:06 +01:00
let mut error = None;
2021-09-02 10:25:22 +02:00
let result = match bytes {
b"any" => SyntaxShape::Any,
2022-03-01 00:31:53 +01:00
b"binary" => SyntaxShape::Binary,
b"block" => SyntaxShape::Block, //FIXME: Blocks should have known output types
b"bool" => SyntaxShape::Boolean,
2021-09-07 00:02:24 +02:00
b"cell-path" => SyntaxShape::CellPath,
b"closure" => SyntaxShape::Closure(None), //FIXME: Blocks should have known output types
b"cond" => SyntaxShape::RowCondition,
// b"custom" => SyntaxShape::Custom(Box::new(SyntaxShape::Any), SyntaxShape::Int),
b"datetime" => SyntaxShape::DateTime,
b"directory" => SyntaxShape::Directory,
b"duration" => SyntaxShape::Duration,
b"error" => SyntaxShape::Error,
b"expr" => SyntaxShape::Expression,
b"float" | b"decimal" => SyntaxShape::Decimal,
b"filesize" => SyntaxShape::Filesize,
b"full-cell-path" => SyntaxShape::FullCellPath,
2021-09-02 10:25:22 +02:00
b"glob" => SyntaxShape::GlobPattern,
b"int" => SyntaxShape::Int,
b"import-pattern" => SyntaxShape::ImportPattern,
b"keyword" => SyntaxShape::Keyword(vec![], Box::new(SyntaxShape::Any)),
allow lists to have type annotations (#8529) this pr refines #8270 and closes #8109 # description examples: the original syntax is okay ```nu def okay [nums: list] {} # the type of list will be list<any> ``` empty annotations are allowed in any variation the last two may be caught by a future formatter, but do not affect `nu` code currently ```nu def okay [nums: list<>] {} # okay def okay [nums: list< >] {} # weird but also okay def okay [nums: list< >] {} # also weird but okay ``` types are allowed (See [notes](#notes) below) ```nu def okay [nums: list<int>] {} # `test [a b c]` will throw an error def okay [nums: list< int > {} # any amount of space within the angle brackets is okay def err [nums: list <int>] {} # this is not okay, `nums` and `<int>` will be parsed as # two separate params, ``` nested annotations are allowed in many variations ```nu def okay [items: list<list<int>>] {} def okay [items: list<list>] {} ``` any unterminated annotation is caught ```nu Error: nu::parser::unexpected_eof × Unexpected end of code. ╭─[source:1:1] 1 │ def err [nums: list<int] {} · ▲ · ╰── expected closing > ╰──── ``` unknown types are flagged ```nu Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<str>] {} · ─┬─ · ╰── unknown type ╰──── Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<int, string>] {} · ─────┬───── · ╰── unknown type ╰──── ``` # notes the error message for mismatched types in not as intuitive ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ┬ · ╰── expected int ╰──── ``` it should be something like this ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ──┬── · ╰── expected list<int> ╰──── ``` this is currently not implemented
2023-03-24 12:54:06 +01:00
_ if bytes.starts_with(b"list") => {
let (sig, err) = parse_list_shape(working_set, bytes, span);
error = error.or(err);
sig
}
2021-09-02 10:25:22 +02:00
b"math" => SyntaxShape::MathExpression,
b"nothing" => SyntaxShape::Nothing,
b"number" => SyntaxShape::Number,
b"one-of" => SyntaxShape::OneOf(vec![]),
b"operator" => SyntaxShape::Operator,
b"path" => SyntaxShape::Filepath,
b"range" => SyntaxShape::Range,
b"record" => SyntaxShape::Record,
b"signature" => SyntaxShape::Signature,
b"string" => SyntaxShape::String,
b"table" => SyntaxShape::Table,
b"variable" => SyntaxShape::Variable,
b"var-with-opt-type" => SyntaxShape::VarWithOptType,
_ => {
if bytes.contains(&b'@') {
let split: Vec<_> = bytes.split(|b| b == &b'@').collect();
let shape_span = Span::new(span.start, span.start + split[0].len());
let cmd_span = Span::new(span.start + split[0].len() + 1, span.end);
let (shape, err) = parse_shape_name(working_set, split[0], shape_span);
let command_name = trim_quotes(split[1]);
if command_name.is_empty() {
let err = ParseError::Expected("a command name".into(), cmd_span);
return (SyntaxShape::Any, Some(err));
}
let decl_id = working_set.find_decl(command_name, &Type::Any);
if let Some(decl_id) = decl_id {
return (SyntaxShape::Custom(Box::new(shape), decl_id), err);
} else {
return (shape, Some(ParseError::UnknownCommand(cmd_span)));
}
} else {
return (SyntaxShape::Any, Some(ParseError::UnknownType(span)));
}
}
2021-09-02 10:25:22 +02:00
};
allow lists to have type annotations (#8529) this pr refines #8270 and closes #8109 # description examples: the original syntax is okay ```nu def okay [nums: list] {} # the type of list will be list<any> ``` empty annotations are allowed in any variation the last two may be caught by a future formatter, but do not affect `nu` code currently ```nu def okay [nums: list<>] {} # okay def okay [nums: list< >] {} # weird but also okay def okay [nums: list< >] {} # also weird but okay ``` types are allowed (See [notes](#notes) below) ```nu def okay [nums: list<int>] {} # `test [a b c]` will throw an error def okay [nums: list< int > {} # any amount of space within the angle brackets is okay def err [nums: list <int>] {} # this is not okay, `nums` and `<int>` will be parsed as # two separate params, ``` nested annotations are allowed in many variations ```nu def okay [items: list<list<int>>] {} def okay [items: list<list>] {} ``` any unterminated annotation is caught ```nu Error: nu::parser::unexpected_eof × Unexpected end of code. ╭─[source:1:1] 1 │ def err [nums: list<int] {} · ▲ · ╰── expected closing > ╰──── ``` unknown types are flagged ```nu Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<str>] {} · ─┬─ · ╰── unknown type ╰──── Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<int, string>] {} · ─────┬───── · ╰── unknown type ╰──── ``` # notes the error message for mismatched types in not as intuitive ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ┬ · ╰── expected int ╰──── ``` it should be something like this ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ──┬── · ╰── expected list<int> ╰──── ``` this is currently not implemented
2023-03-24 12:54:06 +01:00
(result, error)
}
fn parse_list_shape(
working_set: &StateWorkingSet,
bytes: &[u8],
span: Span,
) -> (SyntaxShape, Option<ParseError>) {
assert!(bytes.starts_with(b"list"));
if bytes == b"list" {
(SyntaxShape::List(Box::new(SyntaxShape::Any)), None)
} else if bytes.starts_with(b"list<") {
let start = span.start + 5;
// if the annotation is unterminated, we want to return early to avoid
// overflows with spans
let end = if bytes.ends_with(b">") {
span.end - 1
// extra characters after the >
} else if bytes.contains(&b'>') {
let angle_start = bytes.split(|it| it == &b'>').collect::<Vec<_>>()[0].len() + 1;
let span = Span::new(span.start + angle_start, span.end);
let err = ParseError::LabeledError(
"Extra characters in the parameter name".into(),
"extra characters".into(),
span,
);
return (SyntaxShape::Any, Some(err));
allow lists to have type annotations (#8529) this pr refines #8270 and closes #8109 # description examples: the original syntax is okay ```nu def okay [nums: list] {} # the type of list will be list<any> ``` empty annotations are allowed in any variation the last two may be caught by a future formatter, but do not affect `nu` code currently ```nu def okay [nums: list<>] {} # okay def okay [nums: list< >] {} # weird but also okay def okay [nums: list< >] {} # also weird but okay ``` types are allowed (See [notes](#notes) below) ```nu def okay [nums: list<int>] {} # `test [a b c]` will throw an error def okay [nums: list< int > {} # any amount of space within the angle brackets is okay def err [nums: list <int>] {} # this is not okay, `nums` and `<int>` will be parsed as # two separate params, ``` nested annotations are allowed in many variations ```nu def okay [items: list<list<int>>] {} def okay [items: list<list>] {} ``` any unterminated annotation is caught ```nu Error: nu::parser::unexpected_eof × Unexpected end of code. ╭─[source:1:1] 1 │ def err [nums: list<int] {} · ▲ · ╰── expected closing > ╰──── ``` unknown types are flagged ```nu Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<str>] {} · ─┬─ · ╰── unknown type ╰──── Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<int, string>] {} · ─────┬───── · ╰── unknown type ╰──── ``` # notes the error message for mismatched types in not as intuitive ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ┬ · ╰── expected int ╰──── ``` it should be something like this ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ──┬── · ╰── expected list<int> ╰──── ``` this is currently not implemented
2023-03-24 12:54:06 +01:00
} else {
let err = ParseError::Unclosed(">".into(), span);
return (SyntaxShape::List(Box::new(SyntaxShape::Any)), Some(err));
};
let inner_span = Span::new(start, end);
let inner_text = String::from_utf8_lossy(working_set.get_span_contents(inner_span));
// remove any extra whitespace, for example `list< string >` becomes `list<string>`
let inner_bytes = inner_text.trim().as_bytes();
// list<>
if inner_bytes.is_empty() {
(SyntaxShape::List(Box::new(SyntaxShape::Any)), None)
} else {
let (inner_sig, err) = parse_shape_name(working_set, inner_bytes, inner_span);
(SyntaxShape::List(Box::new(inner_sig)), err)
}
} else {
(
SyntaxShape::List(Box::new(SyntaxShape::Any)),
Some(ParseError::UnknownType(span)),
)
}
2021-09-02 10:25:22 +02:00
}
2021-07-16 23:55:12 +02:00
2021-09-04 09:59:38 +02:00
pub fn parse_type(_working_set: &StateWorkingSet, bytes: &[u8]) -> Type {
2021-10-12 06:49:17 +02:00
match bytes {
b"binary" => Type::Binary,
2021-10-12 06:49:17 +02:00
b"block" => Type::Block,
b"bool" => Type::Bool,
b"cellpath" => Type::CellPath,
b"closure" => Type::Closure,
b"date" => Type::Date,
b"duration" => Type::Duration,
b"error" => Type::Error,
2021-10-12 06:49:17 +02:00
b"filesize" => Type::Filesize,
b"float" | b"decimal" => Type::Float,
b"int" => Type::Int,
b"list" => Type::List(Box::new(Type::Any)),
b"number" => Type::Number,
b"range" => Type::Range,
b"record" => Type::Record(vec![]),
b"string" => Type::String,
b"table" => Type::Table(vec![]), //FIXME
2021-10-12 06:49:17 +02:00
_ => Type::Any,
2021-07-16 08:24:46 +02:00
}
2021-09-02 10:25:22 +02:00
}
2021-07-16 08:24:46 +02:00
2021-09-26 20:39:19 +02:00
pub fn parse_import_pattern(
2021-09-27 02:23:22 +02:00
working_set: &mut StateWorkingSet,
spans: &[Span],
expand_aliases_denylist: &[usize],
) -> (Expression, Option<ParseError>) {
2021-09-26 20:39:19 +02:00
let mut error = None;
let head_span = if let Some(head_span) = spans.get(0) {
head_span
} else {
2021-09-26 20:39:19 +02:00
return (
garbage(span(spans)),
Some(ParseError::WrongImportPattern(span(spans))),
2021-09-26 20:39:19 +02:00
);
};
2021-09-26 20:39:19 +02:00
let (head_expr, err) = parse_value(
working_set,
*head_span,
&SyntaxShape::Any,
expand_aliases_denylist,
);
error = error.or(err);
let (maybe_module_id, head_name) = match eval_constant(working_set, &head_expr) {
Ok(val) => match value_as_string(val, head_expr.span) {
Ok(s) => (working_set.find_module(s.as_bytes()), s.into_bytes()),
Err(err) => {
return (garbage(span(spans)), error.or(Some(err)));
}
},
Err(err) => {
return (garbage(span(spans)), error.or(Some(err)));
}
};
let (import_pattern, err) = if let Some(tail_span) = spans.get(1) {
2021-09-26 20:39:19 +02:00
// FIXME: expand this to handle deeper imports once we support module imports
let tail = working_set.get_span_contents(*tail_span);
2021-09-26 20:39:19 +02:00
if tail == b"*" {
(
ImportPattern {
head: ImportPatternHead {
name: head_name,
Overlays (#5375) * WIP: Start laying overlays * Rename Overlay->Module; Start adding overlay * Revamp adding overlay * Add overlay add tests; Disable debug print * Fix overlay add; Add overlay remove * Add overlay remove tests * Add missing overlay remove file * Add overlay list command * (WIP?) Enable overlays for env vars * Move OverlayFrames to ScopeFrames * (WIP) Move everything to overlays only ScopeFrame contains nothing but overlays now * Fix predecls * Fix wrong overlay id translation and aliases * Fix broken env lookup logic * Remove TODOs * Add overlay add + remove for environment * Add a few overlay tests; Fix overlay add name * Some cleanup; Fix overlay add/remove names * Clippy * Fmt * Remove walls of comments * List overlays from stack; Add debugging flag Currently, the engine state ordering is somehow broken. * Fix (?) overlay list test * Fix tests on Windows * Fix activated overlay ordering * Check for active overlays equality in overlay list This removes the -p flag: Either both parser and engine will have the same overlays, or the command will fail. * Add merging on overlay remove * Change help message and comment * Add some remove-merge/discard tests * (WIP) Track removed overlays properly * Clippy; Fmt * Fix getting last overlay; Fix predecls in overlays * Remove merging; Fix re-add overwriting stuff Also some error message tweaks. * Fix overlay error in the engine * Update variable_completions.rs * Adds flags and optional arguments to view-source (#5446) * added flags and optional arguments to view-source * removed redundant code * removed redundant code * fmt * fix bug in shell_integration (#5450) * fix bug in shell_integration * add some comments * enable cd to work with directory abbreviations (#5452) * enable cd to work with abbreviations * add abbreviation example * fix tests * make it configurable * make cd recornize symblic link (#5454) * implement seq char command to generate single character sequence (#5453) * add tmp code * add seq char command * Add split number flag in `split row` (#5434) Signed-off-by: Yuheng Su <gipsyh.icu@gmail.com> * Add two more overlay tests * Add ModuleId to OverlayFrame * Fix env conversion accidentally activating overlay It activated overlay from permanent state prematurely which would cause `overlay add` to misbehave. * Remove unused parameter; Add overlay list test * Remove added traces * Add overlay commands examples * Modify TODO * Fix $nu.scope iteration * Disallow removing default overlay * Refactor some parser errors * Remove last overlay if no argument * Diversify overlay examples * Make it possible to update overlay's module In case the origin module updates, the overlay add loads the new module, makes it overlay's origin and applies the changes. Before, it was impossible to update the overlay if the module changed. Co-authored-by: JT <547158+jntrnr@users.noreply.github.com> Co-authored-by: pwygab <88221256+merelymyself@users.noreply.github.com> Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> Co-authored-by: WindSoilder <WindSoilder@outlook.com> Co-authored-by: Yuheng Su <gipsyh.icu@gmail.com>
2022-05-07 21:39:22 +02:00
id: maybe_module_id,
span: *head_span,
},
members: vec![ImportPatternMember::Glob { span: *tail_span }],
hidden: HashSet::new(),
2021-09-26 20:39:19 +02:00
},
None,
2021-09-26 20:39:19 +02:00
)
2021-09-27 02:23:22 +02:00
} else if tail.starts_with(b"[") {
let (result, err) = parse_list_expression(
working_set,
*tail_span,
&SyntaxShape::String,
expand_aliases_denylist,
);
2021-09-27 02:23:22 +02:00
error = error.or(err);
let mut output = vec![];
match result {
Expression {
expr: Expr::List(list),
..
} => {
for expr in list {
let contents = working_set.get_span_contents(expr.span);
output.push((trim_quotes(contents).to_vec(), expr.span));
2021-09-27 02:23:22 +02:00
}
(
ImportPattern {
head: ImportPatternHead {
name: head_name,
Overlays (#5375) * WIP: Start laying overlays * Rename Overlay->Module; Start adding overlay * Revamp adding overlay * Add overlay add tests; Disable debug print * Fix overlay add; Add overlay remove * Add overlay remove tests * Add missing overlay remove file * Add overlay list command * (WIP?) Enable overlays for env vars * Move OverlayFrames to ScopeFrames * (WIP) Move everything to overlays only ScopeFrame contains nothing but overlays now * Fix predecls * Fix wrong overlay id translation and aliases * Fix broken env lookup logic * Remove TODOs * Add overlay add + remove for environment * Add a few overlay tests; Fix overlay add name * Some cleanup; Fix overlay add/remove names * Clippy * Fmt * Remove walls of comments * List overlays from stack; Add debugging flag Currently, the engine state ordering is somehow broken. * Fix (?) overlay list test * Fix tests on Windows * Fix activated overlay ordering * Check for active overlays equality in overlay list This removes the -p flag: Either both parser and engine will have the same overlays, or the command will fail. * Add merging on overlay remove * Change help message and comment * Add some remove-merge/discard tests * (WIP) Track removed overlays properly * Clippy; Fmt * Fix getting last overlay; Fix predecls in overlays * Remove merging; Fix re-add overwriting stuff Also some error message tweaks. * Fix overlay error in the engine * Update variable_completions.rs * Adds flags and optional arguments to view-source (#5446) * added flags and optional arguments to view-source * removed redundant code * removed redundant code * fmt * fix bug in shell_integration (#5450) * fix bug in shell_integration * add some comments * enable cd to work with directory abbreviations (#5452) * enable cd to work with abbreviations * add abbreviation example * fix tests * make it configurable * make cd recornize symblic link (#5454) * implement seq char command to generate single character sequence (#5453) * add tmp code * add seq char command * Add split number flag in `split row` (#5434) Signed-off-by: Yuheng Su <gipsyh.icu@gmail.com> * Add two more overlay tests * Add ModuleId to OverlayFrame * Fix env conversion accidentally activating overlay It activated overlay from permanent state prematurely which would cause `overlay add` to misbehave. * Remove unused parameter; Add overlay list test * Remove added traces * Add overlay commands examples * Modify TODO * Fix $nu.scope iteration * Disallow removing default overlay * Refactor some parser errors * Remove last overlay if no argument * Diversify overlay examples * Make it possible to update overlay's module In case the origin module updates, the overlay add loads the new module, makes it overlay's origin and applies the changes. Before, it was impossible to update the overlay if the module changed. Co-authored-by: JT <547158+jntrnr@users.noreply.github.com> Co-authored-by: pwygab <88221256+merelymyself@users.noreply.github.com> Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> Co-authored-by: WindSoilder <WindSoilder@outlook.com> Co-authored-by: Yuheng Su <gipsyh.icu@gmail.com>
2022-05-07 21:39:22 +02:00
id: maybe_module_id,
span: *head_span,
},
2021-09-27 02:23:22 +02:00
members: vec![ImportPatternMember::List { names: output }],
hidden: HashSet::new(),
2021-09-27 02:23:22 +02:00
},
None,
2021-09-27 02:23:22 +02:00
)
}
_ => (
ImportPattern {
head: ImportPatternHead {
name: head_name,
Overlays (#5375) * WIP: Start laying overlays * Rename Overlay->Module; Start adding overlay * Revamp adding overlay * Add overlay add tests; Disable debug print * Fix overlay add; Add overlay remove * Add overlay remove tests * Add missing overlay remove file * Add overlay list command * (WIP?) Enable overlays for env vars * Move OverlayFrames to ScopeFrames * (WIP) Move everything to overlays only ScopeFrame contains nothing but overlays now * Fix predecls * Fix wrong overlay id translation and aliases * Fix broken env lookup logic * Remove TODOs * Add overlay add + remove for environment * Add a few overlay tests; Fix overlay add name * Some cleanup; Fix overlay add/remove names * Clippy * Fmt * Remove walls of comments * List overlays from stack; Add debugging flag Currently, the engine state ordering is somehow broken. * Fix (?) overlay list test * Fix tests on Windows * Fix activated overlay ordering * Check for active overlays equality in overlay list This removes the -p flag: Either both parser and engine will have the same overlays, or the command will fail. * Add merging on overlay remove * Change help message and comment * Add some remove-merge/discard tests * (WIP) Track removed overlays properly * Clippy; Fmt * Fix getting last overlay; Fix predecls in overlays * Remove merging; Fix re-add overwriting stuff Also some error message tweaks. * Fix overlay error in the engine * Update variable_completions.rs * Adds flags and optional arguments to view-source (#5446) * added flags and optional arguments to view-source * removed redundant code * removed redundant code * fmt * fix bug in shell_integration (#5450) * fix bug in shell_integration * add some comments * enable cd to work with directory abbreviations (#5452) * enable cd to work with abbreviations * add abbreviation example * fix tests * make it configurable * make cd recornize symblic link (#5454) * implement seq char command to generate single character sequence (#5453) * add tmp code * add seq char command * Add split number flag in `split row` (#5434) Signed-off-by: Yuheng Su <gipsyh.icu@gmail.com> * Add two more overlay tests * Add ModuleId to OverlayFrame * Fix env conversion accidentally activating overlay It activated overlay from permanent state prematurely which would cause `overlay add` to misbehave. * Remove unused parameter; Add overlay list test * Remove added traces * Add overlay commands examples * Modify TODO * Fix $nu.scope iteration * Disallow removing default overlay * Refactor some parser errors * Remove last overlay if no argument * Diversify overlay examples * Make it possible to update overlay's module In case the origin module updates, the overlay add loads the new module, makes it overlay's origin and applies the changes. Before, it was impossible to update the overlay if the module changed. Co-authored-by: JT <547158+jntrnr@users.noreply.github.com> Co-authored-by: pwygab <88221256+merelymyself@users.noreply.github.com> Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> Co-authored-by: WindSoilder <WindSoilder@outlook.com> Co-authored-by: Yuheng Su <gipsyh.icu@gmail.com>
2022-05-07 21:39:22 +02:00
id: maybe_module_id,
span: *head_span,
},
2021-09-27 02:23:22 +02:00
members: vec![],
hidden: HashSet::new(),
2021-09-27 02:23:22 +02:00
},
Some(ParseError::ExportNotFound(result.span)),
),
}
2021-09-26 20:39:19 +02:00
} else {
2021-11-14 20:40:26 +01:00
let tail = trim_quotes(tail);
2021-09-26 20:39:19 +02:00
(
ImportPattern {
head: ImportPatternHead {
name: head_name,
Overlays (#5375) * WIP: Start laying overlays * Rename Overlay->Module; Start adding overlay * Revamp adding overlay * Add overlay add tests; Disable debug print * Fix overlay add; Add overlay remove * Add overlay remove tests * Add missing overlay remove file * Add overlay list command * (WIP?) Enable overlays for env vars * Move OverlayFrames to ScopeFrames * (WIP) Move everything to overlays only ScopeFrame contains nothing but overlays now * Fix predecls * Fix wrong overlay id translation and aliases * Fix broken env lookup logic * Remove TODOs * Add overlay add + remove for environment * Add a few overlay tests; Fix overlay add name * Some cleanup; Fix overlay add/remove names * Clippy * Fmt * Remove walls of comments * List overlays from stack; Add debugging flag Currently, the engine state ordering is somehow broken. * Fix (?) overlay list test * Fix tests on Windows * Fix activated overlay ordering * Check for active overlays equality in overlay list This removes the -p flag: Either both parser and engine will have the same overlays, or the command will fail. * Add merging on overlay remove * Change help message and comment * Add some remove-merge/discard tests * (WIP) Track removed overlays properly * Clippy; Fmt * Fix getting last overlay; Fix predecls in overlays * Remove merging; Fix re-add overwriting stuff Also some error message tweaks. * Fix overlay error in the engine * Update variable_completions.rs * Adds flags and optional arguments to view-source (#5446) * added flags and optional arguments to view-source * removed redundant code * removed redundant code * fmt * fix bug in shell_integration (#5450) * fix bug in shell_integration * add some comments * enable cd to work with directory abbreviations (#5452) * enable cd to work with abbreviations * add abbreviation example * fix tests * make it configurable * make cd recornize symblic link (#5454) * implement seq char command to generate single character sequence (#5453) * add tmp code * add seq char command * Add split number flag in `split row` (#5434) Signed-off-by: Yuheng Su <gipsyh.icu@gmail.com> * Add two more overlay tests * Add ModuleId to OverlayFrame * Fix env conversion accidentally activating overlay It activated overlay from permanent state prematurely which would cause `overlay add` to misbehave. * Remove unused parameter; Add overlay list test * Remove added traces * Add overlay commands examples * Modify TODO * Fix $nu.scope iteration * Disallow removing default overlay * Refactor some parser errors * Remove last overlay if no argument * Diversify overlay examples * Make it possible to update overlay's module In case the origin module updates, the overlay add loads the new module, makes it overlay's origin and applies the changes. Before, it was impossible to update the overlay if the module changed. Co-authored-by: JT <547158+jntrnr@users.noreply.github.com> Co-authored-by: pwygab <88221256+merelymyself@users.noreply.github.com> Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> Co-authored-by: WindSoilder <WindSoilder@outlook.com> Co-authored-by: Yuheng Su <gipsyh.icu@gmail.com>
2022-05-07 21:39:22 +02:00
id: maybe_module_id,
span: *head_span,
},
2021-09-26 20:39:19 +02:00
members: vec![ImportPatternMember::Name {
name: tail.to_vec(),
span: *tail_span,
2021-09-26 20:39:19 +02:00
}],
hidden: HashSet::new(),
2021-09-26 20:39:19 +02:00
},
None,
2021-09-26 20:39:19 +02:00
)
}
} else {
(
ImportPattern {
head: ImportPatternHead {
name: head_name,
Overlays (#5375) * WIP: Start laying overlays * Rename Overlay->Module; Start adding overlay * Revamp adding overlay * Add overlay add tests; Disable debug print * Fix overlay add; Add overlay remove * Add overlay remove tests * Add missing overlay remove file * Add overlay list command * (WIP?) Enable overlays for env vars * Move OverlayFrames to ScopeFrames * (WIP) Move everything to overlays only ScopeFrame contains nothing but overlays now * Fix predecls * Fix wrong overlay id translation and aliases * Fix broken env lookup logic * Remove TODOs * Add overlay add + remove for environment * Add a few overlay tests; Fix overlay add name * Some cleanup; Fix overlay add/remove names * Clippy * Fmt * Remove walls of comments * List overlays from stack; Add debugging flag Currently, the engine state ordering is somehow broken. * Fix (?) overlay list test * Fix tests on Windows * Fix activated overlay ordering * Check for active overlays equality in overlay list This removes the -p flag: Either both parser and engine will have the same overlays, or the command will fail. * Add merging on overlay remove * Change help message and comment * Add some remove-merge/discard tests * (WIP) Track removed overlays properly * Clippy; Fmt * Fix getting last overlay; Fix predecls in overlays * Remove merging; Fix re-add overwriting stuff Also some error message tweaks. * Fix overlay error in the engine * Update variable_completions.rs * Adds flags and optional arguments to view-source (#5446) * added flags and optional arguments to view-source * removed redundant code * removed redundant code * fmt * fix bug in shell_integration (#5450) * fix bug in shell_integration * add some comments * enable cd to work with directory abbreviations (#5452) * enable cd to work with abbreviations * add abbreviation example * fix tests * make it configurable * make cd recornize symblic link (#5454) * implement seq char command to generate single character sequence (#5453) * add tmp code * add seq char command * Add split number flag in `split row` (#5434) Signed-off-by: Yuheng Su <gipsyh.icu@gmail.com> * Add two more overlay tests * Add ModuleId to OverlayFrame * Fix env conversion accidentally activating overlay It activated overlay from permanent state prematurely which would cause `overlay add` to misbehave. * Remove unused parameter; Add overlay list test * Remove added traces * Add overlay commands examples * Modify TODO * Fix $nu.scope iteration * Disallow removing default overlay * Refactor some parser errors * Remove last overlay if no argument * Diversify overlay examples * Make it possible to update overlay's module In case the origin module updates, the overlay add loads the new module, makes it overlay's origin and applies the changes. Before, it was impossible to update the overlay if the module changed. Co-authored-by: JT <547158+jntrnr@users.noreply.github.com> Co-authored-by: pwygab <88221256+merelymyself@users.noreply.github.com> Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> Co-authored-by: WindSoilder <WindSoilder@outlook.com> Co-authored-by: Yuheng Su <gipsyh.icu@gmail.com>
2022-05-07 21:39:22 +02:00
id: maybe_module_id,
span: *head_span,
},
2021-09-26 20:39:19 +02:00
members: vec![],
hidden: HashSet::new(),
2021-09-26 20:39:19 +02:00
},
None,
)
};
(
Expression {
expr: Expr::ImportPattern(import_pattern),
span: span(&spans[1..]),
ty: Type::List(Box::new(Type::String)),
custom_completion: None,
},
error.or(err),
)
2021-09-26 20:39:19 +02:00
}
2021-09-02 10:25:22 +02:00
pub fn parse_var_with_opt_type(
working_set: &mut StateWorkingSet,
spans: &[Span],
spans_idx: &mut usize,
mutable: bool,
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
let bytes = working_set.get_span_contents(spans[*spans_idx]).to_vec();
2021-07-16 08:24:46 +02:00
if bytes.contains(&b' ')
|| bytes.contains(&b'"')
|| bytes.contains(&b'\'')
|| bytes.contains(&b'`')
{
2021-10-12 07:08:55 +02:00
return (
garbage(spans[*spans_idx]),
Some(ParseError::VariableNotValid(spans[*spans_idx])),
);
}
2021-09-02 10:25:22 +02:00
if bytes.ends_with(b":") {
// We end with colon, so the next span should be the type
if *spans_idx + 1 < spans.len() {
*spans_idx += 1;
let type_bytes = working_set.get_span_contents(spans[*spans_idx]);
2021-07-16 08:24:46 +02:00
2021-09-02 10:25:22 +02:00
let ty = parse_type(working_set, type_bytes);
2021-07-16 08:24:46 +02:00
let var_name = bytes[0..(bytes.len() - 1)].to_vec();
if !is_variable(&var_name) {
return (
garbage(spans[*spans_idx]),
Some(ParseError::Expected(
"valid variable name".into(),
spans[*spans_idx],
)),
);
}
let id = working_set.add_variable(var_name, spans[*spans_idx - 1], ty.clone(), mutable);
2021-07-16 08:24:46 +02:00
2021-09-02 10:25:22 +02:00
(
Expression {
2021-10-25 22:04:23 +02:00
expr: Expr::VarDecl(id),
2021-09-02 10:25:22 +02:00
span: span(&spans[*spans_idx - 1..*spans_idx + 1]),
ty,
custom_completion: None,
2021-09-02 10:25:22 +02:00
},
None,
)
2021-07-16 08:24:46 +02:00
} else {
let var_name = bytes[0..(bytes.len() - 1)].to_vec();
if !is_variable(&var_name) {
return (
garbage(spans[*spans_idx]),
Some(ParseError::Expected(
"valid variable name".into(),
spans[*spans_idx],
)),
);
}
let id = working_set.add_variable(var_name, spans[*spans_idx], Type::Any, mutable);
2021-07-16 08:24:46 +02:00
(
Expression {
2021-10-25 22:04:23 +02:00
expr: Expr::VarDecl(id),
2021-09-02 10:25:22 +02:00
span: spans[*spans_idx],
ty: Type::Any,
custom_completion: None,
2021-07-16 08:24:46 +02:00
},
2021-09-02 10:25:22 +02:00
Some(ParseError::MissingType(spans[*spans_idx])),
2021-07-16 08:24:46 +02:00
)
}
2021-09-02 10:25:22 +02:00
} else {
let var_name = bytes;
if !is_variable(&var_name) {
return (
garbage(spans[*spans_idx]),
Some(ParseError::Expected(
"valid variable name".into(),
spans[*spans_idx],
)),
);
}
let id = working_set.add_variable(
var_name,
span(&spans[*spans_idx..*spans_idx + 1]),
Type::Any,
mutable,
);
2021-07-08 00:55:46 +02:00
2021-09-02 10:25:22 +02:00
(
Expression {
2021-10-25 22:04:23 +02:00
expr: Expr::VarDecl(id),
2021-09-02 10:25:22 +02:00
span: span(&spans[*spans_idx..*spans_idx + 1]),
ty: Type::Any,
custom_completion: None,
2021-09-02 10:25:22 +02:00
},
None,
)
}
}
2021-09-09 23:47:20 +02:00
pub fn expand_to_cell_path(
working_set: &mut StateWorkingSet,
expression: &mut Expression,
var_id: VarId,
expand_aliases_denylist: &[usize],
2021-09-09 23:47:20 +02:00
) {
trace!("parsing: expanding to cell path");
2021-09-09 23:47:20 +02:00
if let Expression {
expr: Expr::String(_),
span,
..
} = expression
{
// Re-parse the string as if it were a cell-path
let (new_expression, _err) =
parse_full_cell_path(working_set, Some(var_id), *span, expand_aliases_denylist);
2021-09-09 23:47:20 +02:00
*expression = new_expression;
}
}
2021-09-02 10:25:22 +02:00
pub fn parse_row_condition(
working_set: &mut StateWorkingSet,
spans: &[Span],
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
let var_id = working_set.add_variable(b"$it".to_vec(), span(spans), Type::Any, false);
let (expression, err) =
parse_math_expression(working_set, spans, Some(var_id), expand_aliases_denylist);
2021-09-09 23:47:20 +02:00
let span = span(spans);
2021-11-26 04:49:03 +01:00
let block_id = match expression.expr {
Expr::Block(block_id) => block_id,
Expr::Closure(block_id) => block_id,
2021-11-26 04:49:03 +01:00
_ => {
// We have an expression, so let's convert this into a block.
let mut block = Block::new();
let mut pipeline = Pipeline::new();
pipeline
.elements
.push(PipelineElement::Expression(None, expression));
2021-11-26 04:49:03 +01:00
block.pipelines.push(pipeline);
2021-11-26 04:49:03 +01:00
block.signature.required_positional.push(PositionalArg {
name: "$it".into(),
desc: "row condition".into(),
shape: SyntaxShape::Any,
var_id: Some(var_id),
2022-03-07 21:08:56 +01:00
default_value: None,
2021-11-26 04:49:03 +01:00
});
working_set.add_block(block)
}
};
2021-09-09 23:47:20 +02:00
(
Expression {
ty: Type::Bool,
span,
2021-11-26 04:49:03 +01:00
expr: Expr::RowCondition(block_id),
custom_completion: None,
2021-09-09 23:47:20 +02:00
},
err,
)
2021-09-02 10:25:22 +02:00
}
2021-07-16 23:55:12 +02:00
2021-09-02 10:25:22 +02:00
pub fn parse_signature(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
let bytes = working_set.get_span_contents(span);
2021-07-16 23:55:12 +02:00
2021-09-02 10:25:22 +02:00
let mut error = None;
let mut start = span.start;
let mut end = span.end;
2021-07-16 23:55:12 +02:00
let mut has_paren = false;
2021-09-02 10:25:22 +02:00
if bytes.starts_with(b"[") {
start += 1;
} else if bytes.starts_with(b"(") {
has_paren = true;
start += 1;
2021-10-11 22:58:38 +02:00
} else {
error = error.or_else(|| {
Some(ParseError::Expected(
"[ or (".into(),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(start, start + 1),
2021-10-11 22:58:38 +02:00
))
});
2021-09-02 10:25:22 +02:00
}
2021-10-11 22:58:38 +02:00
if (has_paren && bytes.ends_with(b")")) || (!has_paren && bytes.ends_with(b"]")) {
2021-09-02 10:25:22 +02:00
end -= 1;
} else {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
error = error.or_else(|| Some(ParseError::Unclosed("] or )".into(), Span::new(end, end))));
2021-09-02 10:25:22 +02:00
}
2021-07-16 23:55:12 +02:00
let (sig, err) =
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
parse_signature_helper(working_set, Span::new(start, end), expand_aliases_denylist);
2021-09-06 01:16:27 +02:00
error = error.or(err);
(
Expression {
expr: Expr::Signature(sig),
span,
ty: Type::Signature,
custom_completion: None,
2021-09-06 01:16:27 +02:00
},
error,
)
}
pub fn parse_signature_helper(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
2021-09-06 01:16:27 +02:00
) -> (Box<Signature>, Option<ParseError>) {
2022-03-07 21:08:56 +01:00
#[allow(clippy::enum_variant_names)]
2021-09-06 01:16:27 +02:00
enum ParseMode {
ArgMode,
AfterCommaArgMode,
2021-09-06 01:16:27 +02:00
TypeMode,
2022-03-07 21:08:56 +01:00
DefaultValueMode,
2021-09-06 01:16:27 +02:00
}
2022-03-07 21:08:56 +01:00
#[derive(Debug)]
2021-09-06 01:16:27 +02:00
enum Arg {
Positional(PositionalArg, bool), // bool - required
RestPositional(PositionalArg),
2021-09-06 01:16:27 +02:00
Flag(Flag),
}
let mut error = None;
2021-09-02 10:25:22 +02:00
let source = working_set.get_span_contents(span);
2021-07-16 23:55:12 +02:00
allow lists to have type annotations (#8529) this pr refines #8270 and closes #8109 # description examples: the original syntax is okay ```nu def okay [nums: list] {} # the type of list will be list<any> ``` empty annotations are allowed in any variation the last two may be caught by a future formatter, but do not affect `nu` code currently ```nu def okay [nums: list<>] {} # okay def okay [nums: list< >] {} # weird but also okay def okay [nums: list< >] {} # also weird but okay ``` types are allowed (See [notes](#notes) below) ```nu def okay [nums: list<int>] {} # `test [a b c]` will throw an error def okay [nums: list< int > {} # any amount of space within the angle brackets is okay def err [nums: list <int>] {} # this is not okay, `nums` and `<int>` will be parsed as # two separate params, ``` nested annotations are allowed in many variations ```nu def okay [items: list<list<int>>] {} def okay [items: list<list>] {} ``` any unterminated annotation is caught ```nu Error: nu::parser::unexpected_eof × Unexpected end of code. ╭─[source:1:1] 1 │ def err [nums: list<int] {} · ▲ · ╰── expected closing > ╰──── ``` unknown types are flagged ```nu Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<str>] {} · ─┬─ · ╰── unknown type ╰──── Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<int, string>] {} · ─────┬───── · ╰── unknown type ╰──── ``` # notes the error message for mismatched types in not as intuitive ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ┬ · ╰── expected int ╰──── ``` it should be something like this ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ──┬── · ╰── expected list<int> ╰──── ``` this is currently not implemented
2023-03-24 12:54:06 +01:00
let (output, err) = lex_signature(
2022-03-07 21:08:56 +01:00
source,
span.start,
&[b'\n', b'\r'],
&[b':', b'=', b','],
2022-03-07 21:08:56 +01:00
false,
);
allow lists to have type annotations (#8529) this pr refines #8270 and closes #8109 # description examples: the original syntax is okay ```nu def okay [nums: list] {} # the type of list will be list<any> ``` empty annotations are allowed in any variation the last two may be caught by a future formatter, but do not affect `nu` code currently ```nu def okay [nums: list<>] {} # okay def okay [nums: list< >] {} # weird but also okay def okay [nums: list< >] {} # also weird but okay ``` types are allowed (See [notes](#notes) below) ```nu def okay [nums: list<int>] {} # `test [a b c]` will throw an error def okay [nums: list< int > {} # any amount of space within the angle brackets is okay def err [nums: list <int>] {} # this is not okay, `nums` and `<int>` will be parsed as # two separate params, ``` nested annotations are allowed in many variations ```nu def okay [items: list<list<int>>] {} def okay [items: list<list>] {} ``` any unterminated annotation is caught ```nu Error: nu::parser::unexpected_eof × Unexpected end of code. ╭─[source:1:1] 1 │ def err [nums: list<int] {} · ▲ · ╰── expected closing > ╰──── ``` unknown types are flagged ```nu Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<str>] {} · ─┬─ · ╰── unknown type ╰──── Error: nu::parser::unknown_type × Unknown type. ╭─[source:1:1] 1 │ def err [nums: list<int, string>] {} · ─────┬───── · ╰── unknown type ╰──── ``` # notes the error message for mismatched types in not as intuitive ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ┬ · ╰── expected int ╰──── ``` it should be something like this ```nu Error: nu::parser::parse_mismatch × Parse mismatch during operation. ╭─[source:1:1] 1 │ def err [nums: list<int>] {}; err [a b c] · ──┬── · ╰── expected list<int> ╰──── ``` this is currently not implemented
2023-03-24 12:54:06 +01:00
2021-09-02 10:25:22 +02:00
error = error.or(err);
2021-07-16 23:55:12 +02:00
2021-09-02 10:25:22 +02:00
let mut args: Vec<Arg> = vec![];
let mut parse_mode = ParseMode::ArgMode;
2021-07-16 23:55:12 +02:00
2021-09-02 10:25:22 +02:00
for token in &output {
match token {
Token {
contents: crate::TokenContents::Item,
span,
} => {
let span = *span;
let contents = working_set.get_span_contents(span);
// The : symbol separates types
2021-09-02 10:25:22 +02:00
if contents == b":" {
match parse_mode {
ParseMode::ArgMode => {
parse_mode = ParseMode::TypeMode;
2021-07-16 23:55:12 +02:00
}
ParseMode::AfterCommaArgMode => {
error = error.or_else(|| {
Some(ParseError::Expected("parameter or flag".into(), span))
});
}
2022-03-07 21:08:56 +01:00
ParseMode::TypeMode | ParseMode::DefaultValueMode => {
2021-09-02 10:25:22 +02:00
// We're seeing two types for the same thing for some reason, error
error =
error.or_else(|| Some(ParseError::Expected("type".into(), span)));
}
}
}
// The = symbol separates a variable from its default value
else if contents == b"=" {
2022-03-07 21:08:56 +01:00
match parse_mode {
ParseMode::ArgMode | ParseMode::TypeMode => {
parse_mode = ParseMode::DefaultValueMode;
}
ParseMode::AfterCommaArgMode => {
error = error.or_else(|| {
Some(ParseError::Expected("parameter or flag".into(), span))
});
}
2022-03-07 21:08:56 +01:00
ParseMode::DefaultValueMode => {
// We're seeing two default values for some reason, error
error = error.or_else(|| {
Some(ParseError::Expected("default value".into(), span))
});
}
}
}
// The , symbol separates params only
else if contents == b"," {
match parse_mode {
ParseMode::ArgMode => parse_mode = ParseMode::AfterCommaArgMode,
ParseMode::AfterCommaArgMode => {
error = error.or_else(|| {
Some(ParseError::Expected("parameter or flag".into(), span))
});
}
ParseMode::TypeMode => {
error =
error.or_else(|| Some(ParseError::Expected("type".into(), span)));
}
ParseMode::DefaultValueMode => {
error = error.or_else(|| {
Some(ParseError::Expected("default value".into(), span))
});
}
}
2021-09-02 10:25:22 +02:00
} else {
match parse_mode {
ParseMode::ArgMode | ParseMode::AfterCommaArgMode => {
// Long flag with optional short form following with no whitespace, e.g. --output, --age(-a)
2021-09-02 10:25:22 +02:00
if contents.starts_with(b"--") && contents.len() > 2 {
// Split the long flag from the short flag with the ( character as delimiter.
// The trailing ) is removed further down.
2021-09-02 10:25:22 +02:00
let flags: Vec<_> =
contents.split(|x| x == &b'(').map(|x| x.to_vec()).collect();
2021-10-13 19:53:27 +02:00
let long = String::from_utf8_lossy(&flags[0][2..]).to_string();
let mut variable_name = flags[0][2..].to_vec();
// Replace the '-' in a variable name with '_'
(0..variable_name.len()).for_each(|idx| {
if variable_name[idx] == b'-' {
variable_name[idx] = b'_';
}
});
if !is_variable(&variable_name) {
error = error.or_else(|| {
Some(ParseError::Expected(
"valid variable name for this long flag".into(),
span,
))
})
}
2022-03-09 10:42:19 +01:00
let var_id =
working_set.add_variable(variable_name, span, Type::Any, false);
2021-09-02 10:25:22 +02:00
// If there's no short flag, exit now. Otherwise, parse it.
2021-09-02 10:25:22 +02:00
if flags.len() == 1 {
args.push(Arg::Flag(Flag {
arg: None,
desc: String::new(),
long,
short: None,
required: false,
var_id: Some(var_id),
2022-03-07 21:08:56 +01:00
default_value: None,
2021-09-02 10:25:22 +02:00
}));
} else if flags.len() >= 3 {
error = error.or_else(|| {
Some(ParseError::Expected(
"only one short flag alternative".into(),
span,
))
});
2021-09-02 10:25:22 +02:00
} else {
let short_flag = &flags[1];
let short_flag = if !short_flag.starts_with(b"-")
|| !short_flag.ends_with(b")")
{
error = error.or_else(|| {
Some(ParseError::Expected(
"short flag alternative for the long flag".into(),
span,
))
2021-09-02 10:25:22 +02:00
});
short_flag
2021-07-17 20:52:50 +02:00
} else {
// Obtain the flag's name by removing the starting - and trailing )
2021-09-02 10:25:22 +02:00
&short_flag[1..(short_flag.len() - 1)]
};
// Note that it is currently possible to make a short flag with non-alphanumeric characters,
// like -).
2021-07-17 00:39:30 +02:00
let short_flag =
String::from_utf8_lossy(short_flag).to_string();
let chars: Vec<char> = short_flag.chars().collect();
2021-10-13 19:53:27 +02:00
let long = String::from_utf8_lossy(&flags[0][2..]).to_string();
let mut variable_name = flags[0][2..].to_vec();
(0..variable_name.len()).for_each(|idx| {
if variable_name[idx] == b'-' {
variable_name[idx] = b'_';
}
});
if !is_variable(&variable_name) {
error = error.or_else(|| {
Some(ParseError::Expected(
"valid variable name for this short flag".into(),
span,
))
})
}
let var_id = working_set.add_variable(
variable_name,
span,
Type::Any,
false,
);
2021-07-17 00:39:30 +02:00
2021-09-02 10:25:22 +02:00
if chars.len() == 1 {
2021-07-17 00:39:30 +02:00
args.push(Arg::Flag(Flag {
arg: None,
desc: String::new(),
2021-09-02 10:25:22 +02:00
long,
2021-07-17 00:39:30 +02:00
short: Some(chars[0]),
required: false,
2021-07-23 23:19:30 +02:00
var_id: Some(var_id),
2022-03-07 21:08:56 +01:00
default_value: None,
2021-07-17 00:39:30 +02:00
}));
2021-09-02 10:25:22 +02:00
} else {
2021-07-30 00:56:51 +02:00
error = error.or_else(|| {
2021-08-17 01:00:00 +02:00
Some(ParseError::Expected("short flag".into(), span))
2021-07-30 00:56:51 +02:00
});
2021-09-02 10:25:22 +02:00
}
}
parse_mode = ParseMode::ArgMode;
}
// Mandatory short flag, e.g. -e (must be one character)
else if contents.starts_with(b"-") && contents.len() > 1 {
2021-09-02 10:25:22 +02:00
let short_flag = &contents[1..];
let short_flag = String::from_utf8_lossy(short_flag).to_string();
let chars: Vec<char> = short_flag.chars().collect();
if chars.len() > 1 {
error = error.or_else(|| {
Some(ParseError::Expected("short flag".into(), span))
});
2022-01-06 22:06:54 +01:00
}
2021-09-02 10:25:22 +02:00
2022-01-06 22:06:54 +01:00
let mut encoded_var_name = vec![0u8; 4];
let len = chars[0].encode_utf8(&mut encoded_var_name).len();
let variable_name = encoded_var_name[0..len].to_vec();
if !is_variable(&variable_name) {
error = error.or_else(|| {
Some(ParseError::Expected(
"valid variable name for this short flag".into(),
span,
))
})
}
2022-03-09 10:42:19 +01:00
let var_id =
working_set.add_variable(variable_name, span, Type::Any, false);
2021-09-02 10:25:22 +02:00
2022-01-06 22:06:54 +01:00
args.push(Arg::Flag(Flag {
arg: None,
desc: String::new(),
long: String::new(),
short: Some(chars[0]),
required: false,
var_id: Some(var_id),
2022-03-07 21:08:56 +01:00
default_value: None,
2022-01-06 22:06:54 +01:00
}));
parse_mode = ParseMode::ArgMode;
}
// Short flag alias for long flag, e.g. --b (-a)
// This is the same as the short flag in --b(-a)
else if contents.starts_with(b"(-") {
if matches!(parse_mode, ParseMode::AfterCommaArgMode) {
error = error.or_else(|| {
Some(ParseError::Expected("parameter or flag".into(), span))
});
}
2021-09-02 10:25:22 +02:00
let short_flag = &contents[2..];
let short_flag = if !short_flag.ends_with(b")") {
error = error.or_else(|| {
Some(ParseError::Expected("short flag".into(), span))
});
short_flag
} else {
&short_flag[..(short_flag.len() - 1)]
};
2021-07-17 20:52:50 +02:00
2021-09-02 10:25:22 +02:00
let short_flag = String::from_utf8_lossy(short_flag).to_string();
let chars: Vec<char> = short_flag.chars().collect();
2021-07-17 20:52:50 +02:00
2021-09-02 10:25:22 +02:00
if chars.len() == 1 {
match args.last_mut() {
Some(Arg::Flag(flag)) => {
if flag.short.is_some() {
2021-07-30 00:56:51 +02:00
error = error.or_else(|| {
2021-08-17 01:00:00 +02:00
Some(ParseError::Expected(
2021-09-02 10:25:22 +02:00
"one short flag".into(),
2021-07-30 00:56:51 +02:00
span,
))
});
2021-09-02 10:25:22 +02:00
} else {
flag.short = Some(chars[0]);
2021-07-17 20:52:50 +02:00
}
}
2021-09-02 10:25:22 +02:00
_ => {
error = error.or_else(|| {
Some(ParseError::Expected(
"unknown flag".into(),
span,
))
});
}
2021-07-17 20:52:50 +02:00
}
2021-07-30 00:56:51 +02:00
} else {
2021-09-02 10:25:22 +02:00
error = error.or_else(|| {
Some(ParseError::Expected("short flag".into(), span))
});
2021-07-16 23:55:12 +02:00
}
}
// Positional arg, optional
else if contents.ends_with(b"?") {
2021-09-02 10:25:22 +02:00
let contents: Vec<_> = contents[..(contents.len() - 1)].into();
let name = String::from_utf8_lossy(&contents).to_string();
if !is_variable(&contents) {
error = error.or_else(|| {
Some(ParseError::Expected(
"valid variable name for this optional parameter"
.into(),
span,
))
})
}
let var_id =
working_set.add_variable(contents, span, Type::Any, false);
2021-09-02 10:25:22 +02:00
args.push(Arg::Positional(
PositionalArg {
desc: String::new(),
name,
shape: SyntaxShape::Any,
var_id: Some(var_id),
2022-03-07 21:08:56 +01:00
default_value: None,
2021-09-02 10:25:22 +02:00
},
false,
));
parse_mode = ParseMode::ArgMode;
}
// Rest param
else if let Some(contents) = contents.strip_prefix(b"...") {
2021-09-07 05:37:02 +02:00
let name = String::from_utf8_lossy(contents).to_string();
let contents_vec: Vec<u8> = contents.to_vec();
if !is_variable(&contents_vec) {
error = error.or_else(|| {
Some(ParseError::Expected(
"valid variable name for this rest parameter".into(),
span,
))
})
}
2021-09-07 05:37:02 +02:00
2022-03-09 10:42:19 +01:00
let var_id =
working_set.add_variable(contents_vec, span, Type::Any, false);
2021-09-07 05:37:02 +02:00
args.push(Arg::RestPositional(PositionalArg {
desc: String::new(),
name,
shape: SyntaxShape::Any,
var_id: Some(var_id),
2022-03-07 21:08:56 +01:00
default_value: None,
}));
parse_mode = ParseMode::ArgMode;
}
// Normal param
else {
2021-09-02 10:25:22 +02:00
let name = String::from_utf8_lossy(contents).to_string();
let contents_vec = contents.to_vec();
if !is_variable(&contents_vec) {
error = error.or_else(|| {
Some(ParseError::Expected(
"valid variable name for this parameter".into(),
span,
))
})
}
2022-03-09 10:42:19 +01:00
let var_id =
working_set.add_variable(contents_vec, span, Type::Any, false);
2021-09-02 10:25:22 +02:00
// Positional arg, required
args.push(Arg::Positional(
PositionalArg {
desc: String::new(),
name,
shape: SyntaxShape::Any,
var_id: Some(var_id),
2022-03-07 21:08:56 +01:00
default_value: None,
2021-09-02 10:25:22 +02:00
},
true,
));
parse_mode = ParseMode::ArgMode;
2021-07-16 23:55:12 +02:00
}
2021-09-02 10:25:22 +02:00
}
ParseMode::TypeMode => {
if let Some(last) = args.last_mut() {
let (syntax_shape, err) =
parse_shape_name(working_set, contents, span);
error = error.or(err);
2021-10-12 19:44:23 +02:00
//TODO check if we're replacing a custom parameter already
2021-09-02 10:25:22 +02:00
match last {
Arg::Positional(PositionalArg { shape, var_id, .. }, ..) => {
working_set.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type());
*shape = syntax_shape;
}
Arg::RestPositional(PositionalArg {
shape, var_id, ..
}) => {
working_set.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type());
*shape = syntax_shape;
}
2021-09-02 10:25:22 +02:00
Arg::Flag(Flag { arg, var_id, .. }) => {
2021-10-12 06:49:17 +02:00
// Flags with a boolean type are just present/not-present switches
if syntax_shape != SyntaxShape::Boolean {
working_set.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type());
*arg = Some(syntax_shape)
}
2021-07-16 23:55:12 +02:00
}
}
}
2021-09-02 10:25:22 +02:00
parse_mode = ParseMode::ArgMode;
2021-07-16 23:55:12 +02:00
}
2022-03-07 21:08:56 +01:00
ParseMode::DefaultValueMode => {
if let Some(last) = args.last_mut() {
let (expression, err) = parse_value(
working_set,
span,
&SyntaxShape::Any,
expand_aliases_denylist,
);
2022-03-07 21:08:56 +01:00
error = error.or(err);
//TODO check if we're replacing a custom parameter already
match last {
Arg::Positional(
PositionalArg {
shape,
var_id,
default_value,
..
},
required,
) => {
let var_id = var_id.expect("internal error: all custom parameters must have var_ids");
2022-03-09 10:42:19 +01:00
let var_type = &working_set.get_variable(var_id).ty;
2022-03-07 21:08:56 +01:00
match var_type {
Type::Any => {
2022-03-07 21:08:56 +01:00
working_set.set_variable_type(
var_id,
expression.ty.clone(),
);
}
Type::List(param_ty) => {
if let Type::List(expr_ty) = &expression.ty {
if param_ty == expr_ty
|| **param_ty == Type::Any
{
working_set.set_variable_type(
var_id,
expression.ty.clone(),
);
} else {
error = error.or_else(|| {
Some(
ParseError::AssignmentMismatch(
"Default value wrong type"
.into(),
format!(
"expected default value to be `{var_type}`",
),
expression.span,
),
)
})
}
} else {
error = error.or_else(|| {
Some(ParseError::AssignmentMismatch(
"Default value wrong type".into(),
format!(
"expected default value to be `{var_type}`",
),
expression.span,
))
})
}
}
2022-03-07 21:08:56 +01:00
t => {
if t != &expression.ty {
error = error.or_else(|| {
Some(ParseError::AssignmentMismatch(
"Default value wrong type".into(),
format!("expected default value to be `{t}`"),
2022-03-07 21:08:56 +01:00
expression.span,
))
})
}
}
}
*shape = expression.ty.to_shape();
*default_value = Some(expression);
*required = false;
}
Arg::RestPositional(..) => {
error = error.or_else(|| {
Some(ParseError::AssignmentMismatch(
"Rest parameter was given a default value".into(),
2022-03-07 21:08:56 +01:00
"can't have default value".into(),
expression.span,
))
})
}
Arg::Flag(Flag {
arg,
var_id,
default_value,
..
}) => {
let var_id = var_id.expect("internal error: all custom parameters must have var_ids");
2022-03-09 10:42:19 +01:00
let var_type = &working_set.get_variable(var_id).ty;
2022-03-07 21:08:56 +01:00
let expression_ty = expression.ty.clone();
let expression_span = expression.span;
*default_value = Some(expression);
// Flags with a boolean type are just present/not-present switches
if var_type != &Type::Bool {
match var_type {
Type::Any => {
2022-03-07 21:08:56 +01:00
*arg = Some(expression_ty.to_shape());
working_set
.set_variable_type(var_id, expression_ty);
}
t => {
if t != &expression_ty {
error = error.or_else(|| {
Some(ParseError::AssignmentMismatch(
"Default value is the wrong type"
.into(),
format!(
"default value should be {t}"
),
2022-03-07 21:08:56 +01:00
expression_span,
))
})
}
}
}
}
}
}
}
parse_mode = ParseMode::ArgMode;
}
2021-07-16 23:55:12 +02:00
}
}
2021-09-02 10:25:22 +02:00
}
Token {
contents: crate::TokenContents::Comment,
span,
} => {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let contents = working_set.get_span_contents(Span::new(span.start + 1, span.end));
2021-09-02 10:25:22 +02:00
let mut contents = String::from_utf8_lossy(contents).to_string();
contents = contents.trim().into();
if let Some(last) = args.last_mut() {
match last {
Arg::Flag(flag) => {
if !flag.desc.is_empty() {
flag.desc.push('\n');
2021-07-17 00:31:36 +02:00
}
2021-09-02 10:25:22 +02:00
flag.desc.push_str(&contents);
}
Arg::Positional(positional, ..) => {
if !positional.desc.is_empty() {
positional.desc.push('\n');
2021-07-17 00:31:36 +02:00
}
2021-09-02 10:25:22 +02:00
positional.desc.push_str(&contents);
2021-07-17 00:31:36 +02:00
}
Arg::RestPositional(positional) => {
if !positional.desc.is_empty() {
positional.desc.push('\n');
}
positional.desc.push_str(&contents);
}
2021-07-17 00:31:36 +02:00
}
}
2021-07-16 23:55:12 +02:00
}
2021-09-02 10:25:22 +02:00
_ => {}
2021-07-16 23:55:12 +02:00
}
2021-09-02 10:25:22 +02:00
}
2021-07-16 23:55:12 +02:00
2021-09-02 10:25:22 +02:00
let mut sig = Signature::new(String::new());
for arg in args {
match arg {
Arg::Positional(positional, required) => {
2021-09-07 05:37:02 +02:00
if required {
2022-03-07 21:08:56 +01:00
if !sig.optional_positional.is_empty() {
error = error.or_else(|| {
Some(ParseError::RequiredAfterOptional(
positional.name.clone(),
span,
))
})
}
2021-09-02 10:25:22 +02:00
sig.required_positional.push(positional)
} else {
sig.optional_positional.push(positional)
2021-07-17 00:53:45 +02:00
}
2021-07-16 23:55:12 +02:00
}
2021-09-02 10:25:22 +02:00
Arg::Flag(flag) => sig.named.push(flag),
Arg::RestPositional(positional) => {
if positional.name.is_empty() {
error = error.or(Some(ParseError::RestNeedsName(span)))
} else if sig.rest_positional.is_none() {
sig.rest_positional = Some(PositionalArg {
name: positional.name,
..positional
})
} else {
// Too many rest params
error = error.or(Some(ParseError::MultipleRestParams(span)))
}
}
2021-07-16 23:55:12 +02:00
}
}
2021-09-06 01:16:27 +02:00
(Box::new(sig), error)
2021-09-02 10:25:22 +02:00
}
2021-07-08 09:49:17 +02:00
2021-09-02 10:25:22 +02:00
pub fn parse_list_expression(
working_set: &mut StateWorkingSet,
span: Span,
element_shape: &SyntaxShape,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
let bytes = working_set.get_span_contents(span);
2021-07-08 09:49:17 +02:00
2021-09-02 10:25:22 +02:00
let mut error = None;
2021-07-08 09:49:17 +02:00
2021-09-02 10:25:22 +02:00
let mut start = span.start;
let mut end = span.end;
2021-07-08 09:49:17 +02:00
2021-09-02 10:25:22 +02:00
if bytes.starts_with(b"[") {
start += 1;
}
if bytes.ends_with(b"]") {
end -= 1;
} else {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
error = error.or_else(|| Some(ParseError::Unclosed("]".into(), Span::new(end, end))));
2021-09-02 10:25:22 +02:00
}
2021-07-08 09:49:17 +02:00
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let inner_span = Span::new(start, end);
let source = working_set.get_span_contents(inner_span);
2021-07-08 09:49:17 +02:00
let (output, err) = lex(source, inner_span.start, &[b'\n', b'\r', b','], &[], true);
2021-09-02 10:25:22 +02:00
error = error.or(err);
2021-07-08 09:49:17 +02:00
2021-09-02 10:25:22 +02:00
let (output, err) = lite_parse(&output);
error = error.or(err);
2021-07-08 23:45:56 +02:00
2021-09-02 10:25:22 +02:00
let mut args = vec![];
2021-08-17 02:26:05 +02:00
2021-09-02 10:25:22 +02:00
let mut contained_type: Option<Type> = None;
2021-07-08 09:49:17 +02:00
2021-09-02 10:25:22 +02:00
if !output.block.is_empty() {
for arg in &output.block[0].commands {
let mut spans_idx = 0;
2021-07-08 23:45:56 +02:00
if let LiteElement::Command(_, command) = arg {
while spans_idx < command.parts.len() {
let (arg, err) = parse_multispan_value(
working_set,
&command.parts,
&mut spans_idx,
element_shape,
expand_aliases_denylist,
);
error = error.or(err);
2021-09-02 10:25:22 +02:00
if let Some(ref ctype) = contained_type {
if *ctype != arg.ty {
contained_type = Some(Type::Any);
}
} else {
contained_type = Some(arg.ty.clone());
2021-08-17 02:26:05 +02:00
}
args.push(arg);
2021-07-16 08:24:46 +02:00
spans_idx += 1;
}
2021-07-08 09:49:17 +02:00
}
}
}
2021-09-02 10:25:22 +02:00
(
Expression {
expr: Expr::List(args),
span,
ty: Type::List(Box::new(if let Some(ty) = contained_type {
2021-09-04 09:59:38 +02:00
ty
2021-09-02 10:25:22 +02:00
} else {
Type::Any
2021-09-02 10:25:22 +02:00
})),
custom_completion: None,
2021-09-02 10:25:22 +02:00
},
error,
)
}
pub fn parse_table_expression(
working_set: &mut StateWorkingSet,
2021-09-08 20:54:27 +02:00
original_span: Span,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
2021-09-08 20:54:27 +02:00
let bytes = working_set.get_span_contents(original_span);
2021-09-02 10:25:22 +02:00
let mut error = None;
2021-07-06 00:58:56 +02:00
2021-09-08 20:54:27 +02:00
let mut start = original_span.start;
let mut end = original_span.end;
2021-07-06 00:58:56 +02:00
2021-09-02 10:25:22 +02:00
if bytes.starts_with(b"[") {
start += 1;
}
if bytes.ends_with(b"]") {
end -= 1;
} else {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
error = error.or_else(|| Some(ParseError::Unclosed("]".into(), Span::new(end, end))));
2021-09-02 10:25:22 +02:00
}
2021-07-06 00:58:56 +02:00
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let inner_span = Span::new(start, end);
2021-07-06 00:58:56 +02:00
let source = working_set.get_span_contents(inner_span);
2021-07-06 00:58:56 +02:00
2021-11-21 19:13:09 +01:00
let (output, err) = lex(source, start, &[b'\n', b'\r', b','], &[], true);
2021-09-02 10:25:22 +02:00
error = error.or(err);
2021-07-06 00:58:56 +02:00
2021-09-02 10:25:22 +02:00
let (output, err) = lite_parse(&output);
error = error.or(err);
2021-07-06 00:58:56 +02:00
2021-09-02 10:25:22 +02:00
match output.block.len() {
0 => (
Expression {
expr: Expr::List(vec![]),
span: original_span,
ty: Type::List(Box::new(Type::Any)),
custom_completion: None,
2021-09-02 10:25:22 +02:00
},
None,
),
1 => {
// List
parse_list_expression(
working_set,
original_span,
&SyntaxShape::Any,
expand_aliases_denylist,
)
2021-09-02 10:25:22 +02:00
}
_ => {
match &output.block[0].commands[0] {
Support redirect `err` and `out` to different streams (#7685) # Description Closes: #7364 # User-Facing Changes Given the following shell script: ```bash x=$(printf '=%.0s' {1..100}) echo $x echo $x 1>&2 ``` It supports the following command: ``` bash test.sh out> out.txt err> err.txt ``` Then both `out.txt` and `err.txt` will contain `=`(100 times) ## About the change The core idea is that when doing lite-parsing, introduce a new variant `LiteElement::SeparateRedirection` if we meet two Redirection token(which is generated by `lex` function), During converting from lite block to block, `LiteElement::SeparateRedirection` will be converted to `PipelineElement::SeparateRedirection`. Then in the block eval process, if we get `PipelineElement::SeparateRedirection`, we invoke `save` command with `--stderr` arguments to acthive our behavior. ## What happened internally? Take the following command as example: ``` ^ls out> out.txt err> err.txt ``` lex parsing result(`Tokens`) are not changed, but `LiteBlock` and `Block` is changed after this pr. ### LiteBlock before ```rust LiteBlock { block: [ LitePipeline { commands: [ Command(None, LiteCommand { comments: [], parts: [Span { start: 39041, end: 39044 }] }), // actually the span of first Redirection is wrong too.. Redirection(Span { start: 39058, end: 39062 }, Stdout, LiteCommand { comments: [], parts: [Span { start: 39050, end: 39057 }] }), Redirection(Span { start: 39058, end: 39062 }, Stderr, LiteCommand { comments: [], parts: [Span { start: 39063, end: 39070 }] }) ] }] } ``` ### LiteBlock after ```rust LiteBlock { block: [ LitePipeline { commands: [ Command( None, LiteCommand { comments: [], parts: [Span { start: 38525, end: 38528 }] }), // new one! two Redirection merged into one SeparateRedirection. SeparateRedirection { out: (Span { start: 38529, end: 38533 }, LiteCommand { comments: [], parts: [Span { start: 38534, end: 38541 }] }), err: (Span { start: 38542, end: 38546 }, LiteCommand { comments: [], parts: [Span { start: 38547, end: 38554 }] }) } ] }] } ``` ### Block before ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 39042, end: 39044 }, ty: String, custom_completion: None }, [], false), span: Span { start: 39041, end: 39044 }, ty: Any, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stdout, Expression { expr: String("out.txt"), span: Span { start: 39050, end: 39057 }, ty: String, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stderr, Expression { expr: String("err.txt"), span: Span { start: 39063, end: 39070 }, ty: String, custom_completion: None })] } ``` ### Block after ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 38526, end: 38528 }, ty: String, custom_completion: None }, [], false), span: Span { start: 38525, end: 38528 }, ty: Any, custom_completion: None }), // new one! SeparateRedirection SeparateRedirection { out: (Span { start: 38529, end: 38533 }, Expression { expr: String("out.txt"), span: Span { start: 38534, end: 38541 }, ty: String, custom_completion: None }), err: (Span { start: 38542, end: 38546 }, Expression { expr: String("err.txt"), span: Span { start: 38547, end: 38554 }, ty: String, custom_completion: None }) } ] } ``` # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-01-12 10:22:30 +01:00
LiteElement::Command(_, command)
| LiteElement::Redirection(_, _, command)
| LiteElement::SeparateRedirection {
out: (_, command), ..
} => {
let mut table_headers = vec![];
let (headers, err) = parse_value(
working_set,
command.parts[0],
&SyntaxShape::List(Box::new(SyntaxShape::Any)),
expand_aliases_denylist,
);
error = error.or(err);
2021-09-02 10:25:22 +02:00
if let Expression {
expr: Expr::List(headers),
..
} = headers
{
table_headers = headers;
}
2021-09-02 10:25:22 +02:00
match &output.block[1].commands[0] {
LiteElement::Command(_, command)
Support redirect `err` and `out` to different streams (#7685) # Description Closes: #7364 # User-Facing Changes Given the following shell script: ```bash x=$(printf '=%.0s' {1..100}) echo $x echo $x 1>&2 ``` It supports the following command: ``` bash test.sh out> out.txt err> err.txt ``` Then both `out.txt` and `err.txt` will contain `=`(100 times) ## About the change The core idea is that when doing lite-parsing, introduce a new variant `LiteElement::SeparateRedirection` if we meet two Redirection token(which is generated by `lex` function), During converting from lite block to block, `LiteElement::SeparateRedirection` will be converted to `PipelineElement::SeparateRedirection`. Then in the block eval process, if we get `PipelineElement::SeparateRedirection`, we invoke `save` command with `--stderr` arguments to acthive our behavior. ## What happened internally? Take the following command as example: ``` ^ls out> out.txt err> err.txt ``` lex parsing result(`Tokens`) are not changed, but `LiteBlock` and `Block` is changed after this pr. ### LiteBlock before ```rust LiteBlock { block: [ LitePipeline { commands: [ Command(None, LiteCommand { comments: [], parts: [Span { start: 39041, end: 39044 }] }), // actually the span of first Redirection is wrong too.. Redirection(Span { start: 39058, end: 39062 }, Stdout, LiteCommand { comments: [], parts: [Span { start: 39050, end: 39057 }] }), Redirection(Span { start: 39058, end: 39062 }, Stderr, LiteCommand { comments: [], parts: [Span { start: 39063, end: 39070 }] }) ] }] } ``` ### LiteBlock after ```rust LiteBlock { block: [ LitePipeline { commands: [ Command( None, LiteCommand { comments: [], parts: [Span { start: 38525, end: 38528 }] }), // new one! two Redirection merged into one SeparateRedirection. SeparateRedirection { out: (Span { start: 38529, end: 38533 }, LiteCommand { comments: [], parts: [Span { start: 38534, end: 38541 }] }), err: (Span { start: 38542, end: 38546 }, LiteCommand { comments: [], parts: [Span { start: 38547, end: 38554 }] }) } ] }] } ``` ### Block before ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 39042, end: 39044 }, ty: String, custom_completion: None }, [], false), span: Span { start: 39041, end: 39044 }, ty: Any, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stdout, Expression { expr: String("out.txt"), span: Span { start: 39050, end: 39057 }, ty: String, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stderr, Expression { expr: String("err.txt"), span: Span { start: 39063, end: 39070 }, ty: String, custom_completion: None })] } ``` ### Block after ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 38526, end: 38528 }, ty: String, custom_completion: None }, [], false), span: Span { start: 38525, end: 38528 }, ty: Any, custom_completion: None }), // new one! SeparateRedirection SeparateRedirection { out: (Span { start: 38529, end: 38533 }, Expression { expr: String("out.txt"), span: Span { start: 38534, end: 38541 }, ty: String, custom_completion: None }), err: (Span { start: 38542, end: 38546 }, Expression { expr: String("err.txt"), span: Span { start: 38547, end: 38554 }, ty: String, custom_completion: None }) } ] } ``` # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-01-12 10:22:30 +01:00
| LiteElement::Redirection(_, _, command)
| LiteElement::SeparateRedirection {
out: (_, command), ..
} => {
let mut rows = vec![];
for part in &command.parts {
let (values, err) = parse_value(
working_set,
*part,
&SyntaxShape::List(Box::new(SyntaxShape::Any)),
expand_aliases_denylist,
);
error = error.or(err);
if let Expression {
expr: Expr::List(values),
span,
..
} = values
{
match values.len().cmp(&table_headers.len()) {
std::cmp::Ordering::Less => {
error = error.or(Some(ParseError::MissingColumns(
table_headers.len(),
span,
)))
}
std::cmp::Ordering::Equal => {}
std::cmp::Ordering::Greater => {
error = error.or_else(|| {
Some(ParseError::ExtraColumns(
table_headers.len(),
values[table_headers.len()].span,
))
})
}
}
2021-07-06 03:48:45 +02:00
rows.push(values);
}
}
(
Expression {
expr: Expr::Table(table_headers, rows),
span: original_span,
ty: Type::Table(vec![]), //FIXME
custom_completion: None,
},
error,
)
2021-09-08 20:54:27 +02:00
}
}
2021-07-06 03:48:45 +02:00
}
}
2021-07-02 09:32:30 +02:00
}
2021-09-02 10:25:22 +02:00
}
}
2021-07-02 09:32:30 +02:00
2021-09-02 10:25:22 +02:00
pub fn parse_block_expression(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
2022-01-01 22:42:50 +01:00
trace!("parsing: block expression");
2021-09-02 10:25:22 +02:00
let bytes = working_set.get_span_contents(span);
let mut error = None;
2021-07-02 09:32:30 +02:00
2021-09-02 10:25:22 +02:00
let mut start = span.start;
let mut end = span.end;
2021-07-02 09:32:30 +02:00
2021-09-02 10:25:22 +02:00
if bytes.starts_with(b"{") {
start += 1;
} else {
return (
garbage(span),
Some(ParseError::Expected("block".into(), span)),
);
}
if bytes.ends_with(b"}") {
end -= 1;
} else {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
error = error.or_else(|| Some(ParseError::Unclosed("}".into(), Span::new(end, end))));
2021-09-02 10:25:22 +02:00
}
2021-07-02 09:32:30 +02:00
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let inner_span = Span::new(start, end);
2021-09-02 10:25:22 +02:00
2022-01-03 06:21:26 +01:00
let source = working_set.get_span_contents(inner_span);
2021-09-02 10:25:22 +02:00
let (output, err) = lex(source, start, &[], &[], false);
2021-09-02 10:25:22 +02:00
error = error.or(err);
2021-09-06 04:20:02 +02:00
working_set.enter_scope();
2021-09-08 00:00:20 +02:00
// Check to see if we have parameters
let (signature, amt_to_skip): (Option<(Box<Signature>, Span)>, usize) = match output.first() {
Some(Token {
contents: TokenContents::Pipe,
span,
}) => {
error = error.or_else(|| {
Some(ParseError::Expected(
"block but found closure".into(),
*span,
))
});
(None, 0)
}
_ => (None, 0),
};
let (mut output, err) = parse_block(
working_set,
&output[amt_to_skip..],
false,
expand_aliases_denylist,
false,
);
error = error.or(err);
if let Some(signature) = signature {
output.signature = signature.0;
} else if let Some(last) = working_set.delta.scope.last() {
// FIXME: this only supports the top $it. Is this sufficient?
if let Some(var_id) = last.get_var(b"$it") {
let mut signature = Signature::new("");
signature.required_positional.push(PositionalArg {
var_id: Some(*var_id),
name: "$it".into(),
desc: String::new(),
shape: SyntaxShape::Any,
default_value: None,
});
output.signature = Box::new(signature);
}
}
output.span = Some(span);
working_set.exit_scope();
let block_id = working_set.add_block(output);
(
Expression {
expr: Expr::Block(block_id),
span,
ty: Type::Block,
custom_completion: None,
},
error,
)
}
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
pub fn parse_match_block_expression(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
) -> (Expression, Option<ParseError>) {
let bytes = working_set.get_span_contents(span);
let mut error = None;
let mut start = span.start;
let mut end = span.end;
if bytes.starts_with(b"{") {
start += 1;
} else {
return (
garbage(span),
Some(ParseError::Expected("closure".into(), span)),
);
}
if bytes.ends_with(b"}") {
end -= 1;
} else {
error = error.or_else(|| Some(ParseError::Unclosed("}".into(), Span::new(end, end))));
}
let inner_span = Span::new(start, end);
let source = working_set.get_span_contents(inner_span);
let (output, err) = lex(source, start, &[b' ', b'\r', b'\n', b',', b'|'], &[], false);
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
error = error.or(err);
let mut position = 0;
let mut output_matches = vec![];
while position < output.len() {
// Each match gets its own scope
working_set.enter_scope();
// First parse the pattern
let (mut pattern, err) = parse_pattern(working_set, output[position].span);
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
error = error.or(err);
position += 1;
if position >= output.len() {
error = error.or(Some(ParseError::Mismatch(
"=>".into(),
"end of input".into(),
Span::new(output[position - 1].span.end, output[position - 1].span.end),
)));
working_set.exit_scope();
break;
}
// Multiple patterns connected by '|'
let mut connector = working_set.get_span_contents(output[position].span);
if connector == b"|" && position < output.len() {
let mut or_pattern = vec![pattern];
while connector == b"|" && position < output.len() {
connector = b"";
position += 1;
if position >= output.len() {
error = error.or(Some(ParseError::Mismatch(
"pattern".into(),
"end of input".into(),
Span::new(output[position - 1].span.end, output[position - 1].span.end),
)));
working_set.exit_scope();
break;
}
let (pattern, err) = parse_pattern(working_set, output[position].span);
error = error.or(err);
or_pattern.push(pattern);
position += 1;
if position >= output.len() {
error = error.or(Some(ParseError::Mismatch(
"=>".into(),
"end of input".into(),
Span::new(output[position - 1].span.end, output[position - 1].span.end),
)));
working_set.exit_scope();
break;
} else {
connector = working_set.get_span_contents(output[position].span);
}
}
let start = or_pattern
.first()
.expect("internal error: unexpected state of or-pattern")
.span
.start;
let end = or_pattern
.last()
.expect("internal error: unexpected state of or-pattern")
.span
.end;
pattern = MatchPattern {
pattern: Pattern::Or(or_pattern),
span: Span::new(start, end),
}
}
// Then the `=>` arrow
if connector != b"=>" {
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
error = error.or(Some(ParseError::Mismatch(
"=>".into(),
"end of input".into(),
Span::new(output[position - 1].span.end, output[position - 1].span.end),
)));
} else {
position += 1;
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
}
// Finally, the value/expression/block that we will run to produce the result
if position >= output.len() {
error = error.or(Some(ParseError::Mismatch(
"match result".into(),
"end of input".into(),
Span::new(output[position - 1].span.end, output[position - 1].span.end),
)));
working_set.exit_scope();
break;
}
let (result, err) = parse_multispan_value(
working_set,
&[output[position].span],
&mut 0,
&SyntaxShape::OneOf(vec![SyntaxShape::Block, SyntaxShape::Expression]),
expand_aliases_denylist,
);
error = error.or(err);
position += 1;
working_set.exit_scope();
output_matches.push((pattern, result));
}
(
Expression {
expr: Expr::MatchBlock(output_matches),
span,
ty: Type::Any,
custom_completion: None,
},
error,
)
}
pub fn parse_closure_expression(
working_set: &mut StateWorkingSet,
shape: &SyntaxShape,
span: Span,
expand_aliases_denylist: &[usize],
Restrict closure expression to be something like `{|| ...}` (#8290) # Description As title, closes: #7921 closes: #8273 # User-Facing Changes when define a closure without pipe, nushell will raise error for now: ``` ❯ let x = {ss ss} Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #2:1:1] 1 │ let x = {ss ss} · ───┬─── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `any`, `each`, `all`, `where` command accepts closure, it forces user input closure like `{||`, or parse error will returned. ``` ❯ {major:2, minor:1, patch:4} | values | each { into string } Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #4:1:1] 1 │ {major:2, minor:1, patch:4} | values | each { into string } · ───────┬─────── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `with-env`, `do`, `def`, `try` are special, they still remain the same, although it says that it accepts a closure, but they don't need to be written like `{||`, it's more likely a block but can capture variable outside of scope: ``` ❯ def test [input] { echo [0 1 2] | do { do { echo $input } } }; test aaa aaa ``` Just realize that It's a big breaking change, we need to update config and scripts... # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-17 13:36:28 +01:00
require_pipe: bool,
) -> (Expression, Option<ParseError>) {
trace!("parsing: closure expression");
let bytes = working_set.get_span_contents(span);
let mut error = None;
let mut start = span.start;
let mut end = span.end;
if bytes.starts_with(b"{") {
start += 1;
} else {
return (
garbage(span),
Some(ParseError::Expected("closure".into(), span)),
);
}
if bytes.ends_with(b"}") {
end -= 1;
} else {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
error = error.or_else(|| Some(ParseError::Unclosed("}".into(), Span::new(end, end))));
}
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let inner_span = Span::new(start, end);
let source = working_set.get_span_contents(inner_span);
let (output, err) = lex(source, start, &[], &[], false);
error = error.or(err);
working_set.enter_scope();
2021-09-02 10:25:22 +02:00
// Check to see if we have parameters
let (signature, amt_to_skip): (Option<(Box<Signature>, Span)>, usize) = match output.first() {
2021-09-02 10:25:22 +02:00
Some(Token {
contents: TokenContents::Pipe,
2021-09-06 01:16:27 +02:00
span,
}) => {
// We've found a parameter list
let start_point = span.start;
let mut token_iter = output.iter().enumerate().skip(1);
let mut end_span = None;
let mut amt_to_skip = 1;
for token in &mut token_iter {
if let Token {
2021-09-02 10:25:22 +02:00
contents: TokenContents::Pipe,
2021-09-06 01:16:27 +02:00
span,
} = token.1
{
end_span = Some(span);
amt_to_skip = token.0;
break;
2021-08-25 21:29:36 +02:00
}
}
2021-09-06 01:16:27 +02:00
let end_point = if let Some(span) = end_span {
span.end
} else {
end
};
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let signature_span = Span::new(start_point, end_point);
let (signature, err) =
parse_signature_helper(working_set, signature_span, expand_aliases_denylist);
2021-09-06 01:16:27 +02:00
error = error.or(err);
(Some((signature, signature_span)), amt_to_skip)
2021-09-02 10:25:22 +02:00
}
Some(Token {
Better errors when bash-like operators are used (#7241) # Description Adds improved errors for when a user uses a bashism that nu doesn't support. fixes #7237 Examples: ``` Error: nu::parser::shell_andand (link) × The '&&' operator is not supported in Nushell ╭─[entry #1:1:1] 1 │ ls && ls · ─┬ · ╰── instead of '&&', use ';' or 'and' ╰──── help: use ';' instead of the shell '&&', or 'and' instead of the boolean '&&' ``` ``` Error: nu::parser::shell_oror (link) × The '||' operator is not supported in Nushell ╭─[entry #8:1:1] 1 │ ls || ls · ─┬ · ╰── instead of '||', use 'try' or 'or' ╰──── help: use 'try' instead of the shell '||', or 'or' instead of the boolean '||' ``` ``` Error: nu::parser::shell_err (link) × The '2>' shell operation is 'err>' in Nushell. ╭─[entry #9:1:1] 1 │ foo 2> bar.txt · ─┬ · ╰── use 'err>' instead of '2>' in Nushell ╰──── ``` ``` Error: nu::parser::shell_outerr (link) × The '2>&1' shell operation is 'out+err>' in Nushell. ╭─[entry #10:1:1] 1 │ foo 2>&1 bar.txt · ──┬─ · ╰── use 'out+err>' instead of '2>&1' in Nushell ╰──── help: Nushell redirection will write all of stdout before stderr. ``` # User-Facing Changes **BREAKING CHANGES** This removes the `&&` and `||` operators. We previously supported by `&&`/`and` and `||`/`or`. With this change, only `and` and `or` are valid boolean operators. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-12-08 00:02:11 +01:00
contents: TokenContents::PipePipe,
span,
Better errors when bash-like operators are used (#7241) # Description Adds improved errors for when a user uses a bashism that nu doesn't support. fixes #7237 Examples: ``` Error: nu::parser::shell_andand (link) × The '&&' operator is not supported in Nushell ╭─[entry #1:1:1] 1 │ ls && ls · ─┬ · ╰── instead of '&&', use ';' or 'and' ╰──── help: use ';' instead of the shell '&&', or 'and' instead of the boolean '&&' ``` ``` Error: nu::parser::shell_oror (link) × The '||' operator is not supported in Nushell ╭─[entry #8:1:1] 1 │ ls || ls · ─┬ · ╰── instead of '||', use 'try' or 'or' ╰──── help: use 'try' instead of the shell '||', or 'or' instead of the boolean '||' ``` ``` Error: nu::parser::shell_err (link) × The '2>' shell operation is 'err>' in Nushell. ╭─[entry #9:1:1] 1 │ foo 2> bar.txt · ─┬ · ╰── use 'err>' instead of '2>' in Nushell ╰──── ``` ``` Error: nu::parser::shell_outerr (link) × The '2>&1' shell operation is 'out+err>' in Nushell. ╭─[entry #10:1:1] 1 │ foo 2>&1 bar.txt · ──┬─ · ╰── use 'out+err>' instead of '2>&1' in Nushell ╰──── help: Nushell redirection will write all of stdout before stderr. ``` # User-Facing Changes **BREAKING CHANGES** This removes the `&&` and `||` operators. We previously supported by `&&`/`and` and `||`/`or`. With this change, only `and` and `or` are valid boolean operators. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-12-08 00:02:11 +01:00
}) => (
Some((Box::new(Signature::new("closure".to_string())), *span)),
1,
),
Restrict closure expression to be something like `{|| ...}` (#8290) # Description As title, closes: #7921 closes: #8273 # User-Facing Changes when define a closure without pipe, nushell will raise error for now: ``` ❯ let x = {ss ss} Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #2:1:1] 1 │ let x = {ss ss} · ───┬─── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `any`, `each`, `all`, `where` command accepts closure, it forces user input closure like `{||`, or parse error will returned. ``` ❯ {major:2, minor:1, patch:4} | values | each { into string } Error: nu::parser::closure_missing_pipe × Missing || inside closure ╭─[entry #4:1:1] 1 │ {major:2, minor:1, patch:4} | values | each { into string } · ───────┬─────── · ╰── Parsing as a closure, but || is missing ╰──── help: Try add || to the beginning of closure ``` `with-env`, `do`, `def`, `try` are special, they still remain the same, although it says that it accepts a closure, but they don't need to be written like `{||`, it's more likely a block but can capture variable outside of scope: ``` ❯ def test [input] { echo [0 1 2] | do { do { echo $input } } }; test aaa aaa ``` Just realize that It's a big breaking change, we need to update config and scripts... # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-17 13:36:28 +01:00
_ => {
if require_pipe {
error = error.or(Some(ParseError::ClosureMissingPipe(span)));
working_set.exit_scope();
return (garbage(span), error);
} else {
(None, 0)
}
}
2021-09-02 10:25:22 +02:00
};
2021-08-25 21:29:36 +02:00
// TODO: Finish this
if let SyntaxShape::Closure(Some(v)) = shape {
if let Some((sig, sig_span)) = &signature {
if sig.num_positionals() > v.len() {
error = error.or_else(|| {
Some(ParseError::Expected(
format!(
"{} closure parameter{}",
v.len(),
if v.len() > 1 { "s" } else { "" }
),
*sig_span,
))
});
}
for (expected, PositionalArg { name, shape, .. }) in
v.iter().zip(sig.required_positional.iter())
{
if expected != shape && *shape != SyntaxShape::Any {
error = error.or_else(|| {
Some(ParseError::ParameterMismatchType(
name.to_owned(),
expected.to_string(),
shape.to_string(),
*sig_span,
))
});
}
}
}
}
let (mut output, err) = parse_block(
working_set,
&output[amt_to_skip..],
false,
expand_aliases_denylist,
false,
);
2021-09-02 10:25:22 +02:00
error = error.or(err);
2021-07-02 09:32:30 +02:00
2021-09-06 01:16:27 +02:00
if let Some(signature) = signature {
output.signature = signature.0;
2021-09-06 04:20:02 +02:00
} else if let Some(last) = working_set.delta.scope.last() {
2021-10-12 19:44:23 +02:00
// FIXME: this only supports the top $it. Is this sufficient?
2021-09-13 09:31:11 +02:00
2021-09-06 04:20:02 +02:00
if let Some(var_id) = last.get_var(b"$it") {
let mut signature = Signature::new("");
signature.required_positional.push(PositionalArg {
var_id: Some(*var_id),
name: "$it".into(),
desc: String::new(),
shape: SyntaxShape::Any,
2022-03-07 21:08:56 +01:00
default_value: None,
2021-09-06 04:20:02 +02:00
});
output.signature = Box::new(signature);
}
2021-09-06 01:16:27 +02:00
}
output.span = Some(span);
2021-10-25 22:04:23 +02:00
2021-09-06 04:20:02 +02:00
working_set.exit_scope();
2021-09-02 10:25:22 +02:00
let block_id = working_set.add_block(output);
2021-07-16 22:26:40 +02:00
2021-09-02 10:25:22 +02:00
(
Expression {
expr: Expr::Closure(block_id),
2021-09-02 10:25:22 +02:00
span,
ty: Type::Closure,
custom_completion: None,
2021-09-02 10:25:22 +02:00
},
error,
)
}
2021-07-02 09:32:30 +02:00
2021-09-02 10:25:22 +02:00
pub fn parse_value(
working_set: &mut StateWorkingSet,
span: Span,
shape: &SyntaxShape,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
let bytes = working_set.get_span_contents(span);
if bytes.is_empty() {
return (garbage(span), Some(ParseError::IncompleteParser(span)));
}
// Check for reserved keyword values
match bytes {
b"true" => {
if matches!(shape, SyntaxShape::Boolean) || matches!(shape, SyntaxShape::Any) {
return (
Expression {
expr: Expr::Bool(true),
span,
ty: Type::Bool,
custom_completion: None,
},
None,
);
} else {
return (
Expression::garbage(span),
Some(ParseError::Expected("non-boolean value".into(), span)),
);
}
}
b"false" => {
if matches!(shape, SyntaxShape::Boolean) || matches!(shape, SyntaxShape::Any) {
return (
Expression {
expr: Expr::Bool(false),
span,
ty: Type::Bool,
custom_completion: None,
},
None,
);
} else {
return (
Expression::garbage(span),
Some(ParseError::Expected("non-boolean value".into(), span)),
);
}
}
b"null" => {
return (
Expression {
expr: Expr::Nothing,
span,
ty: Type::Nothing,
custom_completion: None,
},
None,
);
}
Require that values that look like numbers parse as numberlike (#8635) # Description Require that any value that looks like it might be a number (starts with a digit, or a '-' + digit, or a '+' + digits, or a special form float like `-inf`, `inf`, or `NaN`) must now be treated as a number-like value. Number-like syntax can only parse into number-like values. Number-like values include: durations, ints, floats, ranges, filesizes, binary data, etc. # User-Facing Changes BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE Just making sure we see this for release notes 😅 This breaks any and all numberlike values that were treated as strings before. Example, we used to allow `3,` as a bare word. Anything like this would now require quotes or backticks to be treated as a string or bare word, respectively. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-28 08:31:38 +02:00
b"-inf" | b"inf" | b"NaN" => {
return parse_numberlike_expr(working_set, span, shape, expand_aliases_denylist);
}
_ => {}
}
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
if matches!(shape, SyntaxShape::MatchPattern) {
return parse_match_pattern(working_set, span);
}
match bytes[0] {
b'$' => return parse_dollar_expr(working_set, span, expand_aliases_denylist),
b'(' => return parse_paren_expr(working_set, span, shape, expand_aliases_denylist),
b'{' => return parse_brace_expr(working_set, span, shape, expand_aliases_denylist),
b'[' => match shape {
2021-09-02 10:25:22 +02:00
SyntaxShape::Any
| SyntaxShape::List(_)
| SyntaxShape::Table
| SyntaxShape::Signature => {}
_ => {
return (
Expression::garbage(span),
Some(ParseError::Expected("non-[] value".into(), span)),
);
2021-07-08 23:45:56 +02:00
}
},
Require that values that look like numbers parse as numberlike (#8635) # Description Require that any value that looks like it might be a number (starts with a digit, or a '-' + digit, or a '+' + digits, or a special form float like `-inf`, `inf`, or `NaN`) must now be treated as a number-like value. Number-like syntax can only parse into number-like values. Number-like values include: durations, ints, floats, ranges, filesizes, binary data, etc. # User-Facing Changes BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE Just making sure we see this for release notes 😅 This breaks any and all numberlike values that were treated as strings before. Example, we used to allow `3,` as a bare word. Anything like this would now require quotes or backticks to be treated as a string or bare word, respectively. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-28 08:31:38 +02:00
x if x.is_ascii_digit() => {
// Anything that starts with a number now has to be a number-like value
// These include values like ints, floats, dates, durations, etc
// To create a string, wrap in quotes, to create a bare word, wrap in backticks
return parse_numberlike_expr(working_set, span, shape, expand_aliases_denylist);
}
b'-' | b'+' => {
if bytes.len() > 1 && bytes[1].is_ascii_digit() {
// Anything that starts with a negative number now has to be a number-like value
// These include values like ints, floats, dates, durations, etc
// To create a string, wrap in quotes, to create a bare word, wrap in backticks
return parse_numberlike_expr(working_set, span, shape, expand_aliases_denylist);
}
}
_ => {}
2021-09-02 10:25:22 +02:00
}
2021-07-01 03:31:02 +02:00
2021-09-02 10:25:22 +02:00
match shape {
SyntaxShape::Custom(shape, custom_completion) => {
let (mut expression, err) =
parse_value(working_set, span, shape, expand_aliases_denylist);
expression.custom_completion = Some(*custom_completion);
(expression, err)
}
SyntaxShape::Range => parse_range(working_set, span, expand_aliases_denylist),
2021-10-04 21:21:31 +02:00
SyntaxShape::Filepath => parse_filepath(working_set, span),
SyntaxShape::Directory => parse_directory(working_set, span),
2021-10-04 21:21:31 +02:00
SyntaxShape::GlobPattern => parse_glob_pattern(working_set, span),
SyntaxShape::String => parse_string(working_set, span, expand_aliases_denylist),
2022-03-01 00:31:53 +01:00
SyntaxShape::Binary => parse_binary(working_set, span),
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
SyntaxShape::MatchPattern => parse_match_pattern(working_set, span),
2021-09-02 10:25:22 +02:00
SyntaxShape::Signature => {
if bytes.starts_with(b"[") {
parse_signature(working_set, span, expand_aliases_denylist)
2021-09-02 10:25:22 +02:00
} else {
(
Expression::garbage(span),
Some(ParseError::Expected("signature".into(), span)),
)
2021-07-09 08:23:20 +02:00
}
2021-09-02 10:25:22 +02:00
}
SyntaxShape::List(elem) => {
if bytes.starts_with(b"[") {
parse_list_expression(working_set, span, elem, expand_aliases_denylist)
2021-09-02 10:25:22 +02:00
} else {
(
Expression::garbage(span),
Some(ParseError::Expected("list".into(), span)),
)
2021-07-08 09:49:17 +02:00
}
2021-09-02 10:25:22 +02:00
}
SyntaxShape::Table => {
if bytes.starts_with(b"[") {
parse_table_expression(working_set, span, expand_aliases_denylist)
2021-09-02 10:25:22 +02:00
} else {
2021-07-02 08:44:37 +02:00
(
2021-09-02 10:25:22 +02:00
Expression::garbage(span),
Some(ParseError::Expected("table".into(), span)),
2021-07-02 08:44:37 +02:00
)
}
2021-07-01 02:01:04 +02:00
}
SyntaxShape::CellPath => parse_simple_cell_path(working_set, span, expand_aliases_denylist),
2021-10-12 06:49:17 +02:00
SyntaxShape::Boolean => {
// Redundant, though we catch bad boolean parses here
if bytes == b"true" || bytes == b"false" {
2021-10-12 06:49:17 +02:00
(
Expression {
expr: Expr::Bool(true),
span,
ty: Type::Bool,
custom_completion: None,
},
None,
)
} else {
(
garbage(span),
Some(ParseError::Expected("bool".into(), span)),
)
}
}
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
// Be sure to return ParseError::Expected(..) if invoked for one of these shapes, but lex
// stream doesn't start with '{'} -- parsing in SyntaxShape::Any arm depends on this error variant.
SyntaxShape::Block | SyntaxShape::Closure(..) | SyntaxShape::Record => (
garbage(span),
Some(ParseError::Expected(
"block, closure or record".into(),
span,
)),
),
2021-09-02 10:25:22 +02:00
SyntaxShape::Any => {
if bytes.starts_with(b"[") {
//parse_value(working_set, span, &SyntaxShape::Table)
parse_full_cell_path(working_set, None, span, expand_aliases_denylist)
} else {
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
let shapes = [
SyntaxShape::Range,
SyntaxShape::Record,
SyntaxShape::Closure(None),
SyntaxShape::Block,
SyntaxShape::String,
];
for shape in shapes.iter() {
Syntax errors for string and int (#7952) # Description Added a few syntax errors in ints and strings, changed parser to stop and show that error rather than continue trying to parse those tokens as some other shape. However, I don't see how to push this direction much further, and most of the classic confusing errors can't be changed. Flagged as WIP for the moment, but passes all checks and works better than current release: 1. I have yet to figure out how to make these errors refer back to the book, as I see some other errors do. 2. How to give syntax error when malformed int is first token in line? Currently parsed as external command, user gets confusing error message. 3. Would like to be more strict with *decimal* int literals (lacking, e.g, `0x' prefix). Need to tinker more with the order of parse shape calls, currently, float is tried after int, so '1.4' has to be passed. _(Description of your pull request goes here. **Provide examples and/or screenshots** if your changes affect the user experience.)_ ```bash 〉"\z" Error: ╭─[entry #3:1:1] 1 │ "\z" · ─┬─ · ╰── Syntax error in string, unrecognized character after escape '\'. ╰──── ``` Canonic presentation of a syntax error. ```bash 〉" \u{01ffbogus}" Error: × Invalid syntax ╭─[entry #2:1:1] 1 │ " \u{01ffbogus}" · ───────┬────── · ╰── Syntax error in string, expecting 1 to 6 hex digits in unicode escape '\u{X...}', max value 10FFFF. ╰──── ``` Malformed unicode escape in string, flagged as error. String parse can be opinionated, it's the last shape tried. ```bash 〉0x22bogus Error: nu::shell::external_command (link) × External command failed ╭─[entry #4:1:1] 1 │ 0x22bogus · ────┬──── · ╰── executable was not found ╰──── help: No such file or directory (os error 2) ``` A *correct* number in first token would be evaluated, but an *incorrect* one is treated as external command? Confusing to users. ```bash 〉0 + 0x22bogus Error: × Invalid syntax ╭─[entry #5:1:1] 1 │ 0 + 0x22bogus · ────┬──── · ╰── Syntax error in int, invalid digits in radix 16 int. ╰──── ``` Can give syntax error if token is unambiguously int literal. e.g has 0b or 0x prefix, could not be a float. ```bash 〉0 + 098bogus Error: nu::parser::unsupported_operation (link) × Types mismatched for operation. ╭─[entry #6:1:1] 1 │ 0 + 098bogus · ┬ ┬ ────┬─── · │ │ ╰── string · │ ╰── doesn't support these values. · ╰── int ╰──── help: Change int or string to be the right types and try again. ``` But *decimal* literal (no prefix) can't be too strict. Parser is going to try float later. So '1.4' must be passed. # User-Facing Changes First and foremost, more specific error messages for typos in string and int literals. Probably improves interactive user experience. But a script that was causing and then checking for specific error might notice a different error message. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ # Tests + Formatting Added (positive and negative unit tests in `cargo test -p nu-parser`. Didn't add integration tests. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [x] `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --------- Co-authored-by: Stefan Holderbach <sholderbach@users.noreply.github.com>
2023-02-13 17:09:50 +01:00
let (s, e) = parse_value(working_set, span, shape, expand_aliases_denylist);
match (s, e) {
(s, None) => {
return (s, None);
}
(_, Some(ParseError::Expected(_, _))) => {
// value didn't parse as this shape, try other options
continue;
}
(s, e) => {
// value did parse, but had syntax issues, don't try any more options.
return (s, e);
}
}
}
(
garbage(span),
Some(ParseError::Expected("any shape".into(), span)),
)
2021-07-02 08:44:37 +02:00
}
2021-09-02 10:25:22 +02:00
}
Require that values that look like numbers parse as numberlike (#8635) # Description Require that any value that looks like it might be a number (starts with a digit, or a '-' + digit, or a '+' + digits, or a special form float like `-inf`, `inf`, or `NaN`) must now be treated as a number-like value. Number-like syntax can only parse into number-like values. Number-like values include: durations, ints, floats, ranges, filesizes, binary data, etc. # User-Facing Changes BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE BREAKING CHANGE Just making sure we see this for release notes 😅 This breaks any and all numberlike values that were treated as strings before. Example, we used to allow `3,` as a bare word. Anything like this would now require quotes or backticks to be treated as a string or bare word, respectively. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-28 08:31:38 +02:00
x => (
garbage(span),
Some(ParseError::Expected(x.to_type().to_string(), span)),
),
2021-07-02 08:44:37 +02:00
}
2021-09-02 10:25:22 +02:00
}
2021-07-02 08:44:37 +02:00
2021-09-02 10:25:22 +02:00
pub fn parse_operator(
working_set: &mut StateWorkingSet,
span: Span,
) -> (Expression, Option<ParseError>) {
let contents = working_set.get_span_contents(span);
let operator = match contents {
b"=" => Operator::Assignment(Assignment::Assign),
b"+=" => Operator::Assignment(Assignment::PlusAssign),
b"++=" => Operator::Assignment(Assignment::AppendAssign),
b"-=" => Operator::Assignment(Assignment::MinusAssign),
b"*=" => Operator::Assignment(Assignment::MultiplyAssign),
b"/=" => Operator::Assignment(Assignment::DivideAssign),
b"==" => Operator::Comparison(Comparison::Equal),
b"!=" => Operator::Comparison(Comparison::NotEqual),
b"<" => Operator::Comparison(Comparison::LessThan),
b"<=" => Operator::Comparison(Comparison::LessThanOrEqual),
b">" => Operator::Comparison(Comparison::GreaterThan),
b">=" => Operator::Comparison(Comparison::GreaterThanOrEqual),
b"=~" => Operator::Comparison(Comparison::RegexMatch),
b"!~" => Operator::Comparison(Comparison::NotRegexMatch),
b"+" => Operator::Math(Math::Plus),
b"++" => Operator::Math(Math::Append),
b"-" => Operator::Math(Math::Minus),
b"*" => Operator::Math(Math::Multiply),
b"/" => Operator::Math(Math::Divide),
b"//" => Operator::Math(Math::FloorDivision),
b"in" => Operator::Comparison(Comparison::In),
b"not-in" => Operator::Comparison(Comparison::NotIn),
b"mod" => Operator::Math(Math::Modulo),
b"bit-or" => Operator::Bits(Bits::BitOr),
b"bit-xor" => Operator::Bits(Bits::BitXor),
b"bit-and" => Operator::Bits(Bits::BitAnd),
b"bit-shl" => Operator::Bits(Bits::ShiftLeft),
b"bit-shr" => Operator::Bits(Bits::ShiftRight),
b"starts-with" => Operator::Comparison(Comparison::StartsWith),
b"ends-with" => Operator::Comparison(Comparison::EndsWith),
Better errors when bash-like operators are used (#7241) # Description Adds improved errors for when a user uses a bashism that nu doesn't support. fixes #7237 Examples: ``` Error: nu::parser::shell_andand (link) × The '&&' operator is not supported in Nushell ╭─[entry #1:1:1] 1 │ ls && ls · ─┬ · ╰── instead of '&&', use ';' or 'and' ╰──── help: use ';' instead of the shell '&&', or 'and' instead of the boolean '&&' ``` ``` Error: nu::parser::shell_oror (link) × The '||' operator is not supported in Nushell ╭─[entry #8:1:1] 1 │ ls || ls · ─┬ · ╰── instead of '||', use 'try' or 'or' ╰──── help: use 'try' instead of the shell '||', or 'or' instead of the boolean '||' ``` ``` Error: nu::parser::shell_err (link) × The '2>' shell operation is 'err>' in Nushell. ╭─[entry #9:1:1] 1 │ foo 2> bar.txt · ─┬ · ╰── use 'err>' instead of '2>' in Nushell ╰──── ``` ``` Error: nu::parser::shell_outerr (link) × The '2>&1' shell operation is 'out+err>' in Nushell. ╭─[entry #10:1:1] 1 │ foo 2>&1 bar.txt · ──┬─ · ╰── use 'out+err>' instead of '2>&1' in Nushell ╰──── help: Nushell redirection will write all of stdout before stderr. ``` # User-Facing Changes **BREAKING CHANGES** This removes the `&&` and `||` operators. We previously supported by `&&`/`and` and `||`/`or`. With this change, only `and` and `or` are valid boolean operators. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-12-08 00:02:11 +01:00
b"and" => Operator::Boolean(Boolean::And),
b"or" => Operator::Boolean(Boolean::Or),
b"xor" => Operator::Boolean(Boolean::Xor),
b"**" => Operator::Math(Math::Pow),
// WARNING: not actual operators below! Error handling only
pow @ (b"^" | b"pow") => {
return (
garbage(span),
Some(ParseError::UnknownOperator(
match pow {
b"^" => "^",
b"pow" => "pow",
_ => unreachable!(),
},
"Use '**' for exponentiation or 'bit-xor' for bitwise XOR.",
span,
)),
);
}
equality @ (b"is" | b"===") => {
return (
garbage(span),
Some(ParseError::UnknownOperator(
match equality {
b"is" => "is",
b"===" => "===",
_ => unreachable!(),
},
"Did you mean '=='?",
span,
)),
);
}
b"contains" => {
return (
garbage(span),
Some(ParseError::UnknownOperator(
"contains",
"Did you mean '$string =~ $pattern' or '$element in $container'?",
span,
)),
);
}
b"%" => {
return (
garbage(span),
Some(ParseError::UnknownOperator(
"%",
"Did you mean 'mod'?",
span,
)),
);
}
b"&" => {
return (
garbage(span),
Some(ParseError::UnknownOperator(
"&",
"Did you mean 'bit-and'?",
span,
)),
);
}
b"<<" => {
return (
garbage(span),
Some(ParseError::UnknownOperator(
"<<",
"Did you mean 'bit-shl'?",
span,
)),
);
}
b">>" => {
return (
garbage(span),
Some(ParseError::UnknownOperator(
">>",
"Did you mean 'bit-shr'?",
span,
)),
);
}
bits @ (b"bits-and" | b"bits-xor" | b"bits-or" | b"bits-shl" | b"bits-shr") => {
return (
garbage(span),
Some(ParseError::UnknownOperator(
match bits {
b"bits-and" => "bits-and",
b"bits-xor" => "bits-xor",
b"bits-or" => "bits-or",
b"bits-shl" => "bits-shl",
b"bits-shr" => "bits-shr",
_ => unreachable!(),
},
match bits {
b"bits-and" => "Did you mean 'bit-and'?",
b"bits-xor" => "Did you mean 'bit-xor'?",
b"bits-or" => "Did you mean 'bit-or'?",
b"bits-shl" => "Did you mean 'bit-shl'?",
b"bits-shr" => "Did you mean 'bit-shr'?",
_ => unreachable!(),
},
span,
)),
);
}
2021-09-02 10:25:22 +02:00
_ => {
return (
garbage(span),
Some(ParseError::Expected("operator".into(), span)),
);
}
};
2021-07-02 08:44:37 +02:00
2021-09-02 10:25:22 +02:00
(
Expression {
expr: Expr::Operator(operator),
span,
ty: Type::Any,
custom_completion: None,
2021-09-02 10:25:22 +02:00
},
None,
)
}
2021-07-02 08:44:37 +02:00
2021-09-02 10:25:22 +02:00
pub fn parse_math_expression(
working_set: &mut StateWorkingSet,
spans: &[Span],
2021-09-09 23:47:20 +02:00
lhs_row_var_id: Option<VarId>,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
trace!("parsing: math expression");
2021-09-02 10:25:22 +02:00
// As the expr_stack grows, we increase the required precedence to grow larger
// If, at any time, the operator we're looking at is the same or lower precedence
// of what is in the expression stack, we collapse the expression stack.
//
// This leads to an expression stack that grows under increasing precedence and collapses
// under decreasing/sustained precedence
//
// The end result is a stack that we can fold into binary operations as right associations
// safely.
let mut expr_stack: Vec<Expression> = vec![];
let mut idx = 0;
let mut last_prec = 1000000;
let mut error = None;
2022-04-06 21:10:25 +02:00
let first_span = working_set.get_span_contents(spans[0]);
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
if first_span == b"if" || first_span == b"match" {
// If expression
if spans.len() > 1 {
return parse_call(working_set, spans, spans[0], expand_aliases_denylist, false);
} else {
return (
garbage(spans[0]),
Some(ParseError::Expected(
"expression".into(),
Span::new(spans[0].end, spans[0].end),
)),
);
}
} else if first_span == b"not" {
2022-04-06 21:10:25 +02:00
if spans.len() > 1 {
let (remainder, err) = parse_math_expression(
working_set,
&spans[1..],
lhs_row_var_id,
expand_aliases_denylist,
);
return (
Expression {
expr: Expr::UnaryNot(Box::new(remainder)),
span: span(spans),
ty: Type::Bool,
custom_completion: None,
},
err,
);
} else {
return (
garbage(spans[0]),
Some(ParseError::Expected(
"expression".into(),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(spans[0].end, spans[0].end),
2022-04-06 21:10:25 +02:00
)),
);
}
}
let (mut lhs, err) = parse_value(
working_set,
spans[0],
&SyntaxShape::Any,
expand_aliases_denylist,
);
2021-09-02 10:25:22 +02:00
error = error.or(err);
idx += 1;
if idx >= spans.len() {
// We already found the one part of our expression, so let's expand
if let Some(row_var_id) = lhs_row_var_id {
expand_to_cell_path(working_set, &mut lhs, row_var_id, expand_aliases_denylist);
}
}
2021-09-02 10:25:22 +02:00
expr_stack.push(lhs);
while idx < spans.len() {
let (op, err) = parse_operator(working_set, spans[idx]);
2021-07-02 08:44:37 +02:00
error = error.or(err);
2021-09-02 10:25:22 +02:00
let op_prec = op.precedence();
2021-07-02 08:44:37 +02:00
2021-09-02 10:25:22 +02:00
idx += 1;
2021-07-02 08:44:37 +02:00
2021-09-02 10:25:22 +02:00
if idx == spans.len() {
// Handle broken math expr `1 +` etc
error = error.or(Some(ParseError::IncompleteMathExpression(spans[idx - 1])));
2021-07-02 08:44:37 +02:00
2021-09-02 10:25:22 +02:00
expr_stack.push(Expression::garbage(spans[idx - 1]));
expr_stack.push(Expression::garbage(spans[idx - 1]));
2021-07-22 21:50:59 +02:00
2021-09-02 10:25:22 +02:00
break;
}
2021-07-22 21:50:59 +02:00
let (rhs, err) = parse_value(
working_set,
spans[idx],
&SyntaxShape::Any,
expand_aliases_denylist,
);
2021-09-02 10:25:22 +02:00
error = error.or(err);
2021-07-02 08:44:37 +02:00
2022-03-25 04:23:08 +01:00
while op_prec <= last_prec && expr_stack.len() > 1 {
2021-11-06 08:31:28 +01:00
// Collapse the right associated operations first
// so that we can get back to a stack with a lower precedence
let mut rhs = expr_stack
.pop()
.expect("internal error: expression stack empty");
let mut op = expr_stack
.pop()
.expect("internal error: expression stack empty");
2022-03-25 04:23:08 +01:00
last_prec = op.precedence();
if last_prec < op_prec {
expr_stack.push(op);
expr_stack.push(rhs);
break;
}
2021-11-06 08:31:28 +01:00
let mut lhs = expr_stack
.pop()
.expect("internal error: expression stack empty");
if let Some(row_var_id) = lhs_row_var_id {
expand_to_cell_path(working_set, &mut lhs, row_var_id, expand_aliases_denylist);
2021-11-06 08:31:28 +01:00
}
2021-09-09 23:47:20 +02:00
2021-11-06 08:31:28 +01:00
let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut op, &mut rhs);
error = error.or(err);
2021-07-02 08:44:37 +02:00
2021-11-06 08:31:28 +01:00
let op_span = span(&[lhs.span, rhs.span]);
expr_stack.push(Expression {
expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)),
span: op_span,
ty: result_ty,
custom_completion: None,
});
2021-07-02 08:44:37 +02:00
}
2021-09-02 10:25:22 +02:00
expr_stack.push(op);
expr_stack.push(rhs);
2021-07-02 08:44:37 +02:00
2021-09-02 10:25:22 +02:00
last_prec = op_prec;
2021-07-23 23:19:30 +02:00
2021-09-02 10:25:22 +02:00
idx += 1;
}
2021-07-02 08:44:37 +02:00
2021-09-02 10:25:22 +02:00
while expr_stack.len() != 1 {
let mut rhs = expr_stack
.pop()
.expect("internal error: expression stack empty");
let mut op = expr_stack
2021-07-02 08:44:37 +02:00
.pop()
.expect("internal error: expression stack empty");
2021-09-02 10:25:22 +02:00
let mut lhs = expr_stack
.pop()
.expect("internal error: expression stack empty");
2021-09-09 23:47:20 +02:00
if let Some(row_var_id) = lhs_row_var_id {
expand_to_cell_path(working_set, &mut lhs, row_var_id, expand_aliases_denylist);
2021-09-09 23:47:20 +02:00
}
2021-09-02 10:25:22 +02:00
let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut op, &mut rhs);
error = error.or(err);
2021-07-02 08:44:37 +02:00
2021-09-02 10:25:22 +02:00
let binary_op_span = span(&[lhs.span, rhs.span]);
expr_stack.push(Expression {
expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)),
span: binary_op_span,
ty: result_ty,
custom_completion: None,
2021-09-02 10:25:22 +02:00
});
2021-07-01 02:01:04 +02:00
}
2021-09-02 10:25:22 +02:00
let output = expr_stack
.pop()
.expect("internal error: expression stack empty");
2021-07-02 00:40:08 +02:00
2021-09-02 10:25:22 +02:00
(output, error)
}
pub fn parse_expression(
working_set: &mut StateWorkingSet,
spans: &[Span],
expand_aliases_denylist: &[usize],
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression: bool,
2021-09-02 10:25:22 +02:00
) -> (Expression, Option<ParseError>) {
trace!("parsing: expression");
2021-11-04 03:32:35 +01:00
let mut pos = 0;
let mut shorthand = vec![];
2021-09-02 10:25:22 +02:00
2021-11-04 03:32:35 +01:00
while pos < spans.len() {
// Check if there is any environment shorthand
let name = working_set.get_span_contents(spans[pos]);
let split = name.splitn(2, |x| *x == b'=');
2021-11-04 03:32:35 +01:00
let split: Vec<_> = split.collect();
if !name.starts_with(b"^") && split.len() == 2 && !split[0].is_empty() {
2021-11-04 03:32:35 +01:00
let point = split[0].len() + 1;
let lhs = parse_string_strict(
working_set,
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(spans[pos].start, spans[pos].start + point - 1),
2021-11-04 03:32:35 +01:00
);
let rhs = if spans[pos].start + point < spans[pos].end {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let rhs_span = Span::new(spans[pos].start + point, spans[pos].end);
if working_set.get_span_contents(rhs_span).starts_with(b"$") {
parse_dollar_expr(working_set, rhs_span, expand_aliases_denylist)
} else {
parse_string_strict(working_set, rhs_span)
}
2021-11-04 03:32:35 +01:00
} else {
(
Expression {
expr: Expr::String(String::new()),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
span: Span::unknown(),
2021-11-04 03:32:35 +01:00
ty: Type::Nothing,
custom_completion: None,
},
None,
)
};
if lhs.1.is_none() && rhs.1.is_none() {
shorthand.push((lhs.0, rhs.0));
pos += 1;
} else {
break;
}
} else {
break;
}
}
if pos == spans.len() {
return (
garbage(span(spans)),
Some(ParseError::UnknownCommand(spans[0])),
);
}
let (output, err) = if is_math_expression_like(working_set, spans[pos], expand_aliases_denylist)
{
parse_math_expression(working_set, &spans[pos..], None, expand_aliases_denylist)
2021-11-04 03:32:35 +01:00
} else {
let bytes = working_set.get_span_contents(spans[pos]).to_vec();
// For now, check for special parses of certain keywords
match bytes.as_slice() {
Re-implement aliases (#8123) # Description This PR adds an alternative alias implementation. Old aliases still work but you need to use `old-alias` instead of `alias`. Instead of replacing spans in the original code and re-parsing, which proved to be extremely error-prone and a constant source of panics, the new implementation creates a new command that references the old command. Consider the new alias defined as `alias ll = ls -l`. The parser creates a new command called `ll` and remembers that it is actually a `ls` command called with the `-l` flag. Then, when the parser sees the `ll` command, it will translate it to `ls -l` and passes to it any parameters that were passed to the call to `ll`. It works quite similar to how known externals defined with `extern` are implemented. The new alias implementation should work the same way as the old aliases, including exporting from modules, referencing both known and unknown externals. It seems to preserve custom completions and pipeline metadata. It is quite robust in most cases but there are some rough edges (see later). Fixes https://github.com/nushell/nushell/issues/7648, https://github.com/nushell/nushell/issues/8026, https://github.com/nushell/nushell/issues/7512, https://github.com/nushell/nushell/issues/5780, https://github.com/nushell/nushell/issues/7754 No effect: https://github.com/nushell/nushell/issues/8122 (we might revisit the completions code after this PR) Should use custom command instead: https://github.com/nushell/nushell/issues/6048 # User-Facing Changes Since aliases are now basically commands, it has some new implications: 1. `alias spam = "spam"` (requires command call) * **workaround**: use `alias spam = echo "spam"` 2. `def foo [] { 'foo' }; alias foo = ls -l` (foo defined more than once) * **workaround**: use different name (commands also have this limitation) 4. `alias ls = (ls | sort-by type name -i)` * **workaround**: Use custom command. _The common issue with this is that it is currently not easy to pass flags through custom commands and command referencing itself will lead to stack overflow. Both of these issues are meant to be addressed._ 5. TODO: Help messages, `which` command, `$nu.scope.aliases`, etc. * Should we treat the aliases as commands or should they be separated from regular commands? 6. Needs better error message and syntax highlight for recursed alias (`alias f = f`) 7. Can't create alias with the same name as existing command (`alias ls = ls -a`) * Might be possible to add support for it (not 100% sure) 8. Standalone `alias` doesn't list aliases anymore 9. Can't alias parser keywords (e.g., stuff like `alias ou = overlay use` won't work) * TODO: Needs a better error message when attempting to do so # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-02-27 08:44:05 +01:00
b"def" | b"extern" | b"for" | b"module" | b"use" | b"source" | b"old-alias"
| b"alias" | b"export" | b"hide" => (
parse_call(
working_set,
&spans[pos..],
spans[0],
expand_aliases_denylist,
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression,
)
.0,
Some(ParseError::BuiltinCommandInPipeline(
String::from_utf8(bytes)
.expect("builtin commands bytes should be able to convert to string"),
spans[0],
)),
),
b"let" | b"const" | b"mut" => (
parse_call(
working_set,
&spans[pos..],
spans[0],
expand_aliases_denylist,
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression,
)
.0,
Some(ParseError::AssignInPipeline(
String::from_utf8(bytes)
.expect("builtin commands bytes should be able to convert to string"),
String::from_utf8_lossy(match spans.len() {
1 | 2 | 3 => b"value",
_ => working_set.get_span_contents(spans[3]),
})
.to_string(),
String::from_utf8_lossy(match spans.len() {
1 => b"variable",
_ => working_set.get_span_contents(spans[1]),
})
.to_string(),
spans[0],
)),
),
Overlays (#5375) * WIP: Start laying overlays * Rename Overlay->Module; Start adding overlay * Revamp adding overlay * Add overlay add tests; Disable debug print * Fix overlay add; Add overlay remove * Add overlay remove tests * Add missing overlay remove file * Add overlay list command * (WIP?) Enable overlays for env vars * Move OverlayFrames to ScopeFrames * (WIP) Move everything to overlays only ScopeFrame contains nothing but overlays now * Fix predecls * Fix wrong overlay id translation and aliases * Fix broken env lookup logic * Remove TODOs * Add overlay add + remove for environment * Add a few overlay tests; Fix overlay add name * Some cleanup; Fix overlay add/remove names * Clippy * Fmt * Remove walls of comments * List overlays from stack; Add debugging flag Currently, the engine state ordering is somehow broken. * Fix (?) overlay list test * Fix tests on Windows * Fix activated overlay ordering * Check for active overlays equality in overlay list This removes the -p flag: Either both parser and engine will have the same overlays, or the command will fail. * Add merging on overlay remove * Change help message and comment * Add some remove-merge/discard tests * (WIP) Track removed overlays properly * Clippy; Fmt * Fix getting last overlay; Fix predecls in overlays * Remove merging; Fix re-add overwriting stuff Also some error message tweaks. * Fix overlay error in the engine * Update variable_completions.rs * Adds flags and optional arguments to view-source (#5446) * added flags and optional arguments to view-source * removed redundant code * removed redundant code * fmt * fix bug in shell_integration (#5450) * fix bug in shell_integration * add some comments * enable cd to work with directory abbreviations (#5452) * enable cd to work with abbreviations * add abbreviation example * fix tests * make it configurable * make cd recornize symblic link (#5454) * implement seq char command to generate single character sequence (#5453) * add tmp code * add seq char command * Add split number flag in `split row` (#5434) Signed-off-by: Yuheng Su <gipsyh.icu@gmail.com> * Add two more overlay tests * Add ModuleId to OverlayFrame * Fix env conversion accidentally activating overlay It activated overlay from permanent state prematurely which would cause `overlay add` to misbehave. * Remove unused parameter; Add overlay list test * Remove added traces * Add overlay commands examples * Modify TODO * Fix $nu.scope iteration * Disallow removing default overlay * Refactor some parser errors * Remove last overlay if no argument * Diversify overlay examples * Make it possible to update overlay's module In case the origin module updates, the overlay add loads the new module, makes it overlay's origin and applies the changes. Before, it was impossible to update the overlay if the module changed. Co-authored-by: JT <547158+jntrnr@users.noreply.github.com> Co-authored-by: pwygab <88221256+merelymyself@users.noreply.github.com> Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> Co-authored-by: WindSoilder <WindSoilder@outlook.com> Co-authored-by: Yuheng Su <gipsyh.icu@gmail.com>
2022-05-07 21:39:22 +02:00
b"overlay" => {
if spans.len() > 1 && working_set.get_span_contents(spans[1]) == b"list" {
// whitelist 'overlay list'
parse_call(
working_set,
&spans[pos..],
spans[0],
expand_aliases_denylist,
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression,
Overlays (#5375) * WIP: Start laying overlays * Rename Overlay->Module; Start adding overlay * Revamp adding overlay * Add overlay add tests; Disable debug print * Fix overlay add; Add overlay remove * Add overlay remove tests * Add missing overlay remove file * Add overlay list command * (WIP?) Enable overlays for env vars * Move OverlayFrames to ScopeFrames * (WIP) Move everything to overlays only ScopeFrame contains nothing but overlays now * Fix predecls * Fix wrong overlay id translation and aliases * Fix broken env lookup logic * Remove TODOs * Add overlay add + remove for environment * Add a few overlay tests; Fix overlay add name * Some cleanup; Fix overlay add/remove names * Clippy * Fmt * Remove walls of comments * List overlays from stack; Add debugging flag Currently, the engine state ordering is somehow broken. * Fix (?) overlay list test * Fix tests on Windows * Fix activated overlay ordering * Check for active overlays equality in overlay list This removes the -p flag: Either both parser and engine will have the same overlays, or the command will fail. * Add merging on overlay remove * Change help message and comment * Add some remove-merge/discard tests * (WIP) Track removed overlays properly * Clippy; Fmt * Fix getting last overlay; Fix predecls in overlays * Remove merging; Fix re-add overwriting stuff Also some error message tweaks. * Fix overlay error in the engine * Update variable_completions.rs * Adds flags and optional arguments to view-source (#5446) * added flags and optional arguments to view-source * removed redundant code * removed redundant code * fmt * fix bug in shell_integration (#5450) * fix bug in shell_integration * add some comments * enable cd to work with directory abbreviations (#5452) * enable cd to work with abbreviations * add abbreviation example * fix tests * make it configurable * make cd recornize symblic link (#5454) * implement seq char command to generate single character sequence (#5453) * add tmp code * add seq char command * Add split number flag in `split row` (#5434) Signed-off-by: Yuheng Su <gipsyh.icu@gmail.com> * Add two more overlay tests * Add ModuleId to OverlayFrame * Fix env conversion accidentally activating overlay It activated overlay from permanent state prematurely which would cause `overlay add` to misbehave. * Remove unused parameter; Add overlay list test * Remove added traces * Add overlay commands examples * Modify TODO * Fix $nu.scope iteration * Disallow removing default overlay * Refactor some parser errors * Remove last overlay if no argument * Diversify overlay examples * Make it possible to update overlay's module In case the origin module updates, the overlay add loads the new module, makes it overlay's origin and applies the changes. Before, it was impossible to update the overlay if the module changed. Co-authored-by: JT <547158+jntrnr@users.noreply.github.com> Co-authored-by: pwygab <88221256+merelymyself@users.noreply.github.com> Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> Co-authored-by: WindSoilder <WindSoilder@outlook.com> Co-authored-by: Yuheng Su <gipsyh.icu@gmail.com>
2022-05-07 21:39:22 +02:00
)
} else {
(
parse_call(
working_set,
&spans[pos..],
spans[0],
expand_aliases_denylist,
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression,
Overlays (#5375) * WIP: Start laying overlays * Rename Overlay->Module; Start adding overlay * Revamp adding overlay * Add overlay add tests; Disable debug print * Fix overlay add; Add overlay remove * Add overlay remove tests * Add missing overlay remove file * Add overlay list command * (WIP?) Enable overlays for env vars * Move OverlayFrames to ScopeFrames * (WIP) Move everything to overlays only ScopeFrame contains nothing but overlays now * Fix predecls * Fix wrong overlay id translation and aliases * Fix broken env lookup logic * Remove TODOs * Add overlay add + remove for environment * Add a few overlay tests; Fix overlay add name * Some cleanup; Fix overlay add/remove names * Clippy * Fmt * Remove walls of comments * List overlays from stack; Add debugging flag Currently, the engine state ordering is somehow broken. * Fix (?) overlay list test * Fix tests on Windows * Fix activated overlay ordering * Check for active overlays equality in overlay list This removes the -p flag: Either both parser and engine will have the same overlays, or the command will fail. * Add merging on overlay remove * Change help message and comment * Add some remove-merge/discard tests * (WIP) Track removed overlays properly * Clippy; Fmt * Fix getting last overlay; Fix predecls in overlays * Remove merging; Fix re-add overwriting stuff Also some error message tweaks. * Fix overlay error in the engine * Update variable_completions.rs * Adds flags and optional arguments to view-source (#5446) * added flags and optional arguments to view-source * removed redundant code * removed redundant code * fmt * fix bug in shell_integration (#5450) * fix bug in shell_integration * add some comments * enable cd to work with directory abbreviations (#5452) * enable cd to work with abbreviations * add abbreviation example * fix tests * make it configurable * make cd recornize symblic link (#5454) * implement seq char command to generate single character sequence (#5453) * add tmp code * add seq char command * Add split number flag in `split row` (#5434) Signed-off-by: Yuheng Su <gipsyh.icu@gmail.com> * Add two more overlay tests * Add ModuleId to OverlayFrame * Fix env conversion accidentally activating overlay It activated overlay from permanent state prematurely which would cause `overlay add` to misbehave. * Remove unused parameter; Add overlay list test * Remove added traces * Add overlay commands examples * Modify TODO * Fix $nu.scope iteration * Disallow removing default overlay * Refactor some parser errors * Remove last overlay if no argument * Diversify overlay examples * Make it possible to update overlay's module In case the origin module updates, the overlay add loads the new module, makes it overlay's origin and applies the changes. Before, it was impossible to update the overlay if the module changed. Co-authored-by: JT <547158+jntrnr@users.noreply.github.com> Co-authored-by: pwygab <88221256+merelymyself@users.noreply.github.com> Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> Co-authored-by: WindSoilder <WindSoilder@outlook.com> Co-authored-by: Yuheng Su <gipsyh.icu@gmail.com>
2022-05-07 21:39:22 +02:00
)
.0,
Some(ParseError::BuiltinCommandInPipeline(
"overlay".into(),
spans[0],
)),
)
}
}
b"where" => parse_where_expr(working_set, &spans[pos..], expand_aliases_denylist),
#[cfg(feature = "plugin")]
b"register" => (
parse_call(
working_set,
&spans[pos..],
spans[0],
expand_aliases_denylist,
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression,
)
.0,
Some(ParseError::BuiltinCommandInPipeline(
"plugin".into(),
spans[0],
)),
),
_ => parse_call(
working_set,
&spans[pos..],
spans[0],
expand_aliases_denylist,
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression,
),
}
2021-11-04 03:32:35 +01:00
};
let with_env = working_set.find_decl(b"with-env", &Type::Any);
2021-11-04 03:32:35 +01:00
if !shorthand.is_empty() {
if let Some(decl_id) = with_env {
let mut block = Block::default();
let ty = output.ty.clone();
block.pipelines = vec![Pipeline::from_vec(vec![output])];
2021-11-04 03:32:35 +01:00
let block_id = working_set.add_block(block);
let mut env_vars = vec![];
for sh in shorthand {
env_vars.push(sh.0);
env_vars.push(sh.1);
}
let arguments = vec![
Argument::Positional(Expression {
2021-11-04 03:32:35 +01:00
expr: Expr::List(env_vars),
span: span(&spans[..pos]),
ty: Type::Any,
2021-11-04 03:32:35 +01:00
custom_completion: None,
}),
Argument::Positional(Expression {
expr: Expr::Closure(block_id),
2021-11-04 03:32:35 +01:00
span: span(&spans[pos..]),
ty: Type::Closure,
2021-11-04 03:32:35 +01:00
custom_completion: None,
}),
2021-11-04 03:32:35 +01:00
];
2022-02-21 18:58:04 +01:00
let expr = Expr::Call(Box::new(Call {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
head: Span::unknown(),
2022-02-21 18:58:04 +01:00
decl_id,
arguments,
redirect_stdout: true,
redirect_stderr: false,
parser_info: vec![],
2022-02-21 18:58:04 +01:00
}));
2021-11-04 03:32:35 +01:00
(
Expression {
2022-02-21 18:58:04 +01:00
expr,
2021-11-04 03:32:35 +01:00
custom_completion: None,
span: span(spans),
ty,
2021-11-04 03:32:35 +01:00
},
err,
)
} else {
(output, err)
}
} else {
2021-11-04 03:32:35 +01:00
(output, err)
2021-07-01 02:01:04 +02:00
}
2021-09-02 10:25:22 +02:00
}
2021-07-01 02:01:04 +02:00
2021-09-02 10:25:22 +02:00
pub fn parse_variable(
working_set: &mut StateWorkingSet,
span: Span,
) -> (Option<VarId>, Option<ParseError>) {
let bytes = working_set.get_span_contents(span);
2021-07-01 02:01:04 +02:00
2021-09-02 10:25:22 +02:00
if is_variable(bytes) {
if let Some(var_id) = working_set.find_variable(bytes) {
let input = working_set.get_variable(var_id).ty.clone();
working_set.type_scope.add_type(input);
2021-09-02 10:25:22 +02:00
(Some(var_id), None)
2021-07-01 02:01:04 +02:00
} else {
2021-09-02 10:25:22 +02:00
(None, None)
2021-07-01 02:01:04 +02:00
}
2021-09-02 10:25:22 +02:00
} else {
(
None,
Some(ParseError::Expected("valid variable name".into(), span)),
)
2021-07-01 02:01:04 +02:00
}
2021-09-02 10:25:22 +02:00
}
2021-07-01 02:01:04 +02:00
pub fn parse_builtin_commands(
working_set: &mut StateWorkingSet,
lite_command: &LiteCommand,
expand_aliases_denylist: &[usize],
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression: bool,
) -> (Pipeline, Option<ParseError>) {
2023-03-10 22:20:31 +01:00
if !is_math_expression_like(working_set, lite_command.parts[0], expand_aliases_denylist)
&& !is_unaliasable_parser_keyword(working_set, &lite_command.parts)
{
let name = working_set.get_span_contents(lite_command.parts[0]);
if let Some(decl_id) = working_set.find_decl(name, &Type::Any) {
let cmd = working_set.get_decl(decl_id);
if cmd.is_alias() {
// Parse keywords that can be aliased. Note that we check for "unaliasable" keywords
// because alias can have any name, therefore, we can't check for "aliasable" keywords.
let (call_expr, err) = parse_call(
working_set,
&lite_command.parts,
lite_command.parts[0],
expand_aliases_denylist,
is_subexpression,
);
if err.is_none() {
if let Expression {
expr: Expr::Call(call),
..
} = call_expr
{
// Apply parse keyword side effects
let cmd = working_set.get_decl(call.decl_id);
match cmd.name() {
"overlay hide" => return parse_overlay_hide(working_set, call),
"overlay new" => return parse_overlay_new(working_set, call),
"overlay use" => {
return parse_overlay_use(
working_set,
call,
expand_aliases_denylist,
)
}
_ => { /* this alias is not a parser keyword */ }
}
}
}
}
}
}
let name = working_set.get_span_contents(lite_command.parts[0]);
2021-09-13 21:59:11 +02:00
match name {
b"def" | b"def-env" => parse_def(working_set, lite_command, None, expand_aliases_denylist),
b"extern" => parse_extern(working_set, lite_command, None, expand_aliases_denylist),
b"let" | b"const" => {
parse_let_or_const(working_set, &lite_command.parts, expand_aliases_denylist)
}
b"mut" => parse_mut(working_set, &lite_command.parts, expand_aliases_denylist),
b"for" => {
let (expr, err) = parse_for(working_set, &lite_command.parts, expand_aliases_denylist);
(Pipeline::from_vec(vec![expr]), err)
}
Re-implement aliases (#8123) # Description This PR adds an alternative alias implementation. Old aliases still work but you need to use `old-alias` instead of `alias`. Instead of replacing spans in the original code and re-parsing, which proved to be extremely error-prone and a constant source of panics, the new implementation creates a new command that references the old command. Consider the new alias defined as `alias ll = ls -l`. The parser creates a new command called `ll` and remembers that it is actually a `ls` command called with the `-l` flag. Then, when the parser sees the `ll` command, it will translate it to `ls -l` and passes to it any parameters that were passed to the call to `ll`. It works quite similar to how known externals defined with `extern` are implemented. The new alias implementation should work the same way as the old aliases, including exporting from modules, referencing both known and unknown externals. It seems to preserve custom completions and pipeline metadata. It is quite robust in most cases but there are some rough edges (see later). Fixes https://github.com/nushell/nushell/issues/7648, https://github.com/nushell/nushell/issues/8026, https://github.com/nushell/nushell/issues/7512, https://github.com/nushell/nushell/issues/5780, https://github.com/nushell/nushell/issues/7754 No effect: https://github.com/nushell/nushell/issues/8122 (we might revisit the completions code after this PR) Should use custom command instead: https://github.com/nushell/nushell/issues/6048 # User-Facing Changes Since aliases are now basically commands, it has some new implications: 1. `alias spam = "spam"` (requires command call) * **workaround**: use `alias spam = echo "spam"` 2. `def foo [] { 'foo' }; alias foo = ls -l` (foo defined more than once) * **workaround**: use different name (commands also have this limitation) 4. `alias ls = (ls | sort-by type name -i)` * **workaround**: Use custom command. _The common issue with this is that it is currently not easy to pass flags through custom commands and command referencing itself will lead to stack overflow. Both of these issues are meant to be addressed._ 5. TODO: Help messages, `which` command, `$nu.scope.aliases`, etc. * Should we treat the aliases as commands or should they be separated from regular commands? 6. Needs better error message and syntax highlight for recursed alias (`alias f = f`) 7. Can't create alias with the same name as existing command (`alias ls = ls -a`) * Might be possible to add support for it (not 100% sure) 8. Standalone `alias` doesn't list aliases anymore 9. Can't alias parser keywords (e.g., stuff like `alias ou = overlay use` won't work) * TODO: Needs a better error message when attempting to do so # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-02-27 08:44:05 +01:00
b"old-alias" => parse_old_alias(working_set, lite_command, None, expand_aliases_denylist),
b"alias" => parse_alias(working_set, lite_command, None, expand_aliases_denylist),
2022-12-30 16:44:37 +01:00
b"module" => parse_module(working_set, lite_command, expand_aliases_denylist),
b"use" => {
let (pipeline, _, err) =
parse_use(working_set, &lite_command.parts, expand_aliases_denylist);
(pipeline, err)
}
2023-03-10 22:20:31 +01:00
b"overlay" => parse_keyword(
working_set,
lite_command,
expand_aliases_denylist,
is_subexpression,
),
b"source" | b"source-env" => {
parse_source(working_set, &lite_command.parts, expand_aliases_denylist)
}
b"export" => parse_export_in_block(working_set, lite_command, expand_aliases_denylist),
b"hide" => parse_hide(working_set, &lite_command.parts, expand_aliases_denylist),
b"where" => parse_where(working_set, &lite_command.parts, expand_aliases_denylist),
2021-11-02 21:56:00 +01:00
#[cfg(feature = "plugin")]
b"register" => parse_register(working_set, &lite_command.parts, expand_aliases_denylist),
2021-09-13 21:59:11 +02:00
_ => {
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
let (expr, err) = parse_expression(
working_set,
&lite_command.parts,
expand_aliases_denylist,
is_subexpression,
);
2023-03-10 22:20:31 +01:00
(Pipeline::from_vec(vec![expr]), err)
}
}
2021-09-02 10:25:22 +02:00
}
2021-06-30 03:42:56 +02:00
2021-11-11 00:14:00 +01:00
pub fn parse_record(
working_set: &mut StateWorkingSet,
span: Span,
expand_aliases_denylist: &[usize],
2021-11-11 00:14:00 +01:00
) -> (Expression, Option<ParseError>) {
let bytes = working_set.get_span_contents(span);
let mut error = None;
let mut start = span.start;
let mut end = span.end;
if bytes.starts_with(b"{") {
start += 1;
} else {
error = error.or_else(|| {
Some(ParseError::Expected(
"{".into(),
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
Span::new(start, start + 1),
2021-11-11 00:14:00 +01:00
))
});
}
if bytes.ends_with(b"}") {
end -= 1;
} else {
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
error = error.or_else(|| Some(ParseError::Unclosed("}".into(), Span::new(end, end))));
2021-11-11 00:14:00 +01:00
}
Protocol: debug_assert!() Span to reflect a valid slice (#6806) Also enforce this by #[non_exhaustive] span such that going forward we cannot, in debug builds (1), construct invalid spans. The motivation for this stems from #6431 where I've seen crashes due to invalid slice indexing. My hope is this will mitigate such senarios 1. https://github.com/nushell/nushell/pull/6431#issuecomment-1278147241 # Description (description of your pull request here) # Tests Make sure you've done the following: - [ ] Add tests that cover your changes, either in the command examples, the crate/tests folder, or in the /tests folder. - [ ] Try to think about corner cases and various ways how your changes could break. Cover them with tests. - [ ] If adding tests is not possible, please document in the PR body a minimal example with steps on how to reproduce so one can verify your change works. Make sure you've run and fixed any issues with these commands: - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - [ ] `cargo test --workspace --features=extra` to check that all the tests pass # Documentation - [ ] If your PR touches a user-facing nushell feature then make sure that there is an entry in the documentation (https://github.com/nushell/nushell.github.io) for the feature, and update it if necessary.
2022-12-03 10:44:12 +01:00
let inner_span = Span::new(start, end);
2022-01-03 06:21:26 +01:00
let source = working_set.get_span_contents(inner_span);
2021-11-11 00:14:00 +01:00
2021-11-21 19:13:09 +01:00
let (tokens, err) = lex(source, start, &[b'\n', b'\r', b','], &[b':'], true);
2021-11-11 00:14:00 +01:00
error = error.or(err);
let mut output = vec![];
let mut idx = 0;
let mut field_types = Some(vec![]);
2021-11-11 00:14:00 +01:00
while idx < tokens.len() {
let (field, err) = parse_value(
working_set,
tokens[idx].span,
&SyntaxShape::Any,
expand_aliases_denylist,
);
2021-11-11 00:14:00 +01:00
error = error.or(err);
idx += 1;
if idx == tokens.len() {
return (
garbage(span),
Some(ParseError::Expected("record".into(), span)),
);
}
let colon = working_set.get_span_contents(tokens[idx].span);
idx += 1;
if idx == tokens.len() || colon != b":" {
//FIXME: need better error
return (
garbage(span),
Some(ParseError::Expected("record".into(), span)),
);
}
let (value, err) = parse_value(
working_set,
tokens[idx].span,
&SyntaxShape::Any,
expand_aliases_denylist,
);
2021-11-11 00:14:00 +01:00
error = error.or(err);
idx += 1;
if let Some(field) = field.as_string() {
if let Some(fields) = &mut field_types {
fields.push((field, value.ty.clone()));
}
} else {
// We can't properly see all the field types
// so fall back to the Any type later
field_types = None;
}
2021-11-11 00:14:00 +01:00
output.push((field, value));
}
(
Expression {
expr: Expr::Record(output),
span,
ty: (if let Some(fields) = field_types {
Type::Record(fields)
} else {
Type::Any
}),
2021-11-11 00:14:00 +01:00
custom_completion: None,
},
error,
)
}
2021-09-02 10:25:22 +02:00
pub fn parse_block(
working_set: &mut StateWorkingSet,
tokens: &[Token],
2021-09-02 10:25:22 +02:00
scoped: bool,
expand_aliases_denylist: &[usize],
is_subexpression: bool,
2021-09-02 10:25:22 +02:00
) -> (Block, Option<ParseError>) {
let mut error = None;
let (lite_block, err) = lite_parse(tokens);
error = error.or(err);
2022-01-01 22:42:50 +01:00
trace!("parsing block: {:?}", lite_block);
2021-09-02 10:25:22 +02:00
if scoped {
working_set.enter_scope();
}
working_set.type_scope.enter_scope();
2021-06-30 03:42:56 +02:00
2021-09-02 10:25:22 +02:00
// Pre-declare any definition so that definitions
// that share the same block can see each other
for pipeline in &lite_block.block {
if pipeline.commands.len() == 1 {
match &pipeline.commands[0] {
Support redirect `err` and `out` to different streams (#7685) # Description Closes: #7364 # User-Facing Changes Given the following shell script: ```bash x=$(printf '=%.0s' {1..100}) echo $x echo $x 1>&2 ``` It supports the following command: ``` bash test.sh out> out.txt err> err.txt ``` Then both `out.txt` and `err.txt` will contain `=`(100 times) ## About the change The core idea is that when doing lite-parsing, introduce a new variant `LiteElement::SeparateRedirection` if we meet two Redirection token(which is generated by `lex` function), During converting from lite block to block, `LiteElement::SeparateRedirection` will be converted to `PipelineElement::SeparateRedirection`. Then in the block eval process, if we get `PipelineElement::SeparateRedirection`, we invoke `save` command with `--stderr` arguments to acthive our behavior. ## What happened internally? Take the following command as example: ``` ^ls out> out.txt err> err.txt ``` lex parsing result(`Tokens`) are not changed, but `LiteBlock` and `Block` is changed after this pr. ### LiteBlock before ```rust LiteBlock { block: [ LitePipeline { commands: [ Command(None, LiteCommand { comments: [], parts: [Span { start: 39041, end: 39044 }] }), // actually the span of first Redirection is wrong too.. Redirection(Span { start: 39058, end: 39062 }, Stdout, LiteCommand { comments: [], parts: [Span { start: 39050, end: 39057 }] }), Redirection(Span { start: 39058, end: 39062 }, Stderr, LiteCommand { comments: [], parts: [Span { start: 39063, end: 39070 }] }) ] }] } ``` ### LiteBlock after ```rust LiteBlock { block: [ LitePipeline { commands: [ Command( None, LiteCommand { comments: [], parts: [Span { start: 38525, end: 38528 }] }), // new one! two Redirection merged into one SeparateRedirection. SeparateRedirection { out: (Span { start: 38529, end: 38533 }, LiteCommand { comments: [], parts: [Span { start: 38534, end: 38541 }] }), err: (Span { start: 38542, end: 38546 }, LiteCommand { comments: [], parts: [Span { start: 38547, end: 38554 }] }) } ] }] } ``` ### Block before ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 39042, end: 39044 }, ty: String, custom_completion: None }, [], false), span: Span { start: 39041, end: 39044 }, ty: Any, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stdout, Expression { expr: String("out.txt"), span: Span { start: 39050, end: 39057 }, ty: String, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stderr, Expression { expr: String("err.txt"), span: Span { start: 39063, end: 39070 }, ty: String, custom_completion: None })] } ``` ### Block after ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 38526, end: 38528 }, ty: String, custom_completion: None }, [], false), span: Span { start: 38525, end: 38528 }, ty: Any, custom_completion: None }), // new one! SeparateRedirection SeparateRedirection { out: (Span { start: 38529, end: 38533 }, Expression { expr: String("out.txt"), span: Span { start: 38534, end: 38541 }, ty: String, custom_completion: None }), err: (Span { start: 38542, end: 38546 }, Expression { expr: String("err.txt"), span: Span { start: 38547, end: 38554 }, ty: String, custom_completion: None }) } ] } ``` # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-01-12 10:22:30 +01:00
LiteElement::Command(_, command)
| LiteElement::Redirection(_, _, command)
| LiteElement::SeparateRedirection {
out: (_, command), ..
} => {
if let Some(err) =
parse_def_predecl(working_set, &command.parts, expand_aliases_denylist)
{
error = error.or(Some(err));
}
}
}
2021-09-02 10:25:22 +02:00
}
}
2021-07-01 02:01:04 +02:00
2021-09-10 09:28:43 +02:00
let block: Block = lite_block
.block
.iter()
.enumerate()
.map(|(idx, pipeline)| {
2021-09-10 09:28:43 +02:00
if pipeline.commands.len() > 1 {
let mut output = pipeline
2021-09-10 09:28:43 +02:00
.commands
.iter()
.map(|command| match command {
LiteElement::Command(span, command) => {
trace!("parsing: pipeline element: command");
let (expr, err) = parse_expression(
working_set,
&command.parts,
expand_aliases_denylist,
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
is_subexpression,
);
working_set.type_scope.add_type(expr.ty.clone());
if error.is_none() {
error = err;
}
PipelineElement::Expression(*span, expr)
}
LiteElement::Redirection(span, redirection, command) => {
trace!("parsing: pipeline element: redirection");
let (expr, err) = parse_string(
working_set,
command.parts[0],
expand_aliases_denylist,
);
working_set.type_scope.add_type(expr.ty.clone());
if error.is_none() {
error = err;
}
PipelineElement::Redirection(*span, redirection.clone(), expr)
}
Support redirect `err` and `out` to different streams (#7685) # Description Closes: #7364 # User-Facing Changes Given the following shell script: ```bash x=$(printf '=%.0s' {1..100}) echo $x echo $x 1>&2 ``` It supports the following command: ``` bash test.sh out> out.txt err> err.txt ``` Then both `out.txt` and `err.txt` will contain `=`(100 times) ## About the change The core idea is that when doing lite-parsing, introduce a new variant `LiteElement::SeparateRedirection` if we meet two Redirection token(which is generated by `lex` function), During converting from lite block to block, `LiteElement::SeparateRedirection` will be converted to `PipelineElement::SeparateRedirection`. Then in the block eval process, if we get `PipelineElement::SeparateRedirection`, we invoke `save` command with `--stderr` arguments to acthive our behavior. ## What happened internally? Take the following command as example: ``` ^ls out> out.txt err> err.txt ``` lex parsing result(`Tokens`) are not changed, but `LiteBlock` and `Block` is changed after this pr. ### LiteBlock before ```rust LiteBlock { block: [ LitePipeline { commands: [ Command(None, LiteCommand { comments: [], parts: [Span { start: 39041, end: 39044 }] }), // actually the span of first Redirection is wrong too.. Redirection(Span { start: 39058, end: 39062 }, Stdout, LiteCommand { comments: [], parts: [Span { start: 39050, end: 39057 }] }), Redirection(Span { start: 39058, end: 39062 }, Stderr, LiteCommand { comments: [], parts: [Span { start: 39063, end: 39070 }] }) ] }] } ``` ### LiteBlock after ```rust LiteBlock { block: [ LitePipeline { commands: [ Command( None, LiteCommand { comments: [], parts: [Span { start: 38525, end: 38528 }] }), // new one! two Redirection merged into one SeparateRedirection. SeparateRedirection { out: (Span { start: 38529, end: 38533 }, LiteCommand { comments: [], parts: [Span { start: 38534, end: 38541 }] }), err: (Span { start: 38542, end: 38546 }, LiteCommand { comments: [], parts: [Span { start: 38547, end: 38554 }] }) } ] }] } ``` ### Block before ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 39042, end: 39044 }, ty: String, custom_completion: None }, [], false), span: Span { start: 39041, end: 39044 }, ty: Any, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stdout, Expression { expr: String("out.txt"), span: Span { start: 39050, end: 39057 }, ty: String, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stderr, Expression { expr: String("err.txt"), span: Span { start: 39063, end: 39070 }, ty: String, custom_completion: None })] } ``` ### Block after ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 38526, end: 38528 }, ty: String, custom_completion: None }, [], false), span: Span { start: 38525, end: 38528 }, ty: Any, custom_completion: None }), // new one! SeparateRedirection SeparateRedirection { out: (Span { start: 38529, end: 38533 }, Expression { expr: String("out.txt"), span: Span { start: 38534, end: 38541 }, ty: String, custom_completion: None }), err: (Span { start: 38542, end: 38546 }, Expression { expr: String("err.txt"), span: Span { start: 38547, end: 38554 }, ty: String, custom_completion: None }) } ] } ``` # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-01-12 10:22:30 +01:00
LiteElement::SeparateRedirection {
out: (out_span, out_command),
err: (err_span, err_command),
} => {
trace!("parsing: pipeline element: separate redirection");
let (out_expr, out_err) = parse_string(
working_set,
out_command.parts[0],
expand_aliases_denylist,
);
working_set.type_scope.add_type(out_expr.ty.clone());
if error.is_none() {
error = out_err;
}
let (err_expr, err_err) = parse_string(
working_set,
err_command.parts[0],
expand_aliases_denylist,
);
working_set.type_scope.add_type(err_expr.ty.clone());
if error.is_none() {
error = err_err;
}
PipelineElement::SeparateRedirection {
out: (*out_span, out_expr),
err: (*err_span, err_expr),
}
}
2021-09-10 09:28:43 +02:00
})
.collect::<Vec<PipelineElement>>();
2021-09-10 09:28:43 +02:00
if is_subexpression {
for element in output.iter_mut().skip(1) {
if element.has_in_variable(working_set) {
*element = wrap_element_with_collect(working_set, element);
}
}
} else {
for element in output.iter_mut() {
if element.has_in_variable(working_set) {
*element = wrap_element_with_collect(working_set, element);
}
}
}
2021-11-08 08:13:55 +01:00
Pipeline { elements: output }
2021-09-10 09:28:43 +02:00
} else {
match &pipeline.commands[0] {
Support redirect `err` and `out` to different streams (#7685) # Description Closes: #7364 # User-Facing Changes Given the following shell script: ```bash x=$(printf '=%.0s' {1..100}) echo $x echo $x 1>&2 ``` It supports the following command: ``` bash test.sh out> out.txt err> err.txt ``` Then both `out.txt` and `err.txt` will contain `=`(100 times) ## About the change The core idea is that when doing lite-parsing, introduce a new variant `LiteElement::SeparateRedirection` if we meet two Redirection token(which is generated by `lex` function), During converting from lite block to block, `LiteElement::SeparateRedirection` will be converted to `PipelineElement::SeparateRedirection`. Then in the block eval process, if we get `PipelineElement::SeparateRedirection`, we invoke `save` command with `--stderr` arguments to acthive our behavior. ## What happened internally? Take the following command as example: ``` ^ls out> out.txt err> err.txt ``` lex parsing result(`Tokens`) are not changed, but `LiteBlock` and `Block` is changed after this pr. ### LiteBlock before ```rust LiteBlock { block: [ LitePipeline { commands: [ Command(None, LiteCommand { comments: [], parts: [Span { start: 39041, end: 39044 }] }), // actually the span of first Redirection is wrong too.. Redirection(Span { start: 39058, end: 39062 }, Stdout, LiteCommand { comments: [], parts: [Span { start: 39050, end: 39057 }] }), Redirection(Span { start: 39058, end: 39062 }, Stderr, LiteCommand { comments: [], parts: [Span { start: 39063, end: 39070 }] }) ] }] } ``` ### LiteBlock after ```rust LiteBlock { block: [ LitePipeline { commands: [ Command( None, LiteCommand { comments: [], parts: [Span { start: 38525, end: 38528 }] }), // new one! two Redirection merged into one SeparateRedirection. SeparateRedirection { out: (Span { start: 38529, end: 38533 }, LiteCommand { comments: [], parts: [Span { start: 38534, end: 38541 }] }), err: (Span { start: 38542, end: 38546 }, LiteCommand { comments: [], parts: [Span { start: 38547, end: 38554 }] }) } ] }] } ``` ### Block before ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 39042, end: 39044 }, ty: String, custom_completion: None }, [], false), span: Span { start: 39041, end: 39044 }, ty: Any, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stdout, Expression { expr: String("out.txt"), span: Span { start: 39050, end: 39057 }, ty: String, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stderr, Expression { expr: String("err.txt"), span: Span { start: 39063, end: 39070 }, ty: String, custom_completion: None })] } ``` ### Block after ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 38526, end: 38528 }, ty: String, custom_completion: None }, [], false), span: Span { start: 38525, end: 38528 }, ty: Any, custom_completion: None }), // new one! SeparateRedirection SeparateRedirection { out: (Span { start: 38529, end: 38533 }, Expression { expr: String("out.txt"), span: Span { start: 38534, end: 38541 }, ty: String, custom_completion: None }), err: (Span { start: 38542, end: 38546 }, Expression { expr: String("err.txt"), span: Span { start: 38547, end: 38554 }, ty: String, custom_completion: None }) } ] } ``` # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-01-12 10:22:30 +01:00
LiteElement::Command(_, command)
| LiteElement::Redirection(_, _, command)
| LiteElement::SeparateRedirection {
out: (_, command), ..
} => {
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
let (mut pipeline, err) = parse_builtin_commands(
working_set,
command,
expand_aliases_denylist,
is_subexpression,
);
if idx == 0 {
if let Some(let_decl_id) = working_set.find_decl(b"let", &Type::Any) {
if let Some(let_env_decl_id) =
working_set.find_decl(b"let-env", &Type::Any)
{
for element in pipeline.elements.iter_mut() {
if let PipelineElement::Expression(
_,
Expression {
expr: Expr::Call(call),
..
},
) = element
{
if call.decl_id == let_decl_id
|| call.decl_id == let_env_decl_id
{
// Do an expansion
if let Some(Expression {
expr: Expr::Keyword(_, _, expr),
..
}) = call.positional_iter_mut().nth(1)
{
if expr.has_in_variable(working_set) {
*expr = Box::new(wrap_expr_with_collect(
working_set,
expr,
));
}
}
continue;
} else if element.has_in_variable(working_set)
&& !is_subexpression
{
*element =
wrap_element_with_collect(working_set, element);
}
} else if element.has_in_variable(working_set)
&& !is_subexpression
{
*element =
wrap_element_with_collect(working_set, element);
}
}
}
}
}
if error.is_none() {
error = err;
}
pipeline
}
}
2021-09-10 09:28:43 +02:00
}
})
.into();
2021-06-30 03:42:56 +02:00
2021-09-02 10:25:22 +02:00
if scoped {
working_set.exit_scope();
2021-06-30 03:42:56 +02:00
}
working_set.type_scope.exit_scope();
2021-06-30 03:42:56 +02:00
2021-09-02 10:25:22 +02:00
(block, error)
}
2021-06-30 03:42:56 +02:00
pub fn discover_captures_in_closure(
2021-10-25 22:04:23 +02:00
working_set: &StateWorkingSet,
block: &Block,
seen: &mut Vec<VarId>,
seen_blocks: &mut HashMap<BlockId, Vec<(VarId, Span)>>,
) -> Result<Vec<(VarId, Span)>, ParseError> {
2021-10-25 22:04:23 +02:00
let mut output = vec![];
for flag in &block.signature.named {
if let Some(var_id) = flag.var_id {
seen.push(var_id);
}
}
for positional in &block.signature.required_positional {
if let Some(var_id) = positional.var_id {
seen.push(var_id);
}
}
for positional in &block.signature.optional_positional {
if let Some(var_id) = positional.var_id {
seen.push(var_id);
}
}
for positional in &block.signature.rest_positional {
if let Some(var_id) = positional.var_id {
seen.push(var_id);
}
}
for pipeline in &block.pipelines {
let result = discover_captures_in_pipeline(working_set, pipeline, seen, seen_blocks)?;
output.extend(&result);
2021-10-25 22:04:23 +02:00
}
Ok(output)
2021-10-25 22:04:23 +02:00
}
fn discover_captures_in_pipeline(
2021-10-25 22:04:23 +02:00
working_set: &StateWorkingSet,
pipeline: &Pipeline,
seen: &mut Vec<VarId>,
seen_blocks: &mut HashMap<BlockId, Vec<(VarId, Span)>>,
) -> Result<Vec<(VarId, Span)>, ParseError> {
2021-10-25 22:04:23 +02:00
let mut output = vec![];
for element in &pipeline.elements {
let result =
discover_captures_in_pipeline_element(working_set, element, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
Ok(output)
2021-10-25 22:04:23 +02:00
}
// Closes over captured variables
pub fn discover_captures_in_pipeline_element(
working_set: &StateWorkingSet,
element: &PipelineElement,
seen: &mut Vec<VarId>,
seen_blocks: &mut HashMap<BlockId, Vec<(VarId, Span)>>,
) -> Result<Vec<(VarId, Span)>, ParseError> {
match element {
PipelineElement::Expression(_, expression)
| PipelineElement::Redirection(_, _, expression)
| PipelineElement::And(_, expression)
| PipelineElement::Or(_, expression) => {
discover_captures_in_expr(working_set, expression, seen, seen_blocks)
}
Support redirect `err` and `out` to different streams (#7685) # Description Closes: #7364 # User-Facing Changes Given the following shell script: ```bash x=$(printf '=%.0s' {1..100}) echo $x echo $x 1>&2 ``` It supports the following command: ``` bash test.sh out> out.txt err> err.txt ``` Then both `out.txt` and `err.txt` will contain `=`(100 times) ## About the change The core idea is that when doing lite-parsing, introduce a new variant `LiteElement::SeparateRedirection` if we meet two Redirection token(which is generated by `lex` function), During converting from lite block to block, `LiteElement::SeparateRedirection` will be converted to `PipelineElement::SeparateRedirection`. Then in the block eval process, if we get `PipelineElement::SeparateRedirection`, we invoke `save` command with `--stderr` arguments to acthive our behavior. ## What happened internally? Take the following command as example: ``` ^ls out> out.txt err> err.txt ``` lex parsing result(`Tokens`) are not changed, but `LiteBlock` and `Block` is changed after this pr. ### LiteBlock before ```rust LiteBlock { block: [ LitePipeline { commands: [ Command(None, LiteCommand { comments: [], parts: [Span { start: 39041, end: 39044 }] }), // actually the span of first Redirection is wrong too.. Redirection(Span { start: 39058, end: 39062 }, Stdout, LiteCommand { comments: [], parts: [Span { start: 39050, end: 39057 }] }), Redirection(Span { start: 39058, end: 39062 }, Stderr, LiteCommand { comments: [], parts: [Span { start: 39063, end: 39070 }] }) ] }] } ``` ### LiteBlock after ```rust LiteBlock { block: [ LitePipeline { commands: [ Command( None, LiteCommand { comments: [], parts: [Span { start: 38525, end: 38528 }] }), // new one! two Redirection merged into one SeparateRedirection. SeparateRedirection { out: (Span { start: 38529, end: 38533 }, LiteCommand { comments: [], parts: [Span { start: 38534, end: 38541 }] }), err: (Span { start: 38542, end: 38546 }, LiteCommand { comments: [], parts: [Span { start: 38547, end: 38554 }] }) } ] }] } ``` ### Block before ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 39042, end: 39044 }, ty: String, custom_completion: None }, [], false), span: Span { start: 39041, end: 39044 }, ty: Any, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stdout, Expression { expr: String("out.txt"), span: Span { start: 39050, end: 39057 }, ty: String, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stderr, Expression { expr: String("err.txt"), span: Span { start: 39063, end: 39070 }, ty: String, custom_completion: None })] } ``` ### Block after ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 38526, end: 38528 }, ty: String, custom_completion: None }, [], false), span: Span { start: 38525, end: 38528 }, ty: Any, custom_completion: None }), // new one! SeparateRedirection SeparateRedirection { out: (Span { start: 38529, end: 38533 }, Expression { expr: String("out.txt"), span: Span { start: 38534, end: 38541 }, ty: String, custom_completion: None }), err: (Span { start: 38542, end: 38546 }, Expression { expr: String("err.txt"), span: Span { start: 38547, end: 38554 }, ty: String, custom_completion: None }) } ] } ``` # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-01-12 10:22:30 +01:00
PipelineElement::SeparateRedirection {
out: (_, out_expr),
err: (_, err_expr),
} => {
let mut result = discover_captures_in_expr(working_set, out_expr, seen, seen_blocks)?;
result.append(&mut discover_captures_in_expr(
working_set,
err_expr,
seen,
seen_blocks,
)?);
Ok(result)
}
}
}
pub fn discover_captures_in_pattern(pattern: &MatchPattern, seen: &mut Vec<VarId>) {
match &pattern.pattern {
Pattern::Variable(var_id) => seen.push(*var_id),
Pattern::List(items) => {
for item in items {
discover_captures_in_pattern(item, seen)
}
}
Pattern::Record(items) => {
for item in items {
discover_captures_in_pattern(&item.1, seen)
}
}
Pattern::Or(patterns) => {
for pattern in patterns {
discover_captures_in_pattern(pattern, seen)
}
}
Pattern::Rest(var_id) => seen.push(*var_id),
Pattern::Value(_) | Pattern::IgnoreValue | Pattern::IgnoreRest | Pattern::Garbage => {}
}
}
// Closes over captured variables
pub fn discover_captures_in_expr(
2021-10-25 22:04:23 +02:00
working_set: &StateWorkingSet,
expr: &Expression,
seen: &mut Vec<VarId>,
seen_blocks: &mut HashMap<BlockId, Vec<(VarId, Span)>>,
) -> Result<Vec<(VarId, Span)>, ParseError> {
let mut output: Vec<(VarId, Span)> = vec![];
2021-10-25 22:04:23 +02:00
match &expr.expr {
Expr::BinaryOp(lhs, _, rhs) => {
let lhs_result = discover_captures_in_expr(working_set, lhs, seen, seen_blocks)?;
let rhs_result = discover_captures_in_expr(working_set, rhs, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&lhs_result);
output.extend(&rhs_result);
}
2022-04-06 21:10:25 +02:00
Expr::UnaryNot(expr) => {
let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?;
2022-04-06 21:10:25 +02:00
output.extend(&result);
}
Expr::Closure(block_id) => {
let block = working_set.get_block(*block_id);
let results = {
let mut seen = vec![];
let results =
discover_captures_in_closure(working_set, block, &mut seen, seen_blocks)?;
for (var_id, span) in results.iter() {
if !seen.contains(var_id) {
if let Some(variable) = working_set.get_variable_if_possible(*var_id) {
if variable.mutable {
return Err(ParseError::CaptureOfMutableVar(*span));
}
}
}
}
results
};
seen_blocks.insert(*block_id, results.clone());
for (var_id, span) in results.into_iter() {
if !seen.contains(&var_id) {
output.push((var_id, span))
}
}
}
2021-10-25 22:04:23 +02:00
Expr::Block(block_id) => {
let block = working_set.get_block(*block_id);
// FIXME: is this correct?
let results = {
let mut seen = vec![];
discover_captures_in_closure(working_set, block, &mut seen, seen_blocks)?
};
seen_blocks.insert(*block_id, results.clone());
for (var_id, span) in results.into_iter() {
if !seen.contains(&var_id) {
output.push((var_id, span))
}
}
2021-10-25 22:04:23 +02:00
}
2022-03-01 00:31:53 +01:00
Expr::Binary(_) => {}
2021-10-25 22:04:23 +02:00
Expr::Bool(_) => {}
Expr::Call(call) => {
let decl = working_set.get_decl(call.decl_id);
if let Some(block_id) = decl.get_block_id() {
match seen_blocks.get(&block_id) {
Some(capture_list) => {
output.extend(capture_list);
}
None => {
let block = working_set.get_block(block_id);
if !block.captures.is_empty() {
output.extend(block.captures.iter().map(|var_id| (*var_id, call.head)));
} else {
let mut seen = vec![];
seen_blocks.insert(block_id, output.clone());
let result = discover_captures_in_closure(
working_set,
block,
&mut seen,
seen_blocks,
)?;
output.extend(&result);
seen_blocks.insert(block_id, result);
}
}
}
}
for named in call.named_iter() {
if let Some(arg) = &named.2 {
let result = discover_captures_in_expr(working_set, arg, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
}
for positional in call.positional_iter() {
let result = discover_captures_in_expr(working_set, positional, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
}
Expr::CellPath(_) => {}
Expr::DateTime(_) => {}
Make external command substitution works friendly(like fish shell, trailing ending newlines) (#7156) # Description As title, when execute external sub command, auto-trimming end new-lines, like how fish shell does. And if the command is executed directly like: `cat tmp`, the result won't change. Fixes: #6816 Fixes: #3980 Note that although nushell works correctly by directly replace output of external command to variable(or other places like string interpolation), it's not friendly to user, and users almost want to use `str trim` to trim trailing newline, I think that's why fish shell do this automatically. If the pr is ok, as a result, no more `str trim -r` is required when user is writing scripts which using external commands. # User-Facing Changes Before: <img width="523" alt="img" src="https://user-images.githubusercontent.com/22256154/202468810-86b04dbb-c147-459a-96a5-e0095eeaab3d.png"> After: <img width="505" alt="img" src="https://user-images.githubusercontent.com/22256154/202468599-7b537488-3d6b-458e-9d75-d85780826db0.png"> # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace --features=extra` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2022-11-23 04:51:57 +01:00
Expr::ExternalCall(head, exprs, _) => {
let result = discover_captures_in_expr(working_set, head, seen, seen_blocks)?;
output.extend(&result);
2021-10-25 22:04:23 +02:00
for expr in exprs {
let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
}
Expr::Filepath(_) => {}
Expr::Directory(_) => {}
2021-10-25 22:04:23 +02:00
Expr::Float(_) => {}
Expr::FullCellPath(cell_path) => {
let result =
discover_captures_in_expr(working_set, &cell_path.head, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
Expr::ImportPattern(_) => {}
Expr::Overlay(_) => {}
2021-10-25 22:04:23 +02:00
Expr::Garbage => {}
Expr::Nothing => {}
2021-10-25 22:04:23 +02:00
Expr::GlobPattern(_) => {}
Expr::Int(_) => {}
Expr::Keyword(_, _, expr) => {
let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
Expr::List(exprs) => {
for expr in exprs {
let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
}
Expr::Operator(_) => {}
Expr::Range(expr1, expr2, expr3, _) => {
if let Some(expr) = expr1 {
let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
if let Some(expr) = expr2 {
let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
if let Some(expr) = expr3 {
let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
}
2021-11-11 00:14:00 +01:00
Expr::Record(fields) => {
for (field_name, field_value) in fields {
output.extend(&discover_captures_in_expr(
working_set,
field_name,
seen,
seen_blocks,
)?);
output.extend(&discover_captures_in_expr(
working_set,
field_value,
seen,
seen_blocks,
)?);
2021-11-11 00:14:00 +01:00
}
}
Expr::Signature(sig) => {
// Something with a declaration, similar to a var decl, will introduce more VarIds into the stack at eval
for pos in &sig.required_positional {
if let Some(var_id) = pos.var_id {
seen.push(var_id);
}
}
for pos in &sig.optional_positional {
if let Some(var_id) = pos.var_id {
seen.push(var_id);
}
}
if let Some(rest) = &sig.rest_positional {
if let Some(var_id) = rest.var_id {
seen.push(var_id);
}
}
for named in &sig.named {
if let Some(var_id) = named.var_id {
seen.push(var_id);
}
}
}
2021-10-25 22:04:23 +02:00
Expr::String(_) => {}
Expr::StringInterpolation(exprs) => {
for expr in exprs {
let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?;
output.extend(&result);
}
}
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
Expr::MatchPattern(_) => {}
Expr::MatchBlock(match_block) => {
for match_ in match_block {
discover_captures_in_pattern(&match_.0, seen);
Add pattern matching (#8590) # Description This adds `match` and basic pattern matching. An example: ``` match $x { 1..10 => { print "Value is between 1 and 10" } { foo: $bar } => { print $"Value has a 'foo' field with value ($bar)" } [$a, $b] => { print $"Value is a list with two items: ($a) and ($b)" } _ => { print "Value is none of the above" } } ``` Like the recent changes to `if` to allow it to be used as an expression, `match` can also be used as an expression. This allows you to assign the result to a variable, eg) `let xyz = match ...` I've also included a short-hand pattern for matching records, as I think it might help when doing a lot of record patterns: `{$foo}` which is equivalent to `{foo: $foo}`. There are still missing components, so consider this the first step in full pattern matching support. Currently missing: * Patterns for strings * Or-patterns (like the `|` in Rust) * Patterns for tables (unclear how we want to match a table, so it'll need some design) * Patterns for binary values * And much more # User-Facing Changes [see above] # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-03-24 02:52:01 +01:00
let result = discover_captures_in_expr(working_set, &match_.1, seen, seen_blocks)?;
output.extend(&result);
}
}
2021-11-26 04:49:03 +01:00
Expr::RowCondition(block_id) | Expr::Subexpression(block_id) => {
let block = working_set.get_block(*block_id);
let results = {
let mut seen = vec![];
discover_captures_in_closure(working_set, block, &mut seen, seen_blocks)?
};
seen_blocks.insert(*block_id, results.clone());
for (var_id, span) in results.into_iter() {
if !seen.contains(&var_id) {
output.push((var_id, span))
}
}
2021-10-25 22:04:23 +02:00
}
Expr::Table(headers, values) => {
for header in headers {
let result = discover_captures_in_expr(working_set, header, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
for row in values {
for cell in row {
let result = discover_captures_in_expr(working_set, cell, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
}
}
Expr::ValueWithUnit(expr, _) => {
let result = discover_captures_in_expr(working_set, expr, seen, seen_blocks)?;
2021-10-25 22:04:23 +02:00
output.extend(&result);
}
Expr::Var(var_id) => {
if (*var_id > ENV_VARIABLE_ID || *var_id == IN_VARIABLE_ID) && !seen.contains(var_id) {
output.push((*var_id, expr.span));
2021-10-25 22:04:23 +02:00
}
}
Expr::VarDecl(var_id) => {
seen.push(*var_id);
}
}
Ok(output)
2021-10-25 22:04:23 +02:00
}
fn wrap_element_with_collect(
working_set: &mut StateWorkingSet,
element: &PipelineElement,
) -> PipelineElement {
match element {
PipelineElement::Expression(span, expression) => {
PipelineElement::Expression(*span, wrap_expr_with_collect(working_set, expression))
}
PipelineElement::Redirection(span, redirection, expression) => {
PipelineElement::Redirection(
*span,
redirection.clone(),
wrap_expr_with_collect(working_set, expression),
)
}
Support redirect `err` and `out` to different streams (#7685) # Description Closes: #7364 # User-Facing Changes Given the following shell script: ```bash x=$(printf '=%.0s' {1..100}) echo $x echo $x 1>&2 ``` It supports the following command: ``` bash test.sh out> out.txt err> err.txt ``` Then both `out.txt` and `err.txt` will contain `=`(100 times) ## About the change The core idea is that when doing lite-parsing, introduce a new variant `LiteElement::SeparateRedirection` if we meet two Redirection token(which is generated by `lex` function), During converting from lite block to block, `LiteElement::SeparateRedirection` will be converted to `PipelineElement::SeparateRedirection`. Then in the block eval process, if we get `PipelineElement::SeparateRedirection`, we invoke `save` command with `--stderr` arguments to acthive our behavior. ## What happened internally? Take the following command as example: ``` ^ls out> out.txt err> err.txt ``` lex parsing result(`Tokens`) are not changed, but `LiteBlock` and `Block` is changed after this pr. ### LiteBlock before ```rust LiteBlock { block: [ LitePipeline { commands: [ Command(None, LiteCommand { comments: [], parts: [Span { start: 39041, end: 39044 }] }), // actually the span of first Redirection is wrong too.. Redirection(Span { start: 39058, end: 39062 }, Stdout, LiteCommand { comments: [], parts: [Span { start: 39050, end: 39057 }] }), Redirection(Span { start: 39058, end: 39062 }, Stderr, LiteCommand { comments: [], parts: [Span { start: 39063, end: 39070 }] }) ] }] } ``` ### LiteBlock after ```rust LiteBlock { block: [ LitePipeline { commands: [ Command( None, LiteCommand { comments: [], parts: [Span { start: 38525, end: 38528 }] }), // new one! two Redirection merged into one SeparateRedirection. SeparateRedirection { out: (Span { start: 38529, end: 38533 }, LiteCommand { comments: [], parts: [Span { start: 38534, end: 38541 }] }), err: (Span { start: 38542, end: 38546 }, LiteCommand { comments: [], parts: [Span { start: 38547, end: 38554 }] }) } ] }] } ``` ### Block before ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 39042, end: 39044 }, ty: String, custom_completion: None }, [], false), span: Span { start: 39041, end: 39044 }, ty: Any, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stdout, Expression { expr: String("out.txt"), span: Span { start: 39050, end: 39057 }, ty: String, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, Stderr, Expression { expr: String("err.txt"), span: Span { start: 39063, end: 39070 }, ty: String, custom_completion: None })] } ``` ### Block after ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 38526, end: 38528 }, ty: String, custom_completion: None }, [], false), span: Span { start: 38525, end: 38528 }, ty: Any, custom_completion: None }), // new one! SeparateRedirection SeparateRedirection { out: (Span { start: 38529, end: 38533 }, Expression { expr: String("out.txt"), span: Span { start: 38534, end: 38541 }, ty: String, custom_completion: None }), err: (Span { start: 38542, end: 38546 }, Expression { expr: String("err.txt"), span: Span { start: 38547, end: 38554 }, ty: String, custom_completion: None }) } ] } ``` # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
2023-01-12 10:22:30 +01:00
PipelineElement::SeparateRedirection {
out: (out_span, out_exp),
err: (err_span, err_exp),
} => PipelineElement::SeparateRedirection {
out: (*out_span, wrap_expr_with_collect(working_set, out_exp)),
err: (*err_span, wrap_expr_with_collect(working_set, err_exp)),
},
PipelineElement::And(span, expression) => {
PipelineElement::And(*span, wrap_expr_with_collect(working_set, expression))
}
PipelineElement::Or(span, expression) => {
PipelineElement::Or(*span, wrap_expr_with_collect(working_set, expression))
}
}
}
fn wrap_expr_with_collect(working_set: &mut StateWorkingSet, expr: &Expression) -> Expression {
let span = expr.span;
if let Some(decl_id) = working_set.find_decl(b"collect", &Type::Any) {
let mut output = vec![];
let var_id = IN_VARIABLE_ID;
let mut signature = Signature::new("");
signature.required_positional.push(PositionalArg {
var_id: Some(var_id),
2021-12-30 04:26:40 +01:00
name: "$in".into(),
desc: String::new(),
shape: SyntaxShape::Any,
2022-03-07 21:08:56 +01:00
default_value: None,
});
let mut expr = expr.clone();
expr.replace_in_variable(working_set, var_id);
let block = Block {
pipelines: vec![Pipeline::from_vec(vec![expr])],
signature: Box::new(signature),
..Default::default()
};
let block_id = working_set.add_block(block);
output.push(Argument::Positional(Expression {
expr: Expr::Closure(block_id),
span,
ty: Type::Any,
custom_completion: None,
}));
output.push(Argument::Named((
Spanned {
item: "keep-env".to_string(),
span: Span::new(0, 0),
},
None,
None,
)));
2021-12-30 04:26:40 +01:00
// The containing, synthetic call to `collect`.
// We don't want to have a real span as it will confuse flattening
// The args are where we'll get the real info
Expression {
expr: Expr::Call(Box::new(Call {
2021-12-30 04:26:40 +01:00
head: Span::new(0, 0),
arguments: output,
decl_id,
redirect_stdout: true,
redirect_stderr: false,
parser_info: vec![],
})),
span,
ty: Type::String,
custom_completion: None,
}
} else {
Expression::garbage(span)
}
}
2021-09-06 22:41:30 +02:00
// Parses a vector of u8 to create an AST Block. If a file name is given, then
// the name is stored in the working set. When parsing a source without a file
// name, the source of bytes is stored as "source"
pub fn parse(
2021-09-02 10:25:22 +02:00
working_set: &mut StateWorkingSet,
2021-09-06 22:41:30 +02:00
fname: Option<&str>,
2021-09-02 10:25:22 +02:00
contents: &[u8],
scoped: bool,
expand_aliases_denylist: &[usize],
2021-09-02 10:25:22 +02:00
) -> (Block, Option<ParseError>) {
let mut error = None;
2021-07-03 03:29:56 +02:00
2021-09-02 10:25:22 +02:00
let span_offset = working_set.next_span_start();
2021-09-06 22:41:30 +02:00
let name = match fname {
Some(fname) => fname.to_string(),
None => "source".to_string(),
};
2021-07-01 02:01:04 +02:00
2021-09-06 22:41:30 +02:00
working_set.add_file(name, contents);
2021-07-01 02:01:04 +02:00
let (output, err) = lex(contents, span_offset, &[], &[], false);
2021-09-02 10:25:22 +02:00
error = error.or(err);
2021-07-01 02:01:04 +02:00
let (mut output, err) =
parse_block(working_set, &output, scoped, expand_aliases_denylist, false);
2021-09-02 10:25:22 +02:00
error = error.or(err);
let mut seen = vec![];
let mut seen_blocks = HashMap::new();
let captures = discover_captures_in_closure(working_set, &output, &mut seen, &mut seen_blocks);
match captures {
Ok(captures) => output.captures = captures.into_iter().map(|(var_id, _)| var_id).collect(),
Err(err) => error = Some(err),
}
// Also check other blocks that might have been imported
for (block_idx, block) in working_set.delta.blocks.iter().enumerate() {
let block_id = block_idx + working_set.permanent_state.num_blocks();
if !seen_blocks.contains_key(&block_id) {
let captures =
discover_captures_in_closure(working_set, block, &mut seen, &mut seen_blocks);
match captures {
Ok(captures) => {
seen_blocks.insert(block_id, captures);
}
Err(err) => error = Some(err),
}
}
}
for (block_id, captures) in seen_blocks.into_iter() {
// In theory, we should only be updating captures where we have new information
// the only place where this is possible would be blocks that are newly created
// by our working set delta. If we ever tried to modify the permanent state, we'd
// panic (again, in theory, this shouldn't be possible)
let block = working_set.get_block(block_id);
let block_captures_empty = block.captures.is_empty();
if !captures.is_empty() && block_captures_empty {
let block = working_set.get_block_mut(block_id);
block.captures = captures.into_iter().map(|(var_id, _)| var_id).collect();
}
}
2021-09-02 10:25:22 +02:00
(output, error)
2021-06-30 03:42:56 +02:00
}