Move most of the peculiar argument handling for external calls into the parser (#13089)

# Description

We've had a lot of different issues and PRs related to arg handling with
externals since the rewrite of `run-external` in #12921:

- #12950
- #12955
- #13000
- #13001
- #13021
- #13027
- #13028
- #13073

Many of these are caused by the argument handling of external calls and
`run-external` being very special and involving the parser handing
quoted strings over to `run-external` so that it knows whether to expand
tildes and globs and so on. This is really unusual and also makes it
harder to use `run-external`, and also harder to understand it (and
probably is part of the reason why it was rewritten in the first place).

This PR moves a lot more of that work over to the parser, so that by the
time `run-external` gets it, it's dealing with much more normal Nushell
values. In particular:

- Unquoted strings are handled as globs with no expand
- The unescaped-but-quoted handling of strings was removed, and the
parser constructs normal looking strings instead, removing internal
quotes so that `run-external` doesn't have to do it
- Bare word interpolation is now supported and expansion is done in this
case
- Expressions typed as `Glob` containing `Expr::StringInterpolation` now
produce `Value::Glob` instead, with the quoted status from the expr
passed through so we know if it was a bare word
- Bare word interpolation for values typed as `glob` now possible, but
not implemented
- Because expansion is now triggered by `Value::Glob(_, false)` instead
of looking at the expr, externals now support glob types

# User-Facing Changes

- Bare word interpolation works for external command options, and
otherwise embedded in other strings:
  ```nushell
  ^echo --foo=(2 + 2) # prints --foo=4
  ^echo -foo=$"(2 + 2)" # prints -foo=4
  ^echo foo="(2 + 2)" # prints (no interpolation!) foo=(2 + 2)
  ^echo foo,(2 + 2),bar # prints foo,4,bar
  ```

- Bare word interpolation expands for external command head/args:
  ```nushell
  let name = "exa"
  ~/.cargo/bin/($name) # this works, and expands the tilde
  ^$"~/.cargo/bin/($name)" # this doesn't expand the tilde
  ^echo ~/($name)/* # this glob is expanded
  ^echo $"~/($name)/*" # this isn't expanded
  ```

- Ndots are now supported for the head of an external command
(`^.../foo` works)

- Glob values are now supported for head/args of an external command,
and expanded appropriately:
  ```nushell
  ^("~/.cargo/bin/exa" | into glob) # the tilde is expanded
  ^echo ("*.txt" | into glob) # this glob is expanded
  ```

- `run-external` now works more like any other command, without
expecting a special call convention
  for its args:
  ```nushell
  run-external echo "'foo'"
  # before PR: 'foo'
  # after PR:  foo
  run-external echo "*.txt"
  # before PR: (glob is expanded)
  # after PR:  *.txt
  ```

# Tests + Formatting
Lots of tests added and cleaned up. Some tests that weren't active on
Windows changed to use `nu --testbin cococo` so that they can work.
Added a test for Linux only to make sure tilde expansion of commands
works, because changing `HOME` there causes `~` to reliably change.

- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

# After Submitting
- [ ] release notes: make sure to mention the new syntaxes that are
supported
This commit is contained in:
Devyn Cairns
2024-06-19 21:00:03 -07:00
committed by GitHub
parent 44aa0a2de4
commit bdc32345bd
13 changed files with 880 additions and 476 deletions

View File

@ -1,8 +1,8 @@
use nu_parser::*;
use nu_protocol::{
ast::{Argument, Call, Expr, ExternalArgument, PathMember, Range},
ast::{Argument, Call, Expr, Expression, ExternalArgument, PathMember, Range},
engine::{Command, EngineState, Stack, StateWorkingSet},
ParseError, PipelineData, ShellError, Signature, Span, SyntaxShape,
ParseError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
};
use rstest::rstest;
@ -182,7 +182,7 @@ pub fn multi_test_parse_int() {
Test(
"ranges or relative paths not confused for int",
b"./a/b",
Expr::String("./a/b".into()),
Expr::GlobPattern("./a/b".into(), false),
None,
),
Test(
@ -694,6 +694,50 @@ pub fn parse_call_missing_req_flag() {
));
}
fn test_external_call(input: &str, tag: &str, f: impl FnOnce(&Expression, &[ExternalArgument])) {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, input.as_bytes(), true);
assert!(
working_set.parse_errors.is_empty(),
"{tag}: errors: {:?}",
working_set.parse_errors
);
let pipeline = &block.pipelines[0];
assert_eq!(1, pipeline.len());
let element = &pipeline.elements[0];
match &element.expr.expr {
Expr::ExternalCall(name, args) => f(name, args),
other => {
panic!("{tag}: Unexpected expression in pipeline: {other:?}");
}
}
}
fn check_external_call_interpolation(
tag: &str,
subexpr_count: usize,
quoted: bool,
expr: &Expression,
) -> bool {
match &expr.expr {
Expr::StringInterpolation(exprs) => {
assert!(quoted, "{tag}: quoted");
assert_eq!(expr.ty, Type::String, "{tag}: expr.ty");
assert_eq!(subexpr_count, exprs.len(), "{tag}: subexpr_count");
true
}
Expr::GlobInterpolation(exprs, is_quoted) => {
assert_eq!(quoted, *is_quoted, "{tag}: quoted");
assert_eq!(expr.ty, Type::Glob, "{tag}: expr.ty");
assert_eq!(subexpr_count, exprs.len(), "{tag}: subexpr_count");
true
}
_ => false,
}
}
#[rstest]
#[case("foo-external-call", "foo-external-call", "bare word")]
#[case("^foo-external-call", "foo-external-call", "bare word with caret")]
@ -713,200 +757,370 @@ pub fn parse_call_missing_req_flag() {
r"foo\external-call",
"bare word with backslash and caret"
)]
#[case(
"^'foo external call'",
"'foo external call'",
"single quote with caret"
)]
#[case(
"^'foo/external call'",
"'foo/external call'",
"single quote with forward slash and caret"
)]
#[case(
r"^'foo\external call'",
r"'foo\external call'",
"single quote with backslash and caret"
)]
#[case(
r#"^"foo external call""#,
r#""foo external call""#,
"double quote with caret"
)]
#[case(
r#"^"foo/external call""#,
r#""foo/external call""#,
"double quote with forward slash and caret"
)]
#[case(
r#"^"foo\\external call""#,
r#""foo\external call""#,
"double quote with backslash and caret"
)]
#[case("`foo external call`", "`foo external call`", "backtick quote")]
#[case("`foo external call`", "foo external call", "backtick quote")]
#[case(
"^`foo external call`",
"`foo external call`",
"foo external call",
"backtick quote with caret"
)]
#[case(
"`foo/external call`",
"`foo/external call`",
"foo/external call",
"backtick quote with forward slash"
)]
#[case(
"^`foo/external call`",
"`foo/external call`",
"foo/external call",
"backtick quote with forward slash and caret"
)]
#[case(
r"^`foo\external call`",
r"`foo\external call`",
r"foo\external call",
"backtick quote with backslash"
)]
#[case(
r"^`foo\external call`",
r"`foo\external call`",
r"foo\external call",
"backtick quote with backslash and caret"
)]
fn test_external_call_name(#[case] input: &str, #[case] expected: &str, #[case] tag: &str) {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, input.as_bytes(), true);
assert!(
working_set.parse_errors.is_empty(),
"{tag}: errors: {:?}",
working_set.parse_errors
);
let pipeline = &block.pipelines[0];
assert_eq!(1, pipeline.len());
let element = &pipeline.elements[0];
match &element.expr.expr {
Expr::ExternalCall(name, args) => {
match &name.expr {
Expr::String(string) => {
assert_eq!(expected, string);
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(0, args.len());
}
other => {
panic!("{tag}: Unexpected expression in pipeline: {other:?}");
}
}
}
#[rstest]
#[case("^foo bar-baz", "bar-baz", "bare word")]
#[case("^foo bar/baz", "bar/baz", "bare word with forward slash")]
#[case(r"^foo bar\baz", r"bar\baz", "bare word with backslash")]
#[case("^foo 'bar baz'", "'bar baz'", "single quote")]
#[case("foo 'bar/baz'", "'bar/baz'", "single quote with forward slash")]
#[case(r"foo 'bar\baz'", r"'bar\baz'", "single quote with backslash")]
#[case(r#"^foo "bar baz""#, r#""bar baz""#, "double quote")]
#[case(r#"^foo "bar/baz""#, r#""bar/baz""#, "double quote with forward slash")]
#[case(r#"^foo "bar\\baz""#, r#""bar\baz""#, "double quote with backslash")]
#[case("^foo `bar baz`", "`bar baz`", "backtick quote")]
#[case("^foo `bar/baz`", "`bar/baz`", "backtick quote with forward slash")]
#[case(r"^foo `bar\baz`", r"`bar\baz`", "backtick quote with backslash")]
fn test_external_call_argument_regular(
pub fn test_external_call_head_glob(
#[case] input: &str,
#[case] expected: &str,
#[case] tag: &str,
) {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, input.as_bytes(), true);
assert!(
working_set.parse_errors.is_empty(),
"{tag}: errors: {:?}",
working_set.parse_errors
);
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, is_quoted) => {
assert_eq!(expected, string, "{tag}: incorrect name");
assert!(!*is_quoted);
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(0, args.len());
})
}
let pipeline = &block.pipelines[0];
assert_eq!(1, pipeline.len());
let element = &pipeline.elements[0];
match &element.expr.expr {
Expr::ExternalCall(name, args) => {
match &name.expr {
Expr::String(string) => {
assert_eq!("foo", string, "{tag}: incorrect name");
#[rstest]
#[case(
r##"^r#'foo-external-call'#"##,
"foo-external-call",
"raw string with caret"
)]
#[case(
r##"^r#'foo/external-call'#"##,
"foo/external-call",
"raw string with forward slash and caret"
)]
#[case(
r##"^r#'foo\external-call'#"##,
r"foo\external-call",
"raw string with backslash and caret"
)]
pub fn test_external_call_head_raw_string(
#[case] input: &str,
#[case] expected: &str,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::RawString(string) => {
assert_eq!(expected, string, "{tag}: incorrect name");
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(0, args.len());
})
}
#[rstest]
#[case("^'foo external call'", "foo external call", "single quote with caret")]
#[case(
"^'foo/external call'",
"foo/external call",
"single quote with forward slash and caret"
)]
#[case(
r"^'foo\external call'",
r"foo\external call",
"single quote with backslash and caret"
)]
#[case(
r#"^"foo external call""#,
r#"foo external call"#,
"double quote with caret"
)]
#[case(
r#"^"foo/external call""#,
r#"foo/external call"#,
"double quote with forward slash and caret"
)]
#[case(
r#"^"foo\\external call""#,
r#"foo\external call"#,
"double quote with backslash and caret"
)]
pub fn test_external_call_head_string(
#[case] input: &str,
#[case] expected: &str,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::String(string) => {
assert_eq!(expected, string);
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(0, args.len());
})
}
#[rstest]
#[case(r"~/.foo/(1)", 2, false, "unquoted interpolated string")]
#[case(
r"~\.foo(2)\(1)",
4,
false,
"unquoted interpolated string with backslash"
)]
#[case(r"^~/.foo/(1)", 2, false, "unquoted interpolated string with caret")]
#[case(r#"^$"~/.foo/(1)""#, 2, true, "quoted interpolated string with caret")]
pub fn test_external_call_head_interpolated_string(
#[case] input: &str,
#[case] subexpr_count: usize,
#[case] quoted: bool,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
if !check_external_call_interpolation(tag, subexpr_count, quoted, name) {
panic!("{tag}: Unexpected expression in command name position: {name:?}");
}
assert_eq!(0, args.len());
})
}
#[rstest]
#[case("^foo foo-external-call", "foo-external-call", "bare word")]
#[case(
"^foo foo/external-call",
"foo/external-call",
"bare word with forward slash"
)]
#[case(
r"^foo foo\external-call",
r"foo\external-call",
"bare word with backslash"
)]
#[case(
"^foo `foo external call`",
"foo external call",
"backtick quote with caret"
)]
#[case(
"^foo `foo/external call`",
"foo/external call",
"backtick quote with forward slash"
)]
#[case(
r"^foo `foo\external call`",
r"foo\external call",
"backtick quote with backslash"
)]
pub fn test_external_call_arg_glob(#[case] input: &str, #[case] expected: &str, #[case] tag: &str) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, _) => {
assert_eq!("foo", string, "{tag}: incorrect name");
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Regular(expr) => match &expr.expr {
Expr::GlobPattern(string, is_quoted) => {
assert_eq!(expected, string, "{tag}: incorrect arg");
assert!(!*is_quoted);
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Regular(expr) => match &expr.expr {
Expr::String(string) => {
assert_eq!(expected, string, "{tag}: incorrect arg");
}
other => {
panic!("Unexpected expression in command arg position: {other:?}")
}
},
other @ ExternalArgument::Spread(..) => {
panic!("Unexpected external spread argument in command arg position: {other:?}")
panic!("Unexpected expression in command arg position: {other:?}")
}
},
other @ ExternalArgument::Spread(..) => {
panic!("Unexpected external spread argument in command arg position: {other:?}")
}
}
other => {
panic!("{tag}: Unexpected expression in pipeline: {other:?}");
})
}
#[rstest]
#[case(r##"^foo r#'foo-external-call'#"##, "foo-external-call", "raw string")]
#[case(
r##"^foo r#'foo/external-call'#"##,
"foo/external-call",
"raw string with forward slash"
)]
#[case(
r##"^foo r#'foo\external-call'#"##,
r"foo\external-call",
"raw string with backslash"
)]
pub fn test_external_call_arg_raw_string(
#[case] input: &str,
#[case] expected: &str,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, _) => {
assert_eq!("foo", string, "{tag}: incorrect name");
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Regular(expr) => match &expr.expr {
Expr::RawString(string) => {
assert_eq!(expected, string, "{tag}: incorrect arg");
}
other => {
panic!("Unexpected expression in command arg position: {other:?}")
}
},
other @ ExternalArgument::Spread(..) => {
panic!("Unexpected external spread argument in command arg position: {other:?}")
}
}
})
}
#[rstest]
#[case("^foo 'foo external call'", "foo external call", "single quote")]
#[case(
"^foo 'foo/external call'",
"foo/external call",
"single quote with forward slash"
)]
#[case(
r"^foo 'foo\external call'",
r"foo\external call",
"single quote with backslash"
)]
#[case(r#"^foo "foo external call""#, r#"foo external call"#, "double quote")]
#[case(
r#"^foo "foo/external call""#,
r#"foo/external call"#,
"double quote with forward slash"
)]
#[case(
r#"^foo "foo\\external call""#,
r#"foo\external call"#,
"double quote with backslash"
)]
pub fn test_external_call_arg_string(
#[case] input: &str,
#[case] expected: &str,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, _) => {
assert_eq!("foo", string, "{tag}: incorrect name");
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Regular(expr) => match &expr.expr {
Expr::String(string) => {
assert_eq!(expected, string, "{tag}: incorrect arg");
}
other => {
panic!("{tag}: Unexpected expression in command arg position: {other:?}")
}
},
other @ ExternalArgument::Spread(..) => {
panic!(
"{tag}: Unexpected external spread argument in command arg position: {other:?}"
)
}
}
})
}
#[rstest]
#[case(r"^foo ~/.foo/(1)", 2, false, "unquoted interpolated string")]
#[case(r#"^foo $"~/.foo/(1)""#, 2, true, "quoted interpolated string")]
pub fn test_external_call_arg_interpolated_string(
#[case] input: &str,
#[case] subexpr_count: usize,
#[case] quoted: bool,
#[case] tag: &str,
) {
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, _) => {
assert_eq!("foo", string, "{tag}: incorrect name");
}
other => {
panic!("{tag}: Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Regular(expr) => {
if !check_external_call_interpolation(tag, subexpr_count, quoted, expr) {
panic!("Unexpected expression in command arg position: {expr:?}")
}
}
other @ ExternalArgument::Spread(..) => {
panic!("Unexpected external spread argument in command arg position: {other:?}")
}
}
})
}
#[test]
fn test_external_call_argument_spread() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(&mut working_set, None, b"^foo ...[a b c]", true);
assert!(
working_set.parse_errors.is_empty(),
"errors: {:?}",
working_set.parse_errors
);
let input = r"^foo ...[a b c]";
let tag = "spread";
let pipeline = &block.pipelines[0];
assert_eq!(1, pipeline.len());
let element = &pipeline.elements[0];
match &element.expr.expr {
Expr::ExternalCall(name, args) => {
match &name.expr {
Expr::String(string) => {
assert_eq!("foo", string, "incorrect name");
test_external_call(input, tag, |name, args| {
match &name.expr {
Expr::GlobPattern(string, _) => {
assert_eq!("foo", string, "incorrect name");
}
other => {
panic!("Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Spread(expr) => match &expr.expr {
Expr::List(items) => {
assert_eq!(3, items.len());
// that's good enough, don't really need to go so deep into it...
}
other => {
panic!("Unexpected expression in command name position: {other:?}");
}
}
assert_eq!(1, args.len());
match &args[0] {
ExternalArgument::Spread(expr) => match &expr.expr {
Expr::List(items) => {
assert_eq!(3, items.len());
// that's good enough, don't really need to go so deep into it...
}
other => {
panic!("Unexpected expression in command arg position: {other:?}")
}
},
other @ ExternalArgument::Regular(..) => {
panic!(
"Unexpected external regular argument in command arg position: {other:?}"
)
panic!("Unexpected expression in command arg position: {other:?}")
}
},
other @ ExternalArgument::Regular(..) => {
panic!("Unexpected external regular argument in command arg position: {other:?}")
}
}
other => {
panic!("Unexpected expression in pipeline: {other:?}");
}
}
})
}
#[test]
@ -1132,6 +1346,44 @@ mod string {
assert_eq!(subexprs[0], &Expr::String("(1 + 3)(7 - 5)".to_string()));
}
#[test]
pub fn parse_string_interpolation_bare() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
let block = parse(
&mut working_set,
None,
b"\"\" ++ foo(1 + 3)bar(7 - 5)",
true,
);
assert!(working_set.parse_errors.is_empty());
assert_eq!(block.len(), 1);
let pipeline = &block.pipelines[0];
assert_eq!(pipeline.len(), 1);
let element = &pipeline.elements[0];
assert!(element.redirection.is_none());
let subexprs: Vec<&Expr> = match &element.expr.expr {
Expr::BinaryOp(_, _, rhs) => match &rhs.expr {
Expr::StringInterpolation(expressions) => {
expressions.iter().map(|e| &e.expr).collect()
}
_ => panic!("Expected an `Expr::StringInterpolation`"),
},
_ => panic!("Expected an `Expr::BinaryOp`"),
};
assert_eq!(subexprs.len(), 4);
assert_eq!(subexprs[0], &Expr::String("foo".to_string()));
assert!(matches!(subexprs[1], &Expr::FullCellPath(..)));
assert_eq!(subexprs[2], &Expr::String("bar".to_string()));
assert!(matches!(subexprs[3], &Expr::FullCellPath(..)));
}
#[test]
pub fn parse_nested_expressions() {
let engine_state = EngineState::new();