refactor(completion): flatten_shape -> expression for internal/external/operator (#15086)

# Description

Fixes #14852

As the completion rules are somehow intertwined between internals and
externals,
this PR is relatively messy, and has larger probability to break things,
@fdncred @ysthakur @sholderbach
But I strongly believe this is a better direction to go. Edge cases
should be easier to fix in the dedicated branches.

There're no flattened expression based completion rules left.

# User-Facing Changes

# Tests + Formatting
+7
# After Submitting

---------

Co-authored-by: Yash Thakur <45539777+ysthakur@users.noreply.github.com>

crates/nu-cli/tests/completions/mod.rs

@@ -14,7 +14,9 @@ use nu_protocol::{debugger::WithoutDebug, engine::StateWorkingSet, PipelineData}
 use reedline::{Completer, Suggestion};
 use rstest::{fixture, rstest};
 use support::{
-    completions_helpers::{new_dotnu_engine, new_partial_engine, new_quote_engine},
+    completions_helpers::{
+        new_dotnu_engine, new_external_engine, new_partial_engine, new_quote_engine,
+    },
     file, folder, match_suggestions, new_engine,
 };
 
@@ -292,6 +294,105 @@ fn customcompletions_fallback() {
     match_suggestions(&expected, &suggestions);
 }
 
+/// Custom function arguments mixed with subcommands
+#[test]
+fn custom_arguments_and_subcommands() {
+    let (_, _, mut engine, mut stack) = new_engine();
+    let command = r#"
+        def foo [i: directory] {}
+        def "foo test bar" [] {}"#;
+    assert!(support::merge_input(command.as_bytes(), &mut engine, &mut stack).is_ok());
+
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
+    let completion_str = "foo test";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    // including both subcommand and directory completions
+    let expected: Vec<String> = vec!["foo test bar".into(), folder("test_a"), folder("test_b")];
+    match_suggestions(&expected, &suggestions);
+}
+
+/// Custom function flags mixed with subcommands
+#[test]
+fn custom_flags_and_subcommands() {
+    let (_, _, mut engine, mut stack) = new_engine();
+    let command = r#"
+        def foo [--test: directory] {}
+        def "foo --test bar" [] {}"#;
+    assert!(support::merge_input(command.as_bytes(), &mut engine, &mut stack).is_ok());
+
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
+    let completion_str = "foo --test";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    // including both flag and directory completions
+    let expected: Vec<String> = vec!["foo --test bar".into(), "--test".into()];
+    match_suggestions(&expected, &suggestions);
+}
+
+/// If argument type is something like int/string, complete only subcommands
+#[test]
+fn custom_arguments_vs_subcommands() {
+    let (_, _, mut engine, mut stack) = new_engine();
+    let command = r#"
+        def foo [i: string] {}
+        def "foo test bar" [] {}"#;
+    assert!(support::merge_input(command.as_bytes(), &mut engine, &mut stack).is_ok());
+
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
+    let completion_str = "foo test";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    // including only subcommand completions
+    let expected: Vec<String> = vec!["foo test bar".into()];
+    match_suggestions(&expected, &suggestions);
+}
+
+/// External command only if starts with `^`
+#[test]
+fn external_commands_only() {
+    let engine = new_external_engine();
+    let mut completer = NuCompleter::new(
+        Arc::new(engine),
+        Arc::new(nu_protocol::engine::Stack::new()),
+    );
+    let completion_str = "^sleep";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    #[cfg(windows)]
+    let expected: Vec<String> = vec!["sleep.exe".into()];
+    #[cfg(not(windows))]
+    let expected: Vec<String> = vec!["sleep".into()];
+    match_suggestions(&expected, &suggestions);
+
+    let completion_str = "sleep";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    #[cfg(windows)]
+    let expected: Vec<String> = vec!["sleep".into(), "sleep.exe".into()];
+    #[cfg(not(windows))]
+    let expected: Vec<String> = vec!["sleep".into(), "^sleep".into()];
+    match_suggestions(&expected, &suggestions);
+}
+
+/// Which completes both internals and externals
+#[test]
+fn which_command_completions() {
+    let engine = new_external_engine();
+    let mut completer = NuCompleter::new(
+        Arc::new(engine),
+        Arc::new(nu_protocol::engine::Stack::new()),
+    );
+    // flags
+    let completion_str = "which --all";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    let expected: Vec<String> = vec!["--all".into()];
+    match_suggestions(&expected, &suggestions);
+    // commands
+    let completion_str = "which sleep";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    #[cfg(windows)]
+    let expected: Vec<String> = vec!["sleep".into(), "sleep.exe".into()];
+    #[cfg(not(windows))]
+    let expected: Vec<String> = vec!["sleep".into(), "^sleep".into()];
+    match_suggestions(&expected, &suggestions);
+}
+
 /// Suppress completions for invalid values
 #[test]
 fn customcompletions_invalid() {
@@ -307,6 +408,25 @@ fn customcompletions_invalid() {
     assert!(suggestions.is_empty());
 }
 
+#[test]
+fn dont_use_dotnu_completions() {
+    // Create a new engine
+    let (_, _, engine, stack) = new_dotnu_engine();
+    // Instantiate a new completer
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
+    // Test nested nu script
+    let completion_str = "go work use `./dir_module/".to_string();
+    let suggestions = completer.complete(&completion_str, completion_str.len());
+
+    // including a plaintext file
+    let expected: Vec<String> = vec![
+        "./dir_module/mod.nu".into(),
+        "./dir_module/plain.txt".into(),
+        "`./dir_module/sub module/`".into(),
+    ];
+    match_suggestions(&expected, &suggestions);
+}
+
 #[test]
 fn dotnu_completions() {
     // Create a new engine
@@ -315,6 +435,15 @@ fn dotnu_completions() {
     // Instantiate a new completer
     let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
 
+    // Flags should still be working
+    let completion_str = "overlay use --".to_string();
+    let suggestions = completer.complete(&completion_str, completion_str.len());
+
+    match_suggestions(
+        &vec!["--help".into(), "--prefix".into(), "--reload".into()],
+        &suggestions,
+    );
+
     // Test nested nu script
     #[cfg(windows)]
     let completion_str = "use `.\\dir_module\\".to_string();
@@ -486,6 +615,17 @@ fn external_completer_fallback() {
     match_suggestions(&expected, &suggestions);
 }
 
+/// Fallback to external completions for flags of `sudo`
+#[test]
+fn external_completer_sudo() {
+    let block = "{|spans| ['--background']}";
+    let input = "sudo --back".to_string();
+
+    let expected = vec!["--background".into()];
+    let suggestions = run_external_completion(block, &input);
+    match_suggestions(&expected, &suggestions);
+}
+
 /// Suppress completions when external completer returns invalid value
 #[test]
 fn external_completer_invalid() {

crates/nu-cli/tests/completions/support/completions_helpers.rs

@@ -14,7 +14,7 @@ fn create_default_context() -> EngineState {
     nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
 }
 
-// creates a new engine with the current path into the completions fixtures folder
+/// creates a new engine with the current path into the completions fixtures folder
 pub fn new_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
     // Target folder inside assets
     let dir = fs::fixtures().join("completions");
@@ -69,7 +69,26 @@ pub fn new_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
     (dir, dir_str, engine_state, stack)
 }
 
-// creates a new engine with the current path into the completions fixtures folder
+/// Adds pseudo PATH env for external completion tests
+pub fn new_external_engine() -> EngineState {
+    let mut engine = create_default_context();
+    let dir = fs::fixtures().join("external_completions").join("path");
+    let dir_str = dir.to_string_lossy().to_string();
+    let internal_span = nu_protocol::Span::new(0, dir_str.len());
+    engine.add_env_var(
+        "PATH".to_string(),
+        Value::List {
+            vals: vec![Value::String {
+                val: dir_str,
+                internal_span,
+            }],
+            internal_span,
+        },
+    );
+    engine
+}
+
+/// creates a new engine with the current path into the completions fixtures folder
 pub fn new_dotnu_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
     // Target folder inside assets
     let dir = fs::fixtures().join("dotnu_completions");
@@ -197,7 +216,7 @@ pub fn new_partial_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
     (dir, dir_str, engine_state, stack)
 }
 
-// match a list of suggestions with the expected values
+/// match a list of suggestions with the expected values
 pub fn match_suggestions(expected: &Vec<String>, suggestions: &Vec<Suggestion>) {
     let expected_len = expected.len();
     let suggestions_len = suggestions.len();
@@ -209,28 +228,28 @@ pub fn match_suggestions(expected: &Vec<String>, suggestions: &Vec<Suggestion>)
         )
     }
 
-    let suggestoins_str = suggestions
+    let suggestions_str = suggestions
         .iter()
         .map(|it| it.value.clone())
         .collect::<Vec<_>>();
 
-    assert_eq!(expected, &suggestoins_str);
+    assert_eq!(expected, &suggestions_str);
 }
 
-// append the separator to the converted path
+/// append the separator to the converted path
 pub fn folder(path: impl Into<PathBuf>) -> String {
     let mut converted_path = file(path);
     converted_path.push(MAIN_SEPARATOR);
     converted_path
 }
 
-// convert a given path to string
+/// convert a given path to string
 pub fn file(path: impl Into<PathBuf>) -> String {
     path.into().into_os_string().into_string().unwrap()
 }
 
-// merge_input executes the given input into the engine
-// and merges the state
+/// merge_input executes the given input into the engine
+/// and merges the state
 pub fn merge_input(
     input: &[u8],
     engine_state: &mut EngineState,