refactor(completion): expression based variable/cell_path completion (#15033)

# Description

fixes #14643 , as well as some nested cell path cases:

```nushell
let foo = {a: [1 {a: 1}]}

$foo.a.1.#<tab>

const bar = {a: 1, b: 2}
$bar.#<tab>
```

So my plan of the refactoring process is that:
1. gradually move those rules of flattened shapes into expression match
branches, until they are gone
2. keep each PR focused, easier to review and track. 

# User-Facing Changes

# Tests + Formatting

+2

# After Submitting

crates/nu-cli/src/completions/cell_path_completions.rs

@@ -0,0 +1,97 @@
+use crate::completions::{Completer, CompletionOptions, SemanticSuggestion, SuggestionKind};
+use nu_engine::{column::get_columns, eval_variable};
+use nu_protocol::{
+    ast::{Expr, FullCellPath, PathMember},
+    engine::{Stack, StateWorkingSet},
+    eval_const::eval_constant,
+    Span, Value,
+};
+use reedline::Suggestion;
+
+use super::completion_options::NuMatcher;
+
+pub struct CellPathCompletion<'a> {
+    pub full_cell_path: &'a FullCellPath,
+}
+
+impl Completer for CellPathCompletion<'_> {
+    fn fetch(
+        &mut self,
+        working_set: &StateWorkingSet,
+        stack: &Stack,
+        _prefix: &[u8],
+        _span: Span,
+        offset: usize,
+        _pos: usize,
+        options: &CompletionOptions,
+    ) -> Vec<SemanticSuggestion> {
+        // empty tail is already handled as variable names completion
+        let Some((prefix_member, path_members)) = self.full_cell_path.tail.split_last() else {
+            return vec![];
+        };
+        let (mut prefix_str, span) = match prefix_member {
+            PathMember::String { val, span, .. } => (val.clone(), span),
+            PathMember::Int { val, span, .. } => (val.to_string(), span),
+        };
+        // strip the placeholder
+        prefix_str.pop();
+        let true_end = std::cmp::max(span.start, span.end - 1);
+        let span = Span::new(span.start, true_end);
+        let current_span = reedline::Span {
+            start: span.start - offset,
+            end: true_end - offset,
+        };
+
+        let mut matcher = NuMatcher::new(prefix_str, options.clone());
+
+        // evaluate the head expression to get its value
+        let value = if let Expr::Var(var_id) = self.full_cell_path.head.expr {
+            working_set
+                .get_variable(var_id)
+                .const_val
+                .to_owned()
+                .or_else(|| eval_variable(working_set.permanent_state, stack, var_id, span).ok())
+        } else {
+            eval_constant(working_set, &self.full_cell_path.head).ok()
+        }
+        .unwrap_or_default();
+
+        for suggestion in nested_suggestions(&value, path_members, current_span) {
+            matcher.add_semantic_suggestion(suggestion);
+        }
+        matcher.results()
+    }
+}
+
+// Find recursively the values for cell_path
+fn nested_suggestions(
+    val: &Value,
+    path_members: &[PathMember],
+    current_span: reedline::Span,
+) -> Vec<SemanticSuggestion> {
+    let value = val
+        .clone()
+        .follow_cell_path(path_members, false)
+        .unwrap_or_default();
+
+    let kind = SuggestionKind::Type(value.get_type());
+    let str_to_suggestion = |s: String| SemanticSuggestion {
+        suggestion: Suggestion {
+            value: s,
+            span: current_span,
+            ..Suggestion::default()
+        },
+        kind: Some(kind.to_owned()),
+    };
+    match value {
+        Value::Record { val, .. } => val
+            .columns()
+            .map(|s| str_to_suggestion(s.to_string()))
+            .collect(),
+        Value::List { vals, .. } => get_columns(vals.as_slice())
+            .into_iter()
+            .map(str_to_suggestion)
+            .collect(),
+        _ => vec![],
+    }
+}

crates/nu-cli/src/completions/completer.rs

@@ -1,6 +1,7 @@
 use crate::completions::{
-    CommandCompletion, Completer, CompletionOptions, CustomCompletion, DirectoryCompletion,
+    CellPathCompletion, CommandCompletion, Completer, CompletionOptions, CustomCompletion,
-    DotNuCompletion, FileCompletion, FlagCompletion, OperatorCompletion, VariableCompletion,
+    DirectoryCompletion, DotNuCompletion, FileCompletion, FlagCompletion, OperatorCompletion,
+    VariableCompletion,
 };
 use log::debug;
 use nu_color_config::{color_record_to_nustyle, lookup_ansi_color_style};
@@ -17,6 +18,10 @@ use std::{str, sync::Arc};
 
 use super::base::{SemanticSuggestion, SuggestionKind};
 
+/// Used as the function `f` in find_map Traverse
+///
+/// returns the inner-most pipeline_element of interest
+/// i.e. the one that contains given position and needs completion
 fn find_pipeline_element_by_position<'a>(
     expr: &'a Expression,
     working_set: &'a StateWorkingSet,
@@ -62,6 +67,15 @@ fn find_pipeline_element_by_position<'a>(
     }
 }
 
+/// Before completion, an additional character `a` is added to the source as a placeholder for correct parsing results.
+/// This function helps to strip it
+fn strip_placeholder<'a>(working_set: &'a StateWorkingSet, span: &Span) -> (Span, &'a [u8]) {
+    let new_end = std::cmp::max(span.end - 1, span.start);
+    let new_span = Span::new(span.start, new_end);
+    let prefix = working_set.get_span_contents(new_span);
+    (new_span, prefix)
+}
+
 #[derive(Clone)]
 pub struct NuCompleter {
     engine_state: Arc<EngineState>,
@@ -80,6 +94,28 @@ impl NuCompleter {
         self.completion_helper(line, pos)
     }
 
+    fn variable_names_completion_helper(
+        &self,
+        working_set: &StateWorkingSet,
+        span: Span,
+        offset: usize,
+    ) -> Vec<SemanticSuggestion> {
+        let (new_span, prefix) = strip_placeholder(working_set, &span);
+        if !prefix.starts_with(b"$") {
+            return vec![];
+        }
+        let mut variable_names_completer = VariableCompletion {};
+        self.process_completion(
+            &mut variable_names_completer,
+            working_set,
+            prefix,
+            new_span,
+            offset,
+            // pos is not required
+            0,
+        )
+    }
+
     // Process the completion for a given completer
     fn process_completion<T: Completer>(
         &self,
@@ -193,6 +229,37 @@ impl NuCompleter {
             return vec![];
         };
 
+        match &element_expression.expr {
+            Expr::Var(_) => {
+                return self.variable_names_completion_helper(
+                    &working_set,
+                    element_expression.span,
+                    fake_offset,
+                );
+            }
+            Expr::FullCellPath(full_cell_path) => {
+                // e.g. `$e<tab>` parsed as FullCellPath
+                if full_cell_path.tail.is_empty() {
+                    return self.variable_names_completion_helper(
+                        &working_set,
+                        element_expression.span,
+                        fake_offset,
+                    );
+                } else {
+                    let mut cell_path_completer = CellPathCompletion { full_cell_path };
+                    return self.process_completion(
+                        &mut cell_path_completer,
+                        &working_set,
+                        &[],
+                        element_expression.span,
+                        fake_offset,
+                        pos,
+                    );
+                }
+            }
+            _ => (),
+        }
+
         let flattened = flatten_expression(&working_set, element_expression);
         let mut spans: Vec<String> = vec![];
 
@@ -223,9 +290,6 @@ impl NuCompleter {
 
             // Complete based on the last span
             if is_last_span {
-                // Context variables
-                let most_left_var = most_left_variable(flat_idx, &working_set, flattened.clone());
-
                 // Create a new span
                 let new_span = Span::new(span.start, span.end - 1);
 
@@ -233,36 +297,6 @@ impl NuCompleter {
                 let index = pos - span.start;
                 let prefix = &current_span[..index];
 
-                // Variables completion
-                if prefix.starts_with(b"$") || most_left_var.is_some() {
-                    let mut variable_names_completer =
-                        VariableCompletion::new(most_left_var.unwrap_or((vec![], vec![])));
-
-                    let mut variable_completions = self.process_completion(
-                        &mut variable_names_completer,
-                        &working_set,
-                        prefix,
-                        new_span,
-                        fake_offset,
-                        pos,
-                    );
-
-                    let mut variable_operations_completer =
-                        OperatorCompletion::new(element_expression.clone());
-
-                    let mut variable_operations_completions = self.process_completion(
-                        &mut variable_operations_completer,
-                        &working_set,
-                        prefix,
-                        new_span,
-                        fake_offset,
-                        pos,
-                    );
-
-                    variable_completions.append(&mut variable_operations_completions);
-                    return variable_completions;
-                }
-
                 // Flags completion
                 if prefix.starts_with(b"-") {
                     // Try to complete flag internally
@@ -474,56 +508,6 @@ impl ReedlineCompleter for NuCompleter {
     }
 }
 
-// reads the most left variable returning it's name (e.g: $myvar)
-// and the depth (a.b.c)
-fn most_left_variable(
-    idx: usize,
-    working_set: &StateWorkingSet<'_>,
-    flattened: Vec<(Span, FlatShape)>,
-) -> Option<(Vec<u8>, Vec<Vec<u8>>)> {
-    // Reverse items to read the list backwards and truncate
-    // because the only items that matters are the ones before the current index
-    let mut rev = flattened;
-    rev.truncate(idx);
-    rev = rev.into_iter().rev().collect();
-
-    // Store the variables and sub levels found and reverse to correct order
-    let mut variables_found: Vec<Vec<u8>> = vec![];
-    let mut found_var = false;
-    for item in rev.clone() {
-        let result = working_set.get_span_contents(item.0).to_vec();
-
-        match item.1 {
-            FlatShape::Variable(_) => {
-                variables_found.push(result);
-                found_var = true;
-
-                break;
-            }
-            FlatShape::String => {
-                variables_found.push(result);
-            }
-            _ => {
-                break;
-            }
-        }
-    }
-
-    // If most left var was not found
-    if !found_var {
-        return None;
-    }
-
-    // Reverse the order back
-    variables_found = variables_found.into_iter().rev().collect();
-
-    // Extract the variable and the sublevels
-    let var = variables_found.first().unwrap_or(&vec![]).to_vec();
-    let sublevels: Vec<Vec<u8>> = variables_found.into_iter().skip(1).collect();
-
-    Some((var, sublevels))
-}
-
 pub fn map_value_completions<'a>(
     list: impl Iterator<Item = &'a Value>,
     span: Span,

crates/nu-cli/src/completions/mod.rs

@@ -1,4 +1,5 @@
 mod base;
+mod cell_path_completions;
 mod command_completions;
 mod completer;
 mod completion_common;
@@ -12,6 +13,7 @@ mod operator_completions;
 mod variable_completions;
 
 pub use base::{Completer, SemanticSuggestion, SuggestionKind};
+pub use cell_path_completions::CellPathCompletion;
 pub use command_completions::CommandCompletion;
 pub use completer::NuCompleter;
 pub use completion_options::{CompletionOptions, MatchAlgorithm};

crates/nu-cli/src/completions/variable_completions.rs

@@ -1,124 +1,34 @@
 use crate::completions::{Completer, CompletionOptions, SemanticSuggestion, SuggestionKind};
-use nu_engine::{column::get_columns, eval_variable};
 use nu_protocol::{
     engine::{Stack, StateWorkingSet},
-    Span, Value,
+    Span, VarId,
 };
 use reedline::Suggestion;
-use std::str;
 
 use super::completion_options::NuMatcher;
 
-#[derive(Clone)]
+pub struct VariableCompletion {}
-pub struct VariableCompletion {
-    var_context: (Vec<u8>, Vec<Vec<u8>>), // tuple with $var and the sublevels (.b.c.d)
-}
-
-impl VariableCompletion {
-    pub fn new(var_context: (Vec<u8>, Vec<Vec<u8>>)) -> Self {
-        Self { var_context }
-    }
-}
 
 impl Completer for VariableCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
-        stack: &Stack,
+        _stack: &Stack,
         prefix: &[u8],
         span: Span,
         offset: usize,
         _pos: usize,
         options: &CompletionOptions,
     ) -> Vec<SemanticSuggestion> {
-        let builtins = ["$nu", "$in", "$env"];
+        let prefix_str = String::from_utf8_lossy(prefix);
-        let var_str = std::str::from_utf8(&self.var_context.0).unwrap_or("");
+        let mut matcher = NuMatcher::new(prefix_str, options.clone());
-        let var_id = working_set.find_variable(&self.var_context.0);
         let current_span = reedline::Span {
             start: span.start - offset,
             end: span.end - offset,
         };
-        let sublevels_count = self.var_context.1.len();
-        let prefix_str = String::from_utf8_lossy(prefix);
-        let mut matcher = NuMatcher::new(prefix_str, options.clone());
-
-        // Completions for the given variable
-        if !var_str.is_empty() {
-            // Completion for $env.<tab>
-            if var_str == "$env" {
-                let env_vars = stack.get_env_vars(working_set.permanent_state);
-
-                // Return nested values
-                if sublevels_count > 0 {
-                    // Extract the target var ($env.<target-var>)
-                    let target_var = self.var_context.1[0].clone();
-                    let target_var_str =
-                        str::from_utf8(&target_var).unwrap_or_default().to_string();
-
-                    // Everything after the target var is the nested level ($env.<target-var>.<nested_levels>...)
-                    let nested_levels: Vec<Vec<u8>> =
-                        self.var_context.1.clone().into_iter().skip(1).collect();
-
-                    if let Some(val) = env_vars.get(&target_var_str) {
-                        for suggestion in nested_suggestions(val, &nested_levels, current_span) {
-                            matcher.add_semantic_suggestion(suggestion);
-                        }
-
-                        return matcher.results();
-                    }
-                } else {
-                    // No nesting provided, return all env vars
-                    for env_var in env_vars {
-                        matcher.add_semantic_suggestion(SemanticSuggestion {
-                            suggestion: Suggestion {
-                                value: env_var.0,
-                                span: current_span,
-                                ..Suggestion::default()
-                            },
-                            kind: Some(SuggestionKind::Type(env_var.1.get_type())),
-                        });
-                    }
-
-                    return matcher.results();
-                }
-            }
-
-            // Completions for $nu.<tab>
-            if var_str == "$nu" {
-                // Eval nu var
-                if let Ok(nuval) = eval_variable(
-                    working_set.permanent_state,
-                    stack,
-                    nu_protocol::NU_VARIABLE_ID,
-                    nu_protocol::Span::new(current_span.start, current_span.end),
-                ) {
-                    for suggestion in nested_suggestions(&nuval, &self.var_context.1, current_span)
-                    {
-                        matcher.add_semantic_suggestion(suggestion);
-                    }
-
-                    return matcher.results();
-                }
-            }
-
-            // Completion other variable types
-            if let Some(var_id) = var_id {
-                // Extract the variable value from the stack
-                let var = stack.get_var(var_id, Span::new(span.start, span.end));
-
-                // If the value exists and it's of type Record
-                if let Ok(value) = var {
-                    for suggestion in nested_suggestions(&value, &self.var_context.1, current_span)
-                    {
-                        matcher.add_semantic_suggestion(suggestion);
-                    }
-
-                    return matcher.results();
-                }
-            }
-        }
 
         // Variable completion (e.g: $en<tab> to complete $env)
+        let builtins = ["$nu", "$in", "$env"];
         for builtin in builtins {
             matcher.add_semantic_suggestion(SemanticSuggestion {
                 suggestion: Suggestion {
@@ -131,27 +41,30 @@ impl Completer for VariableCompletion {
             });
         }
 
+        let mut add_candidate = |name, var_id: &VarId| {
+            matcher.add_semantic_suggestion(SemanticSuggestion {
+                suggestion: Suggestion {
+                    value: String::from_utf8_lossy(name).to_string(),
+                    span: current_span,
+                    ..Suggestion::default()
+                },
+                kind: Some(SuggestionKind::Type(
+                    working_set.get_variable(*var_id).ty.clone(),
+                )),
+            })
+        };
+
         // TODO: The following can be refactored (see find_commands_by_predicate() used in
         // command_completions).
         let mut removed_overlays = vec![];
         // Working set scope vars
         for scope_frame in working_set.delta.scope.iter().rev() {
             for overlay_frame in scope_frame.active_overlays(&mut removed_overlays).rev() {
-                for v in &overlay_frame.vars {
+                for (name, var_id) in &overlay_frame.vars {
-                    matcher.add_semantic_suggestion(SemanticSuggestion {
+                    add_candidate(name, var_id);
-                        suggestion: Suggestion {
-                            value: String::from_utf8_lossy(v.0).to_string(),
-                            span: current_span,
-                            ..Suggestion::default()
-                        },
-                        kind: Some(SuggestionKind::Type(
-                            working_set.get_variable(*v.1).ty.clone(),
-                        )),
-                    });
                 }
             }
         }
-
         // Permanent state vars
         // for scope in &self.engine_state.scope {
         for overlay_frame in working_set
@@ -159,98 +72,11 @@ impl Completer for VariableCompletion {
             .active_overlays(&removed_overlays)
             .rev()
         {
-            for v in &overlay_frame.vars {
+            for (name, var_id) in &overlay_frame.vars {
-                matcher.add_semantic_suggestion(SemanticSuggestion {
+                add_candidate(name, var_id);
-                    suggestion: Suggestion {
-                        value: String::from_utf8_lossy(v.0).to_string(),
-                        span: current_span,
-                        ..Suggestion::default()
-                    },
-                    kind: Some(SuggestionKind::Type(
-                        working_set.get_variable(*v.1).ty.clone(),
-                    )),
-                });
             }
         }
 
         matcher.results()
     }
 }
-
-// Find recursively the values for sublevels
-// if no sublevels are set it returns the current value
-fn nested_suggestions(
-    val: &Value,
-    sublevels: &[Vec<u8>],
-    current_span: reedline::Span,
-) -> Vec<SemanticSuggestion> {
-    let mut output: Vec<SemanticSuggestion> = vec![];
-    let value = recursive_value(val, sublevels).unwrap_or_else(Value::nothing);
-
-    let kind = SuggestionKind::Type(value.get_type());
-    match value {
-        Value::Record { val, .. } => {
-            // Add all the columns as completion
-            for col in val.columns() {
-                output.push(SemanticSuggestion {
-                    suggestion: Suggestion {
-                        value: col.clone(),
-                        span: current_span,
-                        ..Suggestion::default()
-                    },
-                    kind: Some(kind.clone()),
-                });
-            }
-
-            output
-        }
-        Value::List { vals, .. } => {
-            for column_name in get_columns(vals.as_slice()) {
-                output.push(SemanticSuggestion {
-                    suggestion: Suggestion {
-                        value: column_name,
-                        span: current_span,
-                        ..Suggestion::default()
-                    },
-                    kind: Some(kind.clone()),
-                });
-            }
-
-            output
-        }
-        _ => output,
-    }
-}
-
-// Extracts the recursive value (e.g: $var.a.b.c)
-fn recursive_value(val: &Value, sublevels: &[Vec<u8>]) -> Result<Value, Span> {
-    // Go to next sublevel
-    if let Some((sublevel, next_sublevels)) = sublevels.split_first() {
-        let span = val.span();
-        match val {
-            Value::Record { val, .. } => {
-                if let Some((_, value)) = val.iter().find(|(key, _)| key.as_bytes() == sublevel) {
-                    // If matches try to fetch recursively the next
-                    recursive_value(value, next_sublevels)
-                } else {
-                    // Current sublevel value not found
-                    Err(span)
-                }
-            }
-            Value::List { vals, .. } => {
-                for col in get_columns(vals.as_slice()) {
-                    if col.as_bytes() == *sublevel {
-                        let val = val.get_data_by_key(&col).ok_or(span)?;
-                        return recursive_value(&val, next_sublevels);
-                    }
-                }
-
-                // Current sublevel value not found
-                Err(span)
-            }
-            _ => Ok(val.clone()),
-        }
-    } else {
-        Ok(val.clone())
-    }
-}

crates/nu-cli/tests/completions/mod.rs

@@ -1554,6 +1554,58 @@ fn variables_completions() {
     match_suggestions(&expected, &suggestions);
 }
 
+#[test]
+fn record_cell_path_completions() {
+    let (_, _, mut engine, mut stack) = new_engine();
+    let command = r#"let foo = {a: [1 {a: 2}]}; const bar = {a: [1 {a: 2}]}"#;
+    assert!(support::merge_input(command.as_bytes(), &mut engine, &mut stack).is_ok());
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
+
+    let expected: Vec<String> = vec!["a".into()];
+    let completion_str = "$foo.";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    match_suggestions(&expected, &suggestions);
+
+    let completion_str = "$foo.a.1.";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    match_suggestions(&expected, &suggestions);
+
+    let completion_str = "$bar.";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    match_suggestions(&expected, &suggestions);
+
+    let completion_str = "$bar.a.1.";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    match_suggestions(&expected, &suggestions);
+
+    let completion_str = "{a: [1 {a: 2}]}.a.1.";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    match_suggestions(&expected, &suggestions);
+}
+
+#[test]
+fn table_cell_path_completions() {
+    let (_, _, mut engine, mut stack) = new_engine();
+    let command = r#"let foo = [{a:{b:1}}, {a:{b:2}}]; const bar = [[a b]; [1 2]]"#;
+    assert!(support::merge_input(command.as_bytes(), &mut engine, &mut stack).is_ok());
+    let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack));
+
+    let expected: Vec<String> = vec!["a".into()];
+    let completion_str = "$foo.";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    match_suggestions(&expected, &suggestions);
+
+    let expected: Vec<String> = vec!["b".into()];
+    let completion_str = "$foo.a.";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    match_suggestions(&expected, &suggestions);
+
+    let expected: Vec<String> = vec!["a".into(), "b".into()];
+    let completion_str = "$bar.";
+    let suggestions = completer.complete(completion_str, completion_str.len());
+    match_suggestions(&expected, &suggestions);
+}
+
 #[test]
 fn alias_of_command_and_flags() {
     let (_, _, mut engine, mut stack) = new_engine();