Variable completions. (#3666)

In Nu we have variables (E.g. $var-name) and these contain `Value` types.
This means we can bind to variables any structured data and column path syntax
(E.g. `$variable.path.to`) allows flexibility for "querying" said structures.

Here we offer completions for these. For example, in a Nushell session the
variable `$nu` contains environment values among other things. If we wanted to
see in the screen some environment variable (say the var `SHELL`) we do:

```
> echo $nu.env.SHELL
```

with completions we can now do: `echo $nu.env.S[\TAB]` and we get suggestions
that start at the column path `$nu.env` with vars starting with the letter `S`
in this case `SHELL` appears in the suggestions.

Cargo.lock

@@ -3509,12 +3509,14 @@ dependencies = [
  "indexmap",
  "is_executable",
  "nu-data",
+ "nu-engine",
  "nu-errors",
  "nu-parser",
  "nu-path",
  "nu-protocol",
  "nu-source",
  "nu-test-support",
+ "parking_lot 0.11.1",
 ]
 
 [[package]]

crates/nu-cli/src/shell.rs

@@ -27,12 +27,31 @@ impl Helper {
     }
 }
 
+use nu_protocol::{SignatureRegistry, VariableRegistry};
 struct CompletionContext<'a>(&'a EvaluationContext);
 
 impl<'a> nu_completion::CompletionContext for CompletionContext<'a> {
-    fn signature_registry(&self) -> &dyn nu_parser::ParserScope {
+    fn signature_registry(&self) -> &dyn SignatureRegistry {
         &self.0.scope
     }
+
+    fn scope(&self) -> &dyn nu_parser::ParserScope {
+        &self.0.scope
+    }
+
+    fn source(&self) -> &EvaluationContext {
+        self.as_ref()
+    }
+
+    fn variable_registry(&self) -> &dyn VariableRegistry {
+        self.0
+    }
+}
+
+impl<'a> AsRef<EvaluationContext> for CompletionContext<'a> {
+    fn as_ref(&self) -> &EvaluationContext {
+        self.0
+    }
 }
 
 pub struct CompletionSuggestion(nu_completion::Suggestion);

crates/nu-completion/Cargo.toml

@@ -10,6 +10,7 @@ version = "0.33.0"
 doctest = false
 
 [dependencies]
+nu-engine = { version="0.33.0", path="../nu-engine" }
 nu-data = { version="0.33.0", path="../nu-data" }
 nu-errors = { version="0.33.0", path="../nu-errors" }
 nu-parser = { version="0.33.0", path="../nu-parser" }
@@ -19,5 +20,8 @@ nu-source = { version="0.33.0", path="../nu-source" }
 nu-test-support = { version="0.33.0", path="../nu-test-support" }
 
 dirs-next = "2.0.0"
-indexmap = { version="1.6.1", features=["serde-1"] }
+indexmap = { version = "1.6.1", features = ["serde-1"] }
 is_executable = { version="1.0.1", optional=true }
+
+[dev-dependencies]
+parking_lot = "0.11.1"

crates/nu-completion/src/command.rs

@@ -16,7 +16,7 @@ where
 {
     fn complete(&self, ctx: &Context, partial: &str, matcher: &dyn Matcher) -> Vec<Suggestion> {
         let registry = ctx.signature_registry();
-        let mut commands: IndexSet<String> = IndexSet::from_iter(registry.get_names());
+        let mut commands: IndexSet<String> = IndexSet::from_iter(registry.names());
 
         // Command suggestions can come from three possible sets:
         //   1. internal command names,

crates/nu-completion/src/completer.rs

@@ -8,6 +8,7 @@ use crate::flag::FlagCompleter;
 use crate::matchers;
 use crate::matchers::Matcher;
 use crate::path::{PathCompleter, PathSuggestion};
+use crate::variable::VariableCompleter;
 use crate::{Completer, CompletionContext, Suggestion};
 
 pub struct NuCompleter {}
@@ -26,7 +27,7 @@ impl NuCompleter {
         let tokens = nu_parser::lex(line, 0).0;
 
         let locations = Some(nu_parser::parse_block(tokens).0)
-            .map(|block| nu_parser::classify_block(&block, context.signature_registry()))
+            .map(|block| nu_parser::classify_block(&block, context.scope()))
             .map(|(block, _)| engine::completion_location(line, &block, pos))
             .unwrap_or_default();
 
@@ -121,7 +122,10 @@ impl NuCompleter {
                             .collect()
                         }
 
-                        LocationType::Variable => Vec::new(),
+                        LocationType::Variable => {
+                            let variable_completer = VariableCompleter;
+                            variable_completer.complete(context, &partial, matcher.to_owned())
+                        }
                     }
                 })
                 .collect();

crates/nu-completion/src/engine.rs

@@ -301,10 +301,6 @@ mod tests {
     }
 
     impl ParserScope for VecRegistry {
-        fn get_names(&self) -> Vec<String> {
-            self.0.iter().cloned().map(|s| s.name).collect()
-        }
-
         fn has_signature(&self, name: &str) -> bool {
             self.0.iter().any(|v| v.name == name)
         }

crates/nu-completion/src/flag.rs

@@ -10,7 +10,7 @@ where
     Context: CompletionContext,
 {
     fn complete(&self, ctx: &Context, partial: &str, matcher: &dyn Matcher) -> Vec<Suggestion> {
-        if let Some(sig) = ctx.signature_registry().get_signature(&self.cmd) {
+        if let Some(sig) = ctx.signature_registry().get(&self.cmd) {
             let mut suggestions = Vec::new();
             for (name, (named_type, _desc)) in sig.named.iter() {
                 suggestions.push(format!("--{}", name));

crates/nu-completion/src/lib.rs

@@ -4,6 +4,10 @@ pub(crate) mod engine;
 pub(crate) mod flag;
 pub(crate) mod matchers;
 pub(crate) mod path;
+pub(crate) mod variable;
+
+use nu_engine::EvaluationContext;
+use nu_protocol::{SignatureRegistry, VariableRegistry};
 
 use matchers::Matcher;
 
@@ -15,8 +19,20 @@ pub struct Suggestion {
     pub replacement: String,
 }
 
+impl Suggestion {
+    fn new(display: impl Into<String>, replacement: impl Into<String>) -> Self {
+        Self {
+            display: display.into(),
+            replacement: replacement.into(),
+        }
+    }
+}
+
 pub trait CompletionContext {
-    fn signature_registry(&self) -> &dyn nu_parser::ParserScope;
+    fn signature_registry(&self) -> &dyn SignatureRegistry;
+    fn scope(&self) -> &dyn nu_parser::ParserScope;
+    fn source(&self) -> &EvaluationContext;
+    fn variable_registry(&self) -> &dyn VariableRegistry;
 }
 
 pub trait Completer<Context: CompletionContext> {

crates/nu-completion/src/variable.rs

@@ -0,0 +1,182 @@
+use nu_engine::value_shell::ValueShell;
+use nu_protocol::ColumnPath;
+use nu_source::SpannedItem;
+
+use super::matchers::Matcher;
+use crate::{Completer, CompletionContext, Suggestion};
+use std::path::{Path, PathBuf};
+
+fn build_path(head: &str, members: &Path, entry: &str) -> String {
+    let mut full_path = head.to_string();
+    full_path.push_str(
+        &members
+            .join(entry)
+            .display()
+            .to_string()
+            .replace(std::path::MAIN_SEPARATOR, "."),
+    );
+    full_path
+}
+
+fn collect_entries(value_fs: &ValueShell, head: &str, path: &Path) -> Vec<String> {
+    value_fs
+        .members_under(&path)
+        .iter()
+        .flat_map(|entry| {
+            entry
+                .row_entries()
+                .map(|(entry_name, _)| build_path(&head, &path, entry_name))
+        })
+        .collect()
+}
+
+pub struct VariableCompleter;
+
+impl<Context> Completer<Context> for VariableCompleter
+where
+    Context: CompletionContext,
+{
+    fn complete(&self, ctx: &Context, partial: &str, matcher: &dyn Matcher) -> Vec<Suggestion> {
+        let registry = ctx.variable_registry();
+        let variables_available = registry.variables();
+        let partial_column_path = ColumnPath::with_head(&partial.to_string().spanned_unknown());
+
+        partial_column_path
+            .map(|(head, members)| {
+                variables_available
+                    .iter()
+                    .filter(|candidate| matcher.matches(&head, candidate))
+                    .into_iter()
+                    .filter_map(|candidate| {
+                        if !partial.ends_with('.') && members.is_empty() {
+                            Some(vec![candidate.to_string()])
+                        } else {
+                            let value = registry.get_variable(&candidate[..].spanned_unknown());
+                            let path = PathBuf::from(members.path());
+
+                            value.map(|candidate| {
+                                let fs = ValueShell::new(candidate);
+
+                                fs.find(&path)
+                                    .map(|fs| collect_entries(fs, &head, &path))
+                                    .or_else(|| {
+                                        path.parent().map(|parent| {
+                                            fs.find(parent)
+                                                .map(|fs| collect_entries(fs, &head, &parent))
+                                                .unwrap_or_default()
+                                        })
+                                    })
+                                    .unwrap_or_default()
+                            })
+                        }
+                    })
+                    .flatten()
+                    .filter_map(|candidate| {
+                        if matcher.matches(&partial, &candidate) {
+                            Some(Suggestion::new(&candidate, &candidate))
+                        } else {
+                            None
+                        }
+                    })
+                    .collect()
+            })
+            .unwrap_or_default()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{Completer, Suggestion as S, VariableCompleter};
+    use crate::matchers::case_insensitive::Matcher as CaseInsensitiveMatcher;
+
+    use indexmap::IndexMap;
+    use nu_engine::{
+        evaluation_context::EngineState, ConfigHolder, EvaluationContext, FakeHost, Host, Scope,
+        ShellManager,
+    };
+    use nu_protocol::{SignatureRegistry, VariableRegistry};
+    use parking_lot::Mutex;
+    use std::ffi::OsString;
+    use std::sync::{atomic::AtomicBool, Arc};
+
+    struct CompletionContext<'a>(&'a EvaluationContext);
+
+    impl<'a> crate::CompletionContext for CompletionContext<'a> {
+        fn signature_registry(&self) -> &dyn SignatureRegistry {
+            &self.0.scope
+        }
+
+        fn source(&self) -> &nu_engine::EvaluationContext {
+            &self.0
+        }
+
+        fn scope(&self) -> &dyn nu_parser::ParserScope {
+            &self.0.scope
+        }
+
+        fn variable_registry(&self) -> &dyn VariableRegistry {
+            self.0
+        }
+    }
+
+    fn create_context_with_host(host: Box<dyn Host>) -> EvaluationContext {
+        let scope = Scope::new();
+        let env_vars = host.vars().iter().cloned().collect::<IndexMap<_, _>>();
+        scope.add_env(env_vars);
+
+        EvaluationContext {
+            scope,
+            engine_state: Arc::new(EngineState {
+                host: Arc::new(parking_lot::Mutex::new(host)),
+                current_errors: Arc::new(Mutex::new(vec![])),
+                ctrl_c: Arc::new(AtomicBool::new(false)),
+                configs: Arc::new(Mutex::new(ConfigHolder::new())),
+                shell_manager: ShellManager::basic(),
+                windows_drives_previous_cwd: Arc::new(Mutex::new(std::collections::HashMap::new())),
+            }),
+        }
+    }
+
+    fn set_envs(host: &mut FakeHost, values: Vec<(&str, &str)>) {
+        values.iter().for_each(|(key, value)| {
+            host.env_set(OsString::from(key), OsString::from(value));
+        });
+    }
+
+    #[test]
+    fn structure() {
+        let mut host = nu_engine::FakeHost::new();
+        set_envs(&mut host, vec![("COMPLETER", "VARIABLE"), ("SHELL", "NU")]);
+        let context = create_context_with_host(Box::new(host));
+
+        assert_eq!(
+            VariableCompleter {}.complete(
+                &CompletionContext(&context),
+                "$nu.env.",
+                &CaseInsensitiveMatcher
+            ),
+            vec![
+                S::new("$nu.env.COMPLETER", "$nu.env.COMPLETER"),
+                S::new("$nu.env.SHELL", "$nu.env.SHELL")
+            ]
+        );
+
+        assert_eq!(
+            VariableCompleter {}.complete(
+                &CompletionContext(&context),
+                "$nu.env.CO",
+                &CaseInsensitiveMatcher
+            ),
+            vec![S::new("$nu.env.COMPLETER", "$nu.env.COMPLETER"),]
+        );
+
+        assert_eq!(
+            VariableCompleter {}.complete(
+                &CompletionContext(&context),
+                "$nu.en",
+                &CaseInsensitiveMatcher
+            ),
+            vec![S::new("$nu.env", "$nu.env"),]
+        );
+    }
+}

crates/nu-engine/src/env/basic_host.rs

@@ -37,7 +37,7 @@ impl Host for BasicHost {
     }
 
     #[allow(unused_variables)]
-    fn vars(&mut self) -> Vec<(String, String)> {
+    fn vars(&self) -> Vec<(String, String)> {
         #[cfg(not(target_arch = "wasm32"))]
         {
             std::env::vars().collect::<Vec<_>>()

crates/nu-engine/src/env/host.rs

@@ -11,7 +11,7 @@ pub trait Host: Debug + Send {
     fn stderr(&mut self, out: &str);
     fn print_err(&mut self, err: ShellError, source: &Text);
 
-    fn vars(&mut self) -> Vec<(String, String)>;
+    fn vars(&self) -> Vec<(String, String)>;
     fn env_get(&mut self, key: OsString) -> Option<OsString>;
     fn env_set(&mut self, k: OsString, v: OsString);
     fn env_rm(&mut self, k: OsString);
@@ -41,7 +41,7 @@ impl Host for Box<dyn Host> {
         (**self).print_err(err, source)
     }
 
-    fn vars(&mut self) -> Vec<(String, String)> {
+    fn vars(&self) -> Vec<(String, String)> {
         (**self).vars()
     }
 
@@ -104,7 +104,7 @@ impl Host for FakeHost {
         BasicHost {}.print_err(err, source);
     }
 
-    fn vars(&mut self) -> Vec<(String, String)> {
+    fn vars(&self) -> Vec<(String, String)> {
         self.env_vars
             .iter()
             .map(|(k, v)| (k.clone(), v.clone()))

crates/nu-engine/src/evaluate/evaluator.rs

@@ -35,7 +35,7 @@ pub fn evaluate_baseline_expr(
         Expression::Synthetic(hir::Synthetic::String(s)) => {
             Ok(UntaggedValue::string(s).into_untagged_value())
         }
-        Expression::Variable(var, s) => evaluate_reference(var, ctx, *s),
+        expr @ Expression::Variable(_, _) => evaluate_reference(&Variable::from(expr), ctx, span),
         Expression::Command => unimplemented!(),
         Expression::Subexpression(block) => evaluate_subexpression(block, ctx),
         Expression::ExternalCommand(_) => unimplemented!(),
@@ -236,45 +236,64 @@ fn evaluate_literal(literal: &hir::Literal, span: Span) -> Value {
     }
 }
 
-fn evaluate_reference(
+pub enum Variable<'a> {
-    name: &str,
+    Nu,
-    ctx: &EvaluationContext,
+    Scope,
-    span: Span,
+    True,
-) -> Result<Value, ShellError> {
+    False,
-    match name {
+    Nothing,
-        "$nu" => crate::evaluate::variables::nu(&ctx.scope, span, ctx),
+    Other(&'a str),
+}
 
-        "$scope" => crate::evaluate::variables::scope(
+impl<'a> Variable<'a> {
+    pub fn list() -> Vec<String> {
+        vec![
+            String::from("$nu"),
+            String::from("$scope"),
+            String::from("$true"),
+            String::from("$false"),
+            String::from("$nothing"),
+        ]
+    }
+}
+
+impl<'a> From<&'a Expression> for Variable<'a> {
+    fn from(expr: &'a Expression) -> Self {
+        match &expr {
+            Expression::Variable(name, _) => match name.as_str() {
+                "$nu" => Self::Nu,
+                "$scope" => Self::Scope,
+                "$true" => Self::True,
+                "$false" => Self::False,
+                "$nothing" => Self::Nothing,
+                _ => Self::Other(&name),
+            },
+            _ => unreachable!(),
+        }
+    }
+}
+
+pub fn evaluate_reference(
+    variable: &Variable,
+    ctx: &EvaluationContext,
+    tag: impl Into<Tag>,
+) -> Result<Value, ShellError> {
+    match variable {
+        Variable::Nu => crate::evaluate::variables::nu(&ctx.scope, ctx),
+        Variable::Scope => crate::evaluate::variables::scope(
             &ctx.scope.get_aliases(),
             &ctx.scope.get_commands(),
             &ctx.scope.get_vars(),
-            span,
         ),
-
+        Variable::True => Ok(UntaggedValue::boolean(true).into_untagged_value()),
-        "$true" => Ok(Value {
+        Variable::False => Ok(UntaggedValue::boolean(false).into_untagged_value()),
-            value: UntaggedValue::boolean(true),
+        Variable::Nothing => Ok(UntaggedValue::nothing().into_untagged_value()),
-            tag: span.into(),
+        Variable::Other(name) => match ctx.scope.get_var(name) {
-        }),
+            Some(v) => Ok(v),
-
-        "$false" => Ok(Value {
-            value: UntaggedValue::boolean(false),
-            tag: span.into(),
-        }),
-
-        "$nothing" => Ok(Value {
-            value: UntaggedValue::nothing(),
-            tag: span.into(),
-        }),
-
-        x => match ctx.scope.get_var(x) {
-            Some(mut v) => {
-                v.tag.span = span;
-                Ok(v)
-            }
             None => Err(ShellError::labeled_error(
                 "Variable not in scope",
-                format!("unknown variable: {}", x),
+                format!("unknown variable: {}", name),
-                span,
+                tag.into(),
             )),
         },
     }

crates/nu-engine/src/evaluate/mod.rs

@@ -1,6 +1,6 @@
 pub(crate) mod block;
 pub(crate) mod evaluate_args;
-pub(crate) mod evaluator;
+pub mod evaluator;
 pub(crate) mod expr;
 pub(crate) mod internal;
 pub(crate) mod operator;

crates/nu-engine/src/evaluate/scope.rs

@@ -2,7 +2,7 @@ use crate::whole_stream_command::{whole_stream_command, Command};
 use indexmap::IndexMap;
 use nu_errors::ShellError;
 use nu_parser::ParserScope;
-use nu_protocol::{hir::Block, Signature, Value};
+use nu_protocol::{hir::Block, Signature, SignatureRegistry, Value};
 use nu_source::Spanned;
 use std::sync::Arc;
 
@@ -23,6 +23,7 @@ impl Scope {
             frames: Arc::new(parking_lot::Mutex::new(vec![ScopeFrame::new()])),
         }
     }
+
     pub fn get_command(&self, name: &str) -> Option<Command> {
         for frame in self.frames.lock().iter().rev() {
             if let Some(command) = frame.get_command(name) {
@@ -64,6 +65,10 @@ impl Scope {
         output.sorted_by(|k1, _v1, k2, _v2| k1.cmp(k2)).collect()
     }
 
+    pub fn get_variable_names(&self) -> Vec<String> {
+        self.get_vars().iter().map(|(k, _)| k.to_string()).collect()
+    }
+
     pub fn get_vars(&self) -> IndexMap<String, Value> {
         //FIXME: should this be an iterator?
         let mut output: IndexMap<String, Value> = IndexMap::new();
@@ -327,12 +332,26 @@ impl Scope {
     }
 }
 
-impl ParserScope for Scope {
+impl SignatureRegistry for Scope {
-    fn get_names(&self) -> Vec<String> {
+    fn names(&self) -> Vec<String> {
         self.get_command_names()
     }
 
-    fn get_signature(&self, name: &str) -> Option<nu_protocol::Signature> {
+    fn has(&self, name: &str) -> bool {
+        self.get_signature(name).is_some()
+    }
+
+    fn get(&self, name: &str) -> Option<Signature> {
+        self.get_signature(name)
+    }
+
+    fn clone_box(&self) -> Box<dyn SignatureRegistry> {
+        Box::new(self.clone())
+    }
+}
+
+impl ParserScope for Scope {
+    fn get_signature(&self, name: &str) -> Option<Signature> {
         self.get_command(name).map(|x| x.signature())
     }
 

crates/nu-engine/src/evaluate/variables.rs

@@ -5,13 +5,9 @@ use nu_errors::ShellError;
 use nu_protocol::{Dictionary, ShellTypeName, Signature, TaggedDictBuilder, UntaggedValue, Value};
 use nu_source::{Spanned, Tag};
 
-pub fn nu(
+pub fn nu(scope: &Scope, ctx: &EvaluationContext) -> Result<Value, ShellError> {
-    scope: &Scope,
-    tag: impl Into<Tag>,
-    ctx: &EvaluationContext,
-) -> Result<Value, ShellError> {
     let env = &scope.get_env_vars();
-    let tag = tag.into();
+    let tag = Tag::unknown();
 
     let mut nu_dict = TaggedDictBuilder::new(&tag);
 
@@ -99,9 +95,8 @@ pub fn scope(
     aliases: &IndexMap<String, Vec<Spanned<String>>>,
     commands: &IndexMap<String, Signature>,
     variables: &IndexMap<String, Value>,
-    tag: impl Into<Tag>,
 ) -> Result<Value, ShellError> {
-    let tag = tag.into();
+    let tag = Tag::unknown();
 
     let mut scope_dict = TaggedDictBuilder::new(&tag);
 

crates/nu-engine/src/evaluation_context.rs

@@ -1,3 +1,4 @@
+use crate::evaluate::evaluator::Variable;
 use crate::evaluate::scope::{Scope, ScopeFrame};
 use crate::shell::palette::ThemedPalette;
 use crate::shell::shell_manager::ShellManager;
@@ -5,14 +6,17 @@ use crate::whole_stream_command::Command;
 use crate::{call_info::UnevaluatedCallInfo, config_holder::ConfigHolder};
 use crate::{command_args::CommandArgs, script};
 use crate::{env::basic_host::BasicHost, Host};
-use indexmap::IndexMap;
+
-use log::trace;
 use nu_data::config::{self, Conf, NuConfig};
 use nu_errors::ShellError;
-use nu_protocol::{hir, ConfigPath};
+use nu_protocol::{hir, ConfigPath, VariableRegistry};
+use nu_source::Spanned;
 use nu_source::{Span, Tag};
 use nu_stream::InputStream;
 use nu_test_support::NATIVE_PATH_ENV_VAR;
+
+use indexmap::IndexMap;
+use log::trace;
 use parking_lot::Mutex;
 use std::fs::File;
 use std::io::BufReader;
@@ -33,7 +37,7 @@ pub struct EngineState {
 #[derive(Clone, Default)]
 pub struct EvaluationContext {
     pub scope: Scope,
-    engine_state: Arc<EngineState>,
+    pub engine_state: Arc<EngineState>,
 }
 
 impl EvaluationContext {
@@ -61,7 +65,7 @@ impl EvaluationContext {
 
     pub fn basic() -> EvaluationContext {
         let scope = Scope::new();
-        let mut host = BasicHost {};
+        let host = BasicHost {};
         let env_vars = host.vars().iter().cloned().collect::<IndexMap<_, _>>();
         scope.add_env(env_vars);
 
@@ -395,3 +399,24 @@ impl EvaluationContext {
         }
     }
 }
+
+use itertools::Itertools;
+
+impl VariableRegistry for EvaluationContext {
+    fn get_variable(&self, name: &Spanned<&str>) -> Option<nu_protocol::Value> {
+        let span = name.span;
+        let name = nu_protocol::hir::Expression::variable(name.item.to_string(), name.span);
+
+        let var = Variable::from(&name);
+
+        crate::evaluate::evaluator::evaluate_reference(&var, self, span).ok()
+    }
+
+    fn variables(&self) -> Vec<String> {
+        Variable::list()
+            .into_iter()
+            .chain(self.scope.get_variable_names().into_iter())
+            .unique()
+            .collect()
+    }
+}

crates/nu-engine/src/lib.rs

@@ -4,7 +4,7 @@ mod config_holder;
 pub mod documentation;
 mod env;
 mod evaluate;
-mod evaluation_context;
+pub mod evaluation_context;
 mod example;
 pub mod filesystem;
 mod from_value;
@@ -23,8 +23,8 @@ pub use crate::documentation::{generate_docs, get_brief_help, get_documentation,
 pub use crate::env::host::FakeHost;
 pub use crate::env::host::Host;
 pub use crate::evaluate::block::run_block;
-pub use crate::evaluate::evaluator::evaluate_baseline_expr;
 pub use crate::evaluate::scope::Scope;
+pub use crate::evaluate::{evaluator, evaluator::evaluate_baseline_expr};
 pub use crate::evaluation_context::EvaluationContext;
 pub use crate::example::Example;
 pub use crate::filesystem::dir_info::{DirBuilder, DirInfo, FileInfo};
@@ -35,5 +35,5 @@ pub use crate::print::maybe_print_errors;
 pub use crate::shell::painter::Painter;
 pub use crate::shell::palette::{DefaultPalette, Palette};
 pub use crate::shell::shell_manager::ShellManager;
-pub use crate::shell::value_shell::ValueShell;
+pub use crate::shell::value_shell;
 pub use crate::whole_stream_command::{whole_stream_command, Command, WholeStreamCommand};

crates/nu-engine/src/shell/mod.rs

@@ -14,7 +14,7 @@ pub(crate) mod painter;
 pub(crate) mod palette;
 pub(crate) mod shell_args;
 pub(crate) mod shell_manager;
-pub(crate) mod value_shell;
+pub mod value_shell;
 
 pub trait Shell: std::fmt::Debug {
     fn is_interactive(&self) -> bool;

crates/nu-engine/src/shell/value_shell.rs

@@ -38,7 +38,17 @@ impl ValueShell {
         }
     }
 
-    fn members_under(&self, path: &Path) -> VecDeque<Value> {
+    pub fn find(&self, path: &Path) -> Option<&Self> {
+        let mut value_system = ValueStructure::new();
+
+        if value_system.walk_decorate(&self.value).is_ok() {
+            value_system.exists(&path).then(|| self)
+        } else {
+            None
+        }
+    }
+
+    pub fn members_under(&self, path: &Path) -> VecDeque<Value> {
         let mut shell_entries = VecDeque::new();
         let full_path = path.to_path_buf();
         let mut viewed = self.value.clone();
@@ -72,12 +82,6 @@ impl ValueShell {
 
         shell_entries
     }
-
-    // TODO make use of this in the new completion engine
-    #[allow(dead_code)]
-    fn members(&self) -> VecDeque<Value> {
-        self.members_under(Path::new("."))
-    }
 }
 
 impl Shell for ValueShell {
@@ -106,10 +110,7 @@ impl Shell for ValueShell {
             full_path.push(value.as_ref());
         }
 
-        let mut value_system = ValueStructure::new();
+        if self.find(&full_path).is_none() {
-        value_system.walk_decorate(&self.value)?;
-
-        if !value_system.exists(&full_path) {
             if let Some(target) = &path {
                 return Err(ShellError::labeled_error(
                     "Can not list entries inside",

crates/nu-parser/src/lib.rs

@@ -9,11 +9,9 @@ mod lex;
 mod parse;
 mod scope;
 mod shapes;
-mod signature;
 
 pub use lex::lexer::{lex, parse_block};
 pub use lex::tokens::{LiteBlock, LiteCommand, LiteGroup, LitePipeline};
 pub use parse::{classify_block, garbage, parse, parse_full_column_path, parse_math_expression};
 pub use scope::ParserScope;
 pub use shapes::shapes;
-pub use signature::{Signature, SignatureRegistry};

crates/nu-parser/src/scope.rs

@@ -3,8 +3,6 @@ use nu_source::Spanned;
 use std::{fmt::Debug, sync::Arc};
 
 pub trait ParserScope: Debug {
-    fn get_names(&self) -> Vec<String>;
-
     fn get_signature(&self, name: &str) -> Option<nu_protocol::Signature>;
 
     fn has_signature(&self, name: &str) -> bool;

crates/nu-parser/src/signature.rs

@@ -1,48 +0,0 @@
-use std::fmt::Debug;
-
-use nu_source::{DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span};
-
-pub trait SignatureRegistry: Debug {
-    fn has(&self, name: &str) -> bool;
-    fn get(&self, name: &str) -> Option<nu_protocol::Signature>;
-    fn clone_box(&self) -> Box<dyn SignatureRegistry>;
-}
-
-impl SignatureRegistry for Box<dyn SignatureRegistry> {
-    fn has(&self, name: &str) -> bool {
-        (&**self).has(name)
-    }
-    fn get(&self, name: &str) -> Option<nu_protocol::Signature> {
-        (&**self).get(name)
-    }
-    fn clone_box(&self) -> Box<dyn SignatureRegistry> {
-        (&**self).clone_box()
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct Signature {
-    pub(crate) unspanned: nu_protocol::Signature,
-    span: Span,
-}
-
-impl Signature {
-    pub fn new(unspanned: nu_protocol::Signature, span: impl Into<Span>) -> Signature {
-        Signature {
-            unspanned,
-            span: span.into(),
-        }
-    }
-}
-
-impl HasSpan for Signature {
-    fn span(&self) -> Span {
-        self.span
-    }
-}
-
-impl PrettyDebugWithSource for Signature {
-    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
-        self.unspanned.pretty_debug(source)
-    }
-}

crates/nu-protocol/src/lib.rs

@@ -5,6 +5,7 @@ mod call_info;
 pub mod config_path;
 pub mod hir;
 mod maybe_owned;
+mod registry;
 mod return_value;
 mod signature;
 mod syntax_shape;
@@ -18,6 +19,7 @@ pub mod dataframe;
 pub use crate::call_info::{CallInfo, EvaluatedArgs};
 pub use crate::config_path::ConfigPath;
 pub use crate::maybe_owned::MaybeOwned;
+pub use crate::registry::{SignatureRegistry, VariableRegistry};
 pub use crate::return_value::{CommandAction, ReturnSuccess, ReturnValue};
 pub use crate::signature::{NamedType, PositionalType, Signature};
 pub use crate::syntax_shape::SyntaxShape;

crates/nu-protocol/src/registry.rs

@@ -0,0 +1,31 @@
+use crate::{Signature, Value};
+use nu_source::Spanned;
+use std::fmt::Debug;
+
+pub trait VariableRegistry {
+    fn get_variable(&self, name: &Spanned<&str>) -> Option<Value>;
+    fn variables(&self) -> Vec<String>;
+}
+
+pub trait SignatureRegistry: Debug {
+    fn names(&self) -> Vec<String>;
+    fn has(&self, name: &str) -> bool;
+    fn get(&self, name: &str) -> Option<Signature>;
+    fn clone_box(&self) -> Box<dyn SignatureRegistry>;
+}
+
+impl SignatureRegistry for Box<dyn SignatureRegistry> {
+    fn names(&self) -> Vec<String> {
+        (&**self).names()
+    }
+
+    fn has(&self, name: &str) -> bool {
+        (&**self).has(name)
+    }
+    fn get(&self, name: &str) -> Option<Signature> {
+        (&**self).get(name)
+    }
+    fn clone_box(&self) -> Box<dyn SignatureRegistry> {
+        (&**self).clone_box()
+    }
+}

crates/nu-protocol/src/value/column_path.rs

@@ -61,6 +61,10 @@ impl ColumnPath {
         self.members.iter()
     }
 
+    pub fn is_empty(&self) -> bool {
+        self.members.is_empty()
+    }
+
     /// Returns the last member and a slice of the remaining members
     pub fn split_last(&self) -> Option<(&PathMember, &[PathMember])> {
         self.members.split_last()
@@ -71,6 +75,23 @@ impl ColumnPath {
         self.iter().last()
     }
 
+    pub fn path(&self) -> String {
+        let sep = std::path::MAIN_SEPARATOR;
+        let mut members = self.iter();
+        let mut f = String::from(sep);
+
+        if let Some(member) = members.next() {
+            f.push_str(&member.as_string());
+        }
+
+        for member in members {
+            f.push(sep);
+            f.push_str(&member.as_string());
+        }
+
+        f
+    }
+
     pub fn build(text: &Spanned<String>) -> ColumnPath {
         if let (
             SpannedExpression {
@@ -87,6 +108,38 @@ impl ColumnPath {
             ColumnPath { members: vec![] }
         }
     }
+
+    pub fn with_head(text: &Spanned<String>) -> Option<(String, ColumnPath)> {
+        match parse_full_column_path(text) {
+            (
+                SpannedExpression {
+                    expr: Expression::FullColumnPath(path),
+                    ..
+                },
+                _,
+            ) => {
+                if let crate::hir::FullColumnPath {
+                    head:
+                        SpannedExpression {
+                            expr: Expression::Variable(name, _),
+                            span: _,
+                        },
+                    tail,
+                } = *path
+                {
+                    Some((
+                        name,
+                        ColumnPath {
+                            members: tail.to_vec(),
+                        },
+                    ))
+                } else {
+                    None
+                }
+            }
+            _ => None,
+        }
+    }
 }
 
 impl PrettyDebug for ColumnPath {
@@ -136,6 +189,102 @@ impl PathMember {
     }
 }
 
+fn parse_full_column_path(
+    raw_column_path: &Spanned<String>,
+) -> (SpannedExpression, Option<ParseError>) {
+    let mut inside_delimiter = vec![];
+    let mut output = vec![];
+    let mut current_part = String::new();
+    let mut start_index = 0;
+    let mut last_index = 0;
+    let error = None;
+
+    let mut head = None;
+
+    for (idx, c) in raw_column_path.item.char_indices() {
+        last_index = idx;
+        if c == '(' {
+            inside_delimiter.push(')');
+        } else if let Some(delimiter) = inside_delimiter.last() {
+            if c == *delimiter {
+                inside_delimiter.pop();
+            }
+        } else if c == '\'' || c == '"' {
+            inside_delimiter.push(c);
+        } else if c == '.' {
+            let part_span = Span::new(
+                raw_column_path.span.start() + start_index,
+                raw_column_path.span.start() + idx,
+            );
+
+            if head.is_none() && current_part.starts_with('$') {
+                // We have the variable head
+                head = Some(Expression::variable(current_part.clone(), part_span))
+            } else if let Ok(row_number) = current_part.parse::<i64>() {
+                output.push(UnspannedPathMember::Int(row_number).into_path_member(part_span));
+            } else {
+                let current_part = trim_quotes(&current_part);
+                output.push(
+                    UnspannedPathMember::String(current_part.clone()).into_path_member(part_span),
+                );
+            }
+            current_part.clear();
+            // Note: I believe this is safe because of the delimiter we're using,
+            // but if we get fancy with Unicode we'll need to change this.
+            start_index = idx + '.'.len_utf8();
+            continue;
+        }
+        current_part.push(c);
+    }
+
+    if !current_part.is_empty() {
+        let part_span = Span::new(
+            raw_column_path.span.start() + start_index,
+            raw_column_path.span.start() + last_index + 1,
+        );
+
+        if head.is_none() {
+            if current_part.starts_with('$') {
+                head = Some(Expression::variable(current_part, raw_column_path.span));
+            } else if let Ok(row_number) = current_part.parse::<i64>() {
+                output.push(UnspannedPathMember::Int(row_number).into_path_member(part_span));
+            } else {
+                let current_part = trim_quotes(&current_part);
+                output.push(UnspannedPathMember::String(current_part).into_path_member(part_span));
+            }
+        } else if let Ok(row_number) = current_part.parse::<i64>() {
+            output.push(UnspannedPathMember::Int(row_number).into_path_member(part_span));
+        } else {
+            let current_part = trim_quotes(&current_part);
+            output.push(UnspannedPathMember::String(current_part).into_path_member(part_span));
+        }
+    }
+
+    if let Some(head) = head {
+        (
+            SpannedExpression::new(
+                Expression::path(SpannedExpression::new(head, raw_column_path.span), output),
+                raw_column_path.span,
+            ),
+            error,
+        )
+    } else {
+        (
+            SpannedExpression::new(
+                Expression::path(
+                    SpannedExpression::new(
+                        Expression::variable("$it".into(), raw_column_path.span),
+                        raw_column_path.span,
+                    ),
+                    output,
+                ),
+                raw_column_path.span,
+            ),
+            error,
+        )
+    }
+}
+
 fn parse(raw_column_path: &Spanned<String>) -> (SpannedExpression, Option<ParseError>) {
     let mut delimiter = '.';
     let mut inside_delimiter = false;

crates/nu-protocol/src/value/value_structure.rs

@@ -17,7 +17,7 @@ pub struct ValueResource {
 
 impl ValueResource {}
 
-#[derive(Default)]
+#[derive(Default, Debug)]
 pub struct ValueStructure {
     pub resources: Vec<ValueResource>,
 }