Add support for defining known externals with their own custom completions (#4425)

* WIP for known externals

* Now completions can work from scripts

* Add support for definiing externs

* finish cleaning up old proof-of-concept

crates/nu-cli/src/completions.rs

@@ -3,7 +3,7 @@ use nu_parser::{flatten_expression, parse, trim_quotes};
 use nu_protocol::{
     ast::{Expr, Statement},
     engine::{EngineState, Stack, StateWorkingSet},
-    PipelineData, Span,
+    PipelineData, Span, Value, CONFIG_VARIABLE_ID,
 };
 use reedline::Completer;
 
@@ -214,7 +214,16 @@ impl NuCompleter {
                                         false,
                                     );
 
-                                    let mut stack = Stack::default();
+                                    let mut stack = Stack::new();
+                                    // Set up our initial config to start from
+                                    stack.vars.insert(
+                                        CONFIG_VARIABLE_ID,
+                                        Value::Record {
+                                            cols: vec![],
+                                            vals: vec![],
+                                            span: Span { start: 0, end: 0 },
+                                        },
+                                    );
                                     let result = eval_block(
                                         &self.engine_state,
                                         &mut stack,

crates/nu-color-config/src/shape_color.rs

@@ -30,7 +30,7 @@ pub fn get_shape_color(shape: String, conf: &Config) -> Style {
             "flatshape_globpattern" => Style::new().fg(Color::Cyan).bold(),
             "flatshape_variable" => Style::new().fg(Color::Purple),
             "flatshape_flag" => Style::new().fg(Color::Blue).bold(),
-            "flatshape_custom" => Style::new().bold(),
+            "flatshape_custom" => Style::new().fg(Color::Green),
             "flatshape_nothing" => Style::new().fg(Color::LightCyan),
             _ => Style::default(),
         },

crates/nu-command/src/core_commands/extern_.rs

@@ -0,0 +1,33 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{Category, PipelineData, Signature, SyntaxShape};
+
+#[derive(Clone)]
+pub struct Extern;
+
+impl Command for Extern {
+    fn name(&self) -> &str {
+        "extern"
+    }
+
+    fn usage(&self) -> &str {
+        "Define a signature for an external command"
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("extern")
+            .required("def_name", SyntaxShape::String, "definition name")
+            .required("params", SyntaxShape::Signature, "parameters")
+            .category(Category::Core)
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
+        Ok(PipelineData::new(call.head))
+    }
+}

crates/nu-command/src/core_commands/mod.rs

@@ -10,6 +10,7 @@ mod export;
 mod export_def;
 mod export_def_env;
 mod export_env;
+mod extern_;
 mod for_;
 mod help;
 mod hide;
@@ -36,6 +37,7 @@ pub use export::ExportCommand;
 pub use export_def::ExportDef;
 pub use export_def_env::ExportDefEnv;
 pub use export_env::ExportEnv;
+pub use extern_::Extern;
 pub use for_::For;
 pub use help::Help;
 pub use hide::Hide;

crates/nu-command/src/default_context.rs

@@ -38,6 +38,7 @@ pub fn create_default_context(cwd: impl AsRef<Path>) -> EngineState {
             ExportDef,
             ExportDefEnv,
             ExportEnv,
+            Extern,
             For,
             Help,
             Hide,
@@ -358,9 +359,6 @@ pub fn create_default_context(cwd: impl AsRef<Path>) -> EngineState {
         #[cfg(feature = "plugin")]
         bind_command!(Register);
 
-        // This is a WIP proof of concept
-        // bind_command!(ListGitBranches, Git, GitCheckout, Source);
-
         working_set.render()
     };
 

crates/nu-command/src/experimental/git.rs

@@ -1,57 +0,0 @@
-use nu_protocol::ast::Call;
-use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, IntoPipelineData, PipelineData, Signature, Value};
-
-#[derive(Clone)]
-pub struct Git;
-
-impl Command for Git {
-    fn name(&self) -> &str {
-        "git"
-    }
-
-    fn usage(&self) -> &str {
-        "Run a block"
-    }
-
-    fn signature(&self) -> nu_protocol::Signature {
-        Signature::build("git").category(Category::Experimental)
-    }
-
-    fn run(
-        &self,
-        _engine_state: &EngineState,
-        _stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        use std::process::Command as ProcessCommand;
-        use std::process::Stdio;
-
-        let proc = ProcessCommand::new("git").stdout(Stdio::piped()).spawn();
-
-        match proc {
-            Ok(child) => {
-                match child.wait_with_output() {
-                    Ok(val) => {
-                        let result = val.stdout;
-
-                        Ok(Value::String {
-                            val: String::from_utf8_lossy(&result).to_string(),
-                            span: call.head,
-                        }
-                        .into_pipeline_data())
-                    }
-                    Err(_err) => {
-                        // FIXME: Move this to an external signature and add better error handling
-                        Ok(PipelineData::new(call.head))
-                    }
-                }
-            }
-            Err(_err) => {
-                // FIXME: Move this to an external signature and add better error handling
-                Ok(PipelineData::new(call.head))
-            }
-        }
-    }
-}

crates/nu-command/src/experimental/git_checkout.rs

@@ -1,74 +0,0 @@
-use nu_engine::eval_expression;
-use nu_protocol::ast::Call;
-use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{Category, IntoPipelineData, PipelineData, Signature, SyntaxShape, Value};
-
-#[derive(Clone)]
-pub struct GitCheckout;
-
-impl Command for GitCheckout {
-    fn name(&self) -> &str {
-        "git checkout"
-    }
-
-    fn usage(&self) -> &str {
-        "Checkout a git revision"
-    }
-
-    fn signature(&self) -> nu_protocol::Signature {
-        Signature::build("git checkout")
-            .required(
-                "branch",
-                SyntaxShape::Custom(Box::new(SyntaxShape::String), "list-git-branches".into()),
-                "the branch to checkout",
-            )
-            .category(Category::Experimental)
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        use std::process::Command as ProcessCommand;
-        use std::process::Stdio;
-
-        let block = &call.positional[0];
-
-        let out = eval_expression(engine_state, stack, block)?;
-
-        let out = out.as_string()?;
-
-        let proc = ProcessCommand::new("git")
-            .arg("checkout")
-            .arg(out)
-            .stdout(Stdio::piped())
-            .spawn();
-
-        match proc {
-            Ok(child) => {
-                match child.wait_with_output() {
-                    Ok(val) => {
-                        let result = val.stdout;
-
-                        Ok(Value::String {
-                            val: String::from_utf8_lossy(&result).to_string(),
-                            span: call.head,
-                        }
-                        .into_pipeline_data())
-                    }
-                    Err(_err) => {
-                        // FIXME: Move this to an external signature and add better error handling
-                        Ok(PipelineData::new(call.head))
-                    }
-                }
-            }
-            Err(_err) => {
-                // FIXME: Move this to an external signature and add better error handling
-                Ok(PipelineData::new(call.head))
-            }
-        }
-    }
-}

crates/nu-command/src/experimental/list_git_branches.rs

@@ -1,76 +0,0 @@
-// Note: this is a temporary command that later will be converted into a pipeline
-
-use std::process::Command as ProcessCommand;
-use std::process::Stdio;
-
-use nu_protocol::ast::Call;
-use nu_protocol::engine::Command;
-use nu_protocol::engine::EngineState;
-use nu_protocol::engine::Stack;
-use nu_protocol::Category;
-use nu_protocol::IntoInterruptiblePipelineData;
-use nu_protocol::PipelineData;
-use nu_protocol::{Signature, Value};
-
-#[derive(Clone)]
-pub struct ListGitBranches;
-
-//NOTE: this is not a real implementation :D. It's just a simple one to test with until we port the real one.
-impl Command for ListGitBranches {
-    fn name(&self) -> &str {
-        "list-git-branches"
-    }
-
-    fn usage(&self) -> &str {
-        "List the git branches of the current directory."
-    }
-
-    fn signature(&self) -> nu_protocol::Signature {
-        Signature::build("list-git-branches").category(Category::Experimental)
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        _stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
-        let list_branches = ProcessCommand::new("git")
-            .arg("branch")
-            .stdout(Stdio::piped())
-            .spawn();
-
-        if let Ok(child) = list_branches {
-            if let Ok(output) = child.wait_with_output() {
-                let val = output.stdout;
-
-                let s = String::from_utf8_lossy(&val).to_string();
-
-                #[allow(clippy::needless_collect)]
-                let lines: Vec<_> = s
-                    .lines()
-                    .filter_map(|x| {
-                        if x.starts_with("* ") {
-                            None
-                        } else {
-                            Some(x.trim())
-                        }
-                    })
-                    .map(|x| Value::String {
-                        val: x.into(),
-                        span: call.head,
-                    })
-                    .collect();
-
-                Ok(lines
-                    .into_iter()
-                    .into_pipeline_data(engine_state.ctrlc.clone()))
-            } else {
-                Ok(PipelineData::new(call.head))
-            }
-        } else {
-            Ok(PipelineData::new(call.head))
-        }
-    }
-}

crates/nu-command/src/experimental/mod.rs

@@ -1,9 +1,3 @@
-mod git;
-mod git_checkout;
-mod list_git_branches;
 mod view_source;
 
-pub use git::Git;
-pub use git_checkout::GitCheckout;
-pub use list_git_branches::ListGitBranches;
 pub use view_source::ViewSource;

crates/nu-engine/src/eval.rs

@@ -32,7 +32,7 @@ fn eval_call(
 ) -> Result<PipelineData, ShellError> {
     let decl = engine_state.get_decl(call.decl_id);
 
-    if call.named.iter().any(|(flag, _)| flag.item == "help") {
+    if !decl.is_known_external() && call.named.iter().any(|(flag, _)| flag.item == "help") {
         let mut signature = decl.signature();
         signature.usage = decl.usage().to_string();
         signature.extra_usage = decl.extra_usage().to_string();

crates/nu-parser/src/known_external.rs

@@ -0,0 +1,82 @@
+use nu_protocol::ast::Expr;
+use nu_protocol::engine::{EngineState, Stack, StateWorkingSet};
+use nu_protocol::PipelineData;
+use nu_protocol::{ast::Call, engine::Command, ShellError, Signature};
+
+#[derive(Clone)]
+pub struct KnownExternal {
+    pub name: String,
+    pub signature: Box<Signature>,
+    pub usage: String,
+}
+
+impl Command for KnownExternal {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn signature(&self) -> Signature {
+        *self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.usage
+    }
+
+    fn is_known_external(&self) -> bool {
+        true
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        // FIXME: This is a bit of a hack, and it'd be nice for the parser/AST to be able to handle the original
+        // order of the parameters. Until then, we need to recover the original order.
+        let call_span = call.span();
+        let contents = engine_state.get_span_contents(&call_span);
+
+        let (lexed, _) = crate::lex(contents, call_span.start, &[], &[], true);
+
+        let spans: Vec<_> = lexed.into_iter().map(|x| x.span).collect();
+        let mut working_set = StateWorkingSet::new(engine_state);
+        let (external_call, _) = crate::parse_external_call(&mut working_set, &spans);
+
+        match external_call.expr {
+            Expr::ExternalCall(head, args) => {
+                let decl_id = engine_state
+                    .find_decl("run_external".as_bytes())
+                    .ok_or(ShellError::ExternalNotSupported(head.span))?;
+
+                let command = engine_state.get_decl(decl_id);
+
+                let mut call = Call::new(head.span);
+
+                call.positional.push(*head);
+
+                for arg in args {
+                    call.positional.push(arg.clone())
+                }
+
+                // if last_expression {
+                //     call.named.push((
+                //         Spanned {
+                //             item: "last_expression".into(),
+                //             span: head.span,
+                //         },
+                //         None,
+                //     ))
+                // }
+
+                command.run(engine_state, stack, &call, input)
+            }
+            x => {
+                println!("{:?}", x);
+                panic!("internal error: known external not actually external")
+            }
+        }
+    }
+}

crates/nu-parser/src/lib.rs

@@ -1,5 +1,6 @@
 mod errors;
 mod flatten;
+mod known_external;
 mod lex;
 mod lite_parse;
 mod parse_keywords;
@@ -10,10 +11,11 @@ pub use errors::ParseError;
 pub use flatten::{
     flatten_block, flatten_expression, flatten_pipeline, flatten_statement, FlatShape,
 };
+pub use known_external::KnownExternal;
 pub use lex::{lex, Token, TokenContents};
 pub use lite_parse::{lite_parse, LiteBlock};
 
-pub use parser::{parse, parse_block, trim_quotes, Import};
+pub use parser::{parse, parse_block, parse_external_call, trim_quotes, Import};
 
 #[cfg(feature = "plugin")]
 pub use parse_keywords::parse_register;

crates/nu-parser/src/parse_keywords.rs

@@ -10,6 +10,7 @@ use nu_protocol::{
 use std::collections::HashSet;
 
 use crate::{
+    known_external::KnownExternal,
     lex, lite_parse,
     lite_parse::LiteCommand,
     parser::{
@@ -53,6 +54,34 @@ pub fn parse_def_predecl(working_set: &mut StateWorkingSet, spans: &[Span]) -> O
                 return Some(ParseError::DuplicateCommandDef(spans[1]));
             }
         }
+    } else if name == b"extern" && spans.len() == 3 {
+        let (name_expr, ..) = parse_string(working_set, spans[1]);
+        let name = name_expr.as_string();
+
+        working_set.enter_scope();
+        // FIXME: because parse_signature will update the scope with the variables it sees
+        // we end up parsing the signature twice per def. The first time is during the predecl
+        // so that we can see the types that are part of the signature, which we need for parsing.
+        // The second time is when we actually parse the body itworking_set.
+        // We can't reuse the first time because the variables that are created during parse_signature
+        // are lost when we exit the scope below.
+        let (sig, ..) = parse_signature(working_set, spans[2]);
+        let signature = sig.as_signature();
+        working_set.exit_scope();
+
+        if let (Some(name), Some(mut signature)) = (name, signature) {
+            signature.name = name.clone();
+            //let decl = signature.predeclare();
+            let decl = KnownExternal {
+                name,
+                usage: "run external command".into(),
+                signature,
+            };
+
+            if working_set.add_predecl(Box::new(decl)).is_some() {
+                return Some(ParseError::DuplicateCommandDef(spans[1]));
+            }
+        }
     }
 
     None
@@ -82,7 +111,7 @@ pub fn parse_for(
             return (
                 garbage(spans[0]),
                 Some(ParseError::UnknownState(
-                    "internal error: def declaration not found".into(),
+                    "internal error: for declaration not found".into(),
                     span(spans),
                 )),
             )
@@ -346,6 +375,107 @@ pub fn parse_def(
     )
 }
 
+pub fn parse_extern(
+    working_set: &mut StateWorkingSet,
+    lite_command: &LiteCommand,
+) -> (Statement, Option<ParseError>) {
+    let spans = &lite_command.parts[..];
+    let mut error = None;
+
+    let usage = build_usage(working_set, &lite_command.comments);
+
+    // Checking that the function is used with the correct name
+    // Maybe this is not necessary but it is a sanity check
+
+    let extern_call = working_set.get_span_contents(spans[0]).to_vec();
+    if extern_call != b"extern" {
+        return (
+            garbage_statement(spans),
+            Some(ParseError::UnknownState(
+                "internal error: Wrong call name for extern function".into(),
+                span(spans),
+            )),
+        );
+    }
+
+    // Parsing the spans and checking that they match the register signature
+    // Using a parsed call makes more sense than checking for how many spans are in the call
+    // Also, by creating a call, it can be checked if it matches the declaration signature
+    let (call, call_span) = match working_set.find_decl(&extern_call) {
+        None => {
+            return (
+                garbage_statement(spans),
+                Some(ParseError::UnknownState(
+                    "internal error: def declaration not found".into(),
+                    span(spans),
+                )),
+            )
+        }
+        Some(decl_id) => {
+            working_set.enter_scope();
+            let (call, err) = parse_internal_call(working_set, spans[0], &spans[1..], decl_id);
+            working_set.exit_scope();
+
+            error = error.or(err);
+
+            let call_span = span(spans);
+            //let decl = working_set.get_decl(decl_id);
+            //let sig = decl.signature();
+
+            (call, call_span)
+        }
+    };
+    let name_expr = call.positional.get(0);
+    let sig = call.positional.get(1);
+
+    if let (Some(name_expr), Some(sig)) = (name_expr, sig) {
+        if let (Some(name), Some(mut signature)) = (&name_expr.as_string(), sig.as_signature()) {
+            if let Some(decl_id) = working_set.find_decl(name.as_bytes()) {
+                let declaration = working_set.get_decl_mut(decl_id);
+
+                signature.name = name.clone();
+                signature.usage = usage.clone();
+
+                let decl = KnownExternal {
+                    name: name.to_string(),
+                    usage,
+                    signature,
+                };
+
+                *declaration = Box::new(decl);
+            } else {
+                error = error.or_else(|| {
+                    Some(ParseError::InternalError(
+                        "Predeclaration failed to add declaration".into(),
+                        spans[1],
+                    ))
+                });
+            };
+        }
+        if let Some(name) = name_expr.as_string() {
+            // It's OK if it returns None: The decl was already merged in previous parse pass.
+            working_set.merge_predecl(name.as_bytes());
+        } else {
+            error = error.or_else(|| {
+                Some(ParseError::UnknownState(
+                    "Could not get string from string expression".into(),
+                    name_expr.span,
+                ))
+            });
+        }
+    }
+
+    (
+        Statement::Pipeline(Pipeline::from_vec(vec![Expression {
+            expr: Expr::Call(call),
+            span: call_span,
+            ty: Type::Unknown,
+            custom_completion: None,
+        }])),
+        error,
+    )
+}
+
 pub fn parse_alias(
     working_set: &mut StateWorkingSet,
     spans: &[Span],
@@ -752,6 +882,11 @@ pub fn parse_module_block(
 
                         (stmt, err)
                     }
+                    b"extern" => {
+                        let (stmt, err) = parse_extern(working_set, &pipeline.commands[0]);
+
+                        (stmt, err)
+                    }
                     // TODO: Currently, it is not possible to define a private env var.
                     // TODO: Exported env vars are usable iside the module only if correctly
                     // exported by the user. For example:

crates/nu-parser/src/parser.rs

@@ -1,7 +1,7 @@
 use crate::{
     lex, lite_parse,
     lite_parse::LiteCommand,
-    parse_keywords::{parse_for, parse_source},
+    parse_keywords::{parse_extern, parse_for, parse_source},
     type_check::{math_result_type, type_compatible},
     LiteBlock, ParseError, Token, TokenContents,
 };
@@ -2003,7 +2003,7 @@ pub fn parse_string_strict(
 
 //TODO: Handle error case for unknown shapes
 pub fn parse_shape_name(
-    _working_set: &StateWorkingSet,
+    working_set: &StateWorkingSet,
     bytes: &[u8],
     span: Span,
 ) -> (SyntaxShape, Option<ParseError>) {
@@ -2026,7 +2026,31 @@ pub fn parse_shape_name(
         b"signature" => SyntaxShape::Signature,
         b"string" => SyntaxShape::String,
         b"variable" => SyntaxShape::Variable,
-        _ => return (SyntaxShape::Any, Some(ParseError::UnknownType(span))),
+        _ => {
+            if bytes.contains(&b'@') {
+                let str = String::from_utf8_lossy(bytes);
+                let split: Vec<_> = str.split('@').collect();
+                let (shape, err) = parse_shape_name(
+                    working_set,
+                    split[0].as_bytes(),
+                    Span {
+                        start: span.start,
+                        end: span.start + split[0].len(),
+                    },
+                );
+                let command_name = trim_quotes(split[1].as_bytes());
+
+                return (
+                    SyntaxShape::Custom(
+                        Box::new(shape),
+                        String::from_utf8_lossy(command_name).to_string(),
+                    ),
+                    err,
+                );
+            } else {
+                return (SyntaxShape::Any, Some(ParseError::UnknownType(span)));
+            }
+        }
     };
 
     (result, None)
@@ -3394,6 +3418,10 @@ pub fn parse_expression(
                 parse_call(working_set, &spans[pos..], expand_aliases, spans[0]).0,
                 Some(ParseError::StatementInPipeline("def".into(), spans[0])),
             ),
+            b"extern" => (
+                parse_call(working_set, &spans[pos..], expand_aliases, spans[0]).0,
+                Some(ParseError::StatementInPipeline("extern".into(), spans[0])),
+            ),
             b"let" => (
                 parse_call(working_set, &spans[pos..], expand_aliases, spans[0]).0,
                 Some(ParseError::StatementInPipeline("let".into(), spans[0])),
@@ -3513,6 +3541,7 @@ pub fn parse_statement(
 
     match name {
         b"def" | b"def-env" => parse_def(working_set, lite_command),
+        b"extern" => parse_extern(working_set, lite_command),
         b"let" => parse_let(working_set, &lite_command.parts),
         b"for" => {
             let (expr, err) = parse_for(working_set, &lite_command.parts);

crates/nu-protocol/src/ast/call.rs

@@ -53,4 +53,28 @@ impl Call {
     pub fn nth(&self, pos: usize) -> Option<Expression> {
         self.positional.get(pos).cloned()
     }
+
+    pub fn span(&self) -> Span {
+        let mut span = self.head;
+
+        for positional in &self.positional {
+            if positional.span.end > span.end {
+                span.end = positional.span.end;
+            }
+        }
+
+        for (named, val) in &self.named {
+            if named.span.end > span.end {
+                span.end = named.span.end;
+            }
+
+            if let Some(val) = &val {
+                if val.span.end > span.end {
+                    span.end = val.span.end;
+                }
+            }
+        }
+
+        span
+    }
 }

crates/nu-protocol/src/engine/command.rs

@@ -41,6 +41,11 @@ pub trait Command: Send + Sync + CommandClone {
         true
     }
 
+    // This is a signature for a known external command
+    fn is_known_external(&self) -> bool {
+        false
+    }
+
     // Is a sub command
     fn is_sub(&self) -> bool {
         self.name().contains(' ')