Add support for defining known externals with their own custom completions (#4425)

* WIP for known externals

* Now completions can work from scripts

* Add support for definiing externs

* finish cleaning up old proof-of-concept

crates/nu-parser/src/known_external.rs

@@ -0,0 +1,82 @@
+use nu_protocol::ast::Expr;
+use nu_protocol::engine::{EngineState, Stack, StateWorkingSet};
+use nu_protocol::PipelineData;
+use nu_protocol::{ast::Call, engine::Command, ShellError, Signature};
+
+#[derive(Clone)]
+pub struct KnownExternal {
+    pub name: String,
+    pub signature: Box<Signature>,
+    pub usage: String,
+}
+
+impl Command for KnownExternal {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn signature(&self) -> Signature {
+        *self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.usage
+    }
+
+    fn is_known_external(&self) -> bool {
+        true
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        // FIXME: This is a bit of a hack, and it'd be nice for the parser/AST to be able to handle the original
+        // order of the parameters. Until then, we need to recover the original order.
+        let call_span = call.span();
+        let contents = engine_state.get_span_contents(&call_span);
+
+        let (lexed, _) = crate::lex(contents, call_span.start, &[], &[], true);
+
+        let spans: Vec<_> = lexed.into_iter().map(|x| x.span).collect();
+        let mut working_set = StateWorkingSet::new(engine_state);
+        let (external_call, _) = crate::parse_external_call(&mut working_set, &spans);
+
+        match external_call.expr {
+            Expr::ExternalCall(head, args) => {
+                let decl_id = engine_state
+                    .find_decl("run_external".as_bytes())
+                    .ok_or(ShellError::ExternalNotSupported(head.span))?;
+
+                let command = engine_state.get_decl(decl_id);
+
+                let mut call = Call::new(head.span);
+
+                call.positional.push(*head);
+
+                for arg in args {
+                    call.positional.push(arg.clone())
+                }
+
+                // if last_expression {
+                //     call.named.push((
+                //         Spanned {
+                //             item: "last_expression".into(),
+                //             span: head.span,
+                //         },
+                //         None,
+                //     ))
+                // }
+
+                command.run(engine_state, stack, &call, input)
+            }
+            x => {
+                println!("{:?}", x);
+                panic!("internal error: known external not actually external")
+            }
+        }
+    }
+}

crates/nu-parser/src/lib.rs

@@ -1,5 +1,6 @@
 mod errors;
 mod flatten;
+mod known_external;
 mod lex;
 mod lite_parse;
 mod parse_keywords;
@@ -10,10 +11,11 @@ pub use errors::ParseError;
 pub use flatten::{
     flatten_block, flatten_expression, flatten_pipeline, flatten_statement, FlatShape,
 };
+pub use known_external::KnownExternal;
 pub use lex::{lex, Token, TokenContents};
 pub use lite_parse::{lite_parse, LiteBlock};
 
-pub use parser::{parse, parse_block, trim_quotes, Import};
+pub use parser::{parse, parse_block, parse_external_call, trim_quotes, Import};
 
 #[cfg(feature = "plugin")]
 pub use parse_keywords::parse_register;

crates/nu-parser/src/parse_keywords.rs

@@ -10,6 +10,7 @@ use nu_protocol::{
 use std::collections::HashSet;
 
 use crate::{
+    known_external::KnownExternal,
     lex, lite_parse,
     lite_parse::LiteCommand,
     parser::{
@@ -53,6 +54,34 @@ pub fn parse_def_predecl(working_set: &mut StateWorkingSet, spans: &[Span]) -> O
                 return Some(ParseError::DuplicateCommandDef(spans[1]));
             }
         }
+    } else if name == b"extern" && spans.len() == 3 {
+        let (name_expr, ..) = parse_string(working_set, spans[1]);
+        let name = name_expr.as_string();
+
+        working_set.enter_scope();
+        // FIXME: because parse_signature will update the scope with the variables it sees
+        // we end up parsing the signature twice per def. The first time is during the predecl
+        // so that we can see the types that are part of the signature, which we need for parsing.
+        // The second time is when we actually parse the body itworking_set.
+        // We can't reuse the first time because the variables that are created during parse_signature
+        // are lost when we exit the scope below.
+        let (sig, ..) = parse_signature(working_set, spans[2]);
+        let signature = sig.as_signature();
+        working_set.exit_scope();
+
+        if let (Some(name), Some(mut signature)) = (name, signature) {
+            signature.name = name.clone();
+            //let decl = signature.predeclare();
+            let decl = KnownExternal {
+                name,
+                usage: "run external command".into(),
+                signature,
+            };
+
+            if working_set.add_predecl(Box::new(decl)).is_some() {
+                return Some(ParseError::DuplicateCommandDef(spans[1]));
+            }
+        }
     }
 
     None
@@ -82,7 +111,7 @@ pub fn parse_for(
             return (
                 garbage(spans[0]),
                 Some(ParseError::UnknownState(
-                    "internal error: def declaration not found".into(),
+                    "internal error: for declaration not found".into(),
                     span(spans),
                 )),
             )
@@ -346,6 +375,107 @@ pub fn parse_def(
     )
 }
 
+pub fn parse_extern(
+    working_set: &mut StateWorkingSet,
+    lite_command: &LiteCommand,
+) -> (Statement, Option<ParseError>) {
+    let spans = &lite_command.parts[..];
+    let mut error = None;
+
+    let usage = build_usage(working_set, &lite_command.comments);
+
+    // Checking that the function is used with the correct name
+    // Maybe this is not necessary but it is a sanity check
+
+    let extern_call = working_set.get_span_contents(spans[0]).to_vec();
+    if extern_call != b"extern" {
+        return (
+            garbage_statement(spans),
+            Some(ParseError::UnknownState(
+                "internal error: Wrong call name for extern function".into(),
+                span(spans),
+            )),
+        );
+    }
+
+    // Parsing the spans and checking that they match the register signature
+    // Using a parsed call makes more sense than checking for how many spans are in the call
+    // Also, by creating a call, it can be checked if it matches the declaration signature
+    let (call, call_span) = match working_set.find_decl(&extern_call) {
+        None => {
+            return (
+                garbage_statement(spans),
+                Some(ParseError::UnknownState(
+                    "internal error: def declaration not found".into(),
+                    span(spans),
+                )),
+            )
+        }
+        Some(decl_id) => {
+            working_set.enter_scope();
+            let (call, err) = parse_internal_call(working_set, spans[0], &spans[1..], decl_id);
+            working_set.exit_scope();
+
+            error = error.or(err);
+
+            let call_span = span(spans);
+            //let decl = working_set.get_decl(decl_id);
+            //let sig = decl.signature();
+
+            (call, call_span)
+        }
+    };
+    let name_expr = call.positional.get(0);
+    let sig = call.positional.get(1);
+
+    if let (Some(name_expr), Some(sig)) = (name_expr, sig) {
+        if let (Some(name), Some(mut signature)) = (&name_expr.as_string(), sig.as_signature()) {
+            if let Some(decl_id) = working_set.find_decl(name.as_bytes()) {
+                let declaration = working_set.get_decl_mut(decl_id);
+
+                signature.name = name.clone();
+                signature.usage = usage.clone();
+
+                let decl = KnownExternal {
+                    name: name.to_string(),
+                    usage,
+                    signature,
+                };
+
+                *declaration = Box::new(decl);
+            } else {
+                error = error.or_else(|| {
+                    Some(ParseError::InternalError(
+                        "Predeclaration failed to add declaration".into(),
+                        spans[1],
+                    ))
+                });
+            };
+        }
+        if let Some(name) = name_expr.as_string() {
+            // It's OK if it returns None: The decl was already merged in previous parse pass.
+            working_set.merge_predecl(name.as_bytes());
+        } else {
+            error = error.or_else(|| {
+                Some(ParseError::UnknownState(
+                    "Could not get string from string expression".into(),
+                    name_expr.span,
+                ))
+            });
+        }
+    }
+
+    (
+        Statement::Pipeline(Pipeline::from_vec(vec![Expression {
+            expr: Expr::Call(call),
+            span: call_span,
+            ty: Type::Unknown,
+            custom_completion: None,
+        }])),
+        error,
+    )
+}
+
 pub fn parse_alias(
     working_set: &mut StateWorkingSet,
     spans: &[Span],
@@ -752,6 +882,11 @@ pub fn parse_module_block(
 
                         (stmt, err)
                     }
+                    b"extern" => {
+                        let (stmt, err) = parse_extern(working_set, &pipeline.commands[0]);
+
+                        (stmt, err)
+                    }
                     // TODO: Currently, it is not possible to define a private env var.
                     // TODO: Exported env vars are usable iside the module only if correctly
                     // exported by the user. For example:

crates/nu-parser/src/parser.rs

@@ -1,7 +1,7 @@
 use crate::{
     lex, lite_parse,
     lite_parse::LiteCommand,
-    parse_keywords::{parse_for, parse_source},
+    parse_keywords::{parse_extern, parse_for, parse_source},
     type_check::{math_result_type, type_compatible},
     LiteBlock, ParseError, Token, TokenContents,
 };
@@ -2003,7 +2003,7 @@ pub fn parse_string_strict(
 
 //TODO: Handle error case for unknown shapes
 pub fn parse_shape_name(
-    _working_set: &StateWorkingSet,
+    working_set: &StateWorkingSet,
     bytes: &[u8],
     span: Span,
 ) -> (SyntaxShape, Option<ParseError>) {
@@ -2026,7 +2026,31 @@ pub fn parse_shape_name(
         b"signature" => SyntaxShape::Signature,
         b"string" => SyntaxShape::String,
         b"variable" => SyntaxShape::Variable,
-        _ => return (SyntaxShape::Any, Some(ParseError::UnknownType(span))),
+        _ => {
+            if bytes.contains(&b'@') {
+                let str = String::from_utf8_lossy(bytes);
+                let split: Vec<_> = str.split('@').collect();
+                let (shape, err) = parse_shape_name(
+                    working_set,
+                    split[0].as_bytes(),
+                    Span {
+                        start: span.start,
+                        end: span.start + split[0].len(),
+                    },
+                );
+                let command_name = trim_quotes(split[1].as_bytes());
+
+                return (
+                    SyntaxShape::Custom(
+                        Box::new(shape),
+                        String::from_utf8_lossy(command_name).to_string(),
+                    ),
+                    err,
+                );
+            } else {
+                return (SyntaxShape::Any, Some(ParseError::UnknownType(span)));
+            }
+        }
     };
 
     (result, None)
@@ -3394,6 +3418,10 @@ pub fn parse_expression(
                 parse_call(working_set, &spans[pos..], expand_aliases, spans[0]).0,
                 Some(ParseError::StatementInPipeline("def".into(), spans[0])),
             ),
+            b"extern" => (
+                parse_call(working_set, &spans[pos..], expand_aliases, spans[0]).0,
+                Some(ParseError::StatementInPipeline("extern".into(), spans[0])),
+            ),
             b"let" => (
                 parse_call(working_set, &spans[pos..], expand_aliases, spans[0]).0,
                 Some(ParseError::StatementInPipeline("let".into(), spans[0])),
@@ -3513,6 +3541,7 @@ pub fn parse_statement(
 
     match name {
         b"def" | b"def-env" => parse_def(working_set, lite_command),
+        b"extern" => parse_extern(working_set, lite_command),
         b"let" => parse_let(working_set, &lite_command.parts),
         b"for" => {
             let (expr, err) = parse_for(working_set, &lite_command.parts);