Make plugins able to find and call other commands (#13407)

# Description

Adds functionality to the plugin interface to support calling internal
commands from plugins. For example, using `view ir --json`:

```rust
let closure: Value = call.req(0)?;

let Some(decl_id) = engine.find_decl("view ir")? else {
    return Err(LabeledError::new("`view ir` not found"));
};

let ir_json = engine.call_decl(
    decl_id,
    EvaluatedCall::new(call.head)
        .with_named("json".into_spanned(call.head), Value::bool(true, call.head))
        .with_positional(closure),
    PipelineData::Empty,
    true,
    false,
)?.into_value()?.into_string()?;

let ir = serde_json::from_value(&ir_json);

// ...
```

# User-Facing Changes

Plugin developers can now use `EngineInterface::find_decl()` and
`call_decl()` to call internal commands, which could be handy for
formatters like `to csv` or `to nuon`, or for reflection commands that
help gain insight into the engine.

# Tests + Formatting
- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

# After Submitting
- [ ] release notes
- [ ] update plugin protocol documentation: `FindDecl`, `CallDecl`
engine calls; `Identifier` engine call response

crates/nu-plugin-engine/src/context.rs

@@ -1,9 +1,10 @@
 use crate::util::MutableCow;
 use nu_engine::{get_eval_block_with_early_return, get_full_help, ClosureEvalOnce};
+use nu_plugin_protocol::EvaluatedCall;
 use nu_protocol::{
     engine::{Call, Closure, EngineState, Redirection, Stack},
-    Config, IntoSpanned, OutDest, PipelineData, PluginIdentity, ShellError, Signals, Span, Spanned,
+    ir, Config, DeclId, IntoSpanned, OutDest, PipelineData, PluginIdentity, ShellError, Signals,
-    Value,
+    Span, Spanned, Value,
 };
 use std::{
     borrow::Cow,
@@ -44,6 +45,17 @@ pub trait PluginExecutionContext: Send + Sync {
         redirect_stdout: bool,
         redirect_stderr: bool,
     ) -> Result<PipelineData, ShellError>;
+    /// Find a declaration by name
+    fn find_decl(&self, name: &str) -> Result<Option<DeclId>, ShellError>;
+    /// Call a declaration with arguments and input
+    fn call_decl(
+        &mut self,
+        decl_id: DeclId,
+        call: EvaluatedCall,
+        input: PipelineData,
+        redirect_stdout: bool,
+        redirect_stderr: bool,
+    ) -> Result<PipelineData, ShellError>;
     /// Create an owned version of the context with `'static` lifetime
     fn boxed(&self) -> Box<dyn PluginExecutionContext>;
 }
@@ -177,19 +189,10 @@ impl<'a> PluginExecutionContext for PluginExecutionCommandContext<'a> {
             .captures_to_stack(closure.item.captures)
             .reset_pipes();
 
-        let stdout = if redirect_stdout {
+        let stack = &mut stack.push_redirection(
-            Some(Redirection::Pipe(OutDest::Capture))
+            redirect_stdout.then_some(Redirection::Pipe(OutDest::Capture)),
-        } else {
+            redirect_stderr.then_some(Redirection::Pipe(OutDest::Capture)),
-            None
+        );
-        };
-
-        let stderr = if redirect_stderr {
-            Some(Redirection::Pipe(OutDest::Capture))
-        } else {
-            None
-        };
-
-        let stack = &mut stack.push_redirection(stdout, stderr);
 
         // Set up the positional arguments
         for (idx, value) in positional.into_iter().enumerate() {
@@ -211,6 +214,57 @@ impl<'a> PluginExecutionContext for PluginExecutionCommandContext<'a> {
         eval_block_with_early_return(&self.engine_state, stack, block, input)
     }
 
+    fn find_decl(&self, name: &str) -> Result<Option<DeclId>, ShellError> {
+        Ok(self.engine_state.find_decl(name.as_bytes(), &[]))
+    }
+
+    fn call_decl(
+        &mut self,
+        decl_id: DeclId,
+        call: EvaluatedCall,
+        input: PipelineData,
+        redirect_stdout: bool,
+        redirect_stderr: bool,
+    ) -> Result<PipelineData, ShellError> {
+        if decl_id >= self.engine_state.num_decls() {
+            return Err(ShellError::GenericError {
+                error: "Plugin misbehaving".into(),
+                msg: format!("Tried to call unknown decl id: {}", decl_id),
+                span: Some(call.head),
+                help: None,
+                inner: vec![],
+            });
+        }
+
+        let decl = self.engine_state.get_decl(decl_id);
+
+        let stack = &mut self.stack.push_redirection(
+            redirect_stdout.then_some(Redirection::Pipe(OutDest::Capture)),
+            redirect_stderr.then_some(Redirection::Pipe(OutDest::Capture)),
+        );
+
+        let mut call_builder = ir::Call::build(decl_id, call.head);
+
+        for positional in call.positional {
+            call_builder.add_positional(stack, positional.span(), positional);
+        }
+
+        for (name, value) in call.named {
+            if let Some(value) = value {
+                call_builder.add_named(stack, &name.item, "", name.span, value);
+            } else {
+                call_builder.add_flag(stack, &name.item, "", name.span);
+            }
+        }
+
+        decl.run(
+            &self.engine_state,
+            stack,
+            &(&call_builder.finish()).into(),
+            input,
+        )
+    }
+
     fn boxed(&self) -> Box<dyn PluginExecutionContext + 'static> {
         Box::new(PluginExecutionCommandContext {
             identity: self.identity.clone(),
@@ -298,6 +352,25 @@ impl PluginExecutionContext for PluginExecutionBogusContext {
         })
     }
 
+    fn find_decl(&self, _name: &str) -> Result<Option<DeclId>, ShellError> {
+        Err(ShellError::NushellFailed {
+            msg: "find_decl not implemented on bogus".into(),
+        })
+    }
+
+    fn call_decl(
+        &mut self,
+        _decl_id: DeclId,
+        _call: EvaluatedCall,
+        _input: PipelineData,
+        _redirect_stdout: bool,
+        _redirect_stderr: bool,
+    ) -> Result<PipelineData, ShellError> {
+        Err(ShellError::NushellFailed {
+            msg: "call_decl not implemented on bogus".into(),
+        })
+    }
+
     fn boxed(&self) -> Box<dyn PluginExecutionContext + 'static> {
         Box::new(PluginExecutionBogusContext)
     }

crates/nu-plugin-engine/src/interface/mod.rs

@@ -1316,6 +1316,22 @@ pub(crate) fn handle_engine_call(
         } => context
             .eval_closure(closure, positional, input, redirect_stdout, redirect_stderr)
             .map(EngineCallResponse::PipelineData),
+        EngineCall::FindDecl(name) => context.find_decl(&name).map(|decl_id| {
+            if let Some(decl_id) = decl_id {
+                EngineCallResponse::Identifier(decl_id)
+            } else {
+                EngineCallResponse::empty()
+            }
+        }),
+        EngineCall::CallDecl {
+            decl_id,
+            call,
+            input,
+            redirect_stdout,
+            redirect_stderr,
+        } => context
+            .call_decl(decl_id, call, input, redirect_stdout, redirect_stderr)
+            .map(EngineCallResponse::PipelineData),
     }
 }
 

crates/nu-plugin-protocol/src/evaluated_call.rs

@@ -27,6 +27,82 @@ pub struct EvaluatedCall {
 }
 
 impl EvaluatedCall {
+    /// Create a new [`EvaluatedCall`] with the given head span.
+    pub fn new(head: Span) -> EvaluatedCall {
+        EvaluatedCall {
+            head,
+            positional: vec![],
+            named: vec![],
+        }
+    }
+
+    /// Add a positional argument to an [`EvaluatedCall`].
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// # use nu_protocol::{Value, Span, IntoSpanned};
+    /// # use nu_plugin_protocol::EvaluatedCall;
+    /// # let head = Span::test_data();
+    /// let mut call = EvaluatedCall::new(head);
+    /// call.add_positional(Value::test_int(1337));
+    /// ```
+    pub fn add_positional(&mut self, value: Value) -> &mut Self {
+        self.positional.push(value);
+        self
+    }
+
+    /// Add a named argument to an [`EvaluatedCall`].
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// # use nu_protocol::{Value, Span, IntoSpanned};
+    /// # use nu_plugin_protocol::EvaluatedCall;
+    /// # let head = Span::test_data();
+    /// let mut call = EvaluatedCall::new(head);
+    /// call.add_named("foo".into_spanned(head), Value::test_string("bar"));
+    /// ```
+    pub fn add_named(&mut self, name: Spanned<impl Into<String>>, value: Value) -> &mut Self {
+        self.named.push((name.map(Into::into), Some(value)));
+        self
+    }
+
+    /// Add a flag argument to an [`EvaluatedCall`]. A flag argument is a named argument with no
+    /// value.
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// # use nu_protocol::{Value, Span, IntoSpanned};
+    /// # use nu_plugin_protocol::EvaluatedCall;
+    /// # let head = Span::test_data();
+    /// let mut call = EvaluatedCall::new(head);
+    /// call.add_flag("pretty".into_spanned(head));
+    /// ```
+    pub fn add_flag(&mut self, name: Spanned<impl Into<String>>) -> &mut Self {
+        self.named.push((name.map(Into::into), None));
+        self
+    }
+
+    /// Builder variant of [`.add_positional()`].
+    pub fn with_positional(mut self, value: Value) -> Self {
+        self.add_positional(value);
+        self
+    }
+
+    /// Builder variant of [`.add_named()`].
+    pub fn with_named(mut self, name: Spanned<impl Into<String>>, value: Value) -> Self {
+        self.add_named(name, value);
+        self
+    }
+
+    /// Builder variant of [`.add_flag()`].
+    pub fn with_flag(mut self, name: Spanned<impl Into<String>>) -> Self {
+        self.add_flag(name);
+        self
+    }
+
     /// Try to create an [`EvaluatedCall`] from a command `Call`.
     pub fn try_from_call(
         call: &Call,
@@ -192,6 +268,16 @@ impl EvaluatedCall {
         Ok(false)
     }
 
+    /// Returns the [`Span`] of the name of an optional named argument.
+    ///
+    /// This can be used in errors for named arguments that don't take values.
+    pub fn get_flag_span(&self, flag_name: &str) -> Option<Span> {
+        self.named
+            .iter()
+            .find(|(name, _)| name.item == flag_name)
+            .map(|(name, _)| name.span)
+    }
+
     /// Returns the [`Value`] of an optional named argument
     ///
     /// # Examples

crates/nu-plugin-protocol/src/lib.rs

@@ -22,7 +22,7 @@ mod tests;
 pub mod test_util;
 
 use nu_protocol::{
-    ast::Operator, engine::Closure, ByteStreamType, Config, LabeledError, PipelineData,
+    ast::Operator, engine::Closure, ByteStreamType, Config, DeclId, LabeledError, PipelineData,
     PluginMetadata, PluginSignature, ShellError, Span, Spanned, Value,
 };
 use nu_utils::SharedCow;
@@ -494,6 +494,21 @@ pub enum EngineCall<D> {
         /// Whether to redirect stderr from external commands
         redirect_stderr: bool,
     },
+    /// Find a declaration by name
+    FindDecl(String),
+    /// Call a declaration with args
+    CallDecl {
+        /// The id of the declaration to be called (can be found with `FindDecl`)
+        decl_id: DeclId,
+        /// Information about the call (head span, arguments, etc.)
+        call: EvaluatedCall,
+        /// Pipeline input to the call
+        input: D,
+        /// Whether to redirect stdout from external commands
+        redirect_stdout: bool,
+        /// Whether to redirect stderr from external commands
+        redirect_stderr: bool,
+    },
 }
 
 impl<D> EngineCall<D> {
@@ -511,6 +526,8 @@ impl<D> EngineCall<D> {
             EngineCall::LeaveForeground => "LeaveForeground",
             EngineCall::GetSpanContents(_) => "GetSpanContents",
             EngineCall::EvalClosure { .. } => "EvalClosure",
+            EngineCall::FindDecl(_) => "FindDecl",
+            EngineCall::CallDecl { .. } => "CallDecl",
         }
     }
 
@@ -544,6 +561,20 @@ impl<D> EngineCall<D> {
                 redirect_stdout,
                 redirect_stderr,
             },
+            EngineCall::FindDecl(name) => EngineCall::FindDecl(name),
+            EngineCall::CallDecl {
+                decl_id,
+                call,
+                input,
+                redirect_stdout,
+                redirect_stderr,
+            } => EngineCall::CallDecl {
+                decl_id,
+                call,
+                input: f(input)?,
+                redirect_stdout,
+                redirect_stderr,
+            },
         })
     }
 }
@@ -556,6 +587,7 @@ pub enum EngineCallResponse<D> {
     PipelineData(D),
     Config(SharedCow<Config>),
     ValueMap(HashMap<String, Value>),
+    Identifier(usize),
 }
 
 impl<D> EngineCallResponse<D> {
@@ -570,6 +602,7 @@ impl<D> EngineCallResponse<D> {
             EngineCallResponse::PipelineData(data) => EngineCallResponse::PipelineData(f(data)?),
             EngineCallResponse::Config(config) => EngineCallResponse::Config(config),
             EngineCallResponse::ValueMap(map) => EngineCallResponse::ValueMap(map),
+            EngineCallResponse::Identifier(id) => EngineCallResponse::Identifier(id),
         })
     }
 }

crates/nu-plugin/src/plugin/interface/mod.rs

@@ -6,12 +6,12 @@ use nu_plugin_core::{
     StreamManagerHandle,
 };
 use nu_plugin_protocol::{
-    CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, Ordering, PluginCall,
+    CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, EvaluatedCall, Ordering,
-    PluginCallId, PluginCallResponse, PluginCustomValue, PluginInput, PluginOption, PluginOutput,
+    PluginCall, PluginCallId, PluginCallResponse, PluginCustomValue, PluginInput, PluginOption,
-    ProtocolInfo,
+    PluginOutput, ProtocolInfo,
 };
 use nu_protocol::{
-    engine::Closure, Config, LabeledError, PipelineData, PluginMetadata, PluginSignature,
+    engine::Closure, Config, DeclId, LabeledError, PipelineData, PluginMetadata, PluginSignature,
     ShellError, Signals, Span, Spanned, Value,
 };
 use nu_utils::SharedCow;
@@ -872,6 +872,71 @@ impl EngineInterface {
         }
     }
 
+    /// Ask the engine for the identifier for a declaration. If found, the result can then be passed
+    /// to [`.call_decl()`] to call other internal commands.
+    ///
+    /// See [`.call_decl()`] for an example.
+    pub fn find_decl(&self, name: impl Into<String>) -> Result<Option<DeclId>, ShellError> {
+        let call = EngineCall::FindDecl(name.into());
+
+        match self.engine_call(call)? {
+            EngineCallResponse::Error(err) => Err(err),
+            EngineCallResponse::Identifier(id) => Ok(Some(id)),
+            EngineCallResponse::PipelineData(PipelineData::Empty) => Ok(None),
+            _ => Err(ShellError::PluginFailedToDecode {
+                msg: "Received unexpected response type for EngineCall::FindDecl".into(),
+            }),
+        }
+    }
+
+    /// Ask the engine to call an internal command, using the declaration ID previously looked up
+    /// with [`.find_decl()`].
+    ///
+    /// # Example
+    ///
+    /// ```rust,no_run
+    /// # use nu_protocol::{Value, ShellError, PipelineData};
+    /// # use nu_plugin::{EngineInterface, EvaluatedCall};
+    /// # fn example(engine: &EngineInterface, call: &EvaluatedCall) -> Result<Value, ShellError> {
+    /// if let Some(decl_id) = engine.find_decl("scope commands")? {
+    ///     let commands = engine.call_decl(
+    ///         decl_id,
+    ///         EvaluatedCall::new(call.head),
+    ///         PipelineData::Empty,
+    ///         true,
+    ///         false,
+    ///     )?;
+    ///     commands.into_value(call.head)
+    /// } else {
+    ///     Ok(Value::list(vec![], call.head))
+    /// }
+    /// # }
+    /// ```
+    pub fn call_decl(
+        &self,
+        decl_id: DeclId,
+        call: EvaluatedCall,
+        input: PipelineData,
+        redirect_stdout: bool,
+        redirect_stderr: bool,
+    ) -> Result<PipelineData, ShellError> {
+        let call = EngineCall::CallDecl {
+            decl_id,
+            call,
+            input,
+            redirect_stdout,
+            redirect_stderr,
+        };
+
+        match self.engine_call(call)? {
+            EngineCallResponse::Error(err) => Err(err),
+            EngineCallResponse::PipelineData(data) => Ok(data),
+            _ => Err(ShellError::PluginFailedToDecode {
+                msg: "Received unexpected response type for EngineCall::CallDecl".into(),
+            }),
+        }
+    }
+
     /// Tell the engine whether to disable garbage collection for this plugin.
     ///
     /// The garbage collector is enabled by default, but plugins can turn it off (ideally

crates/nu-protocol/src/ir/call.rs

@@ -239,7 +239,7 @@ impl CallBuilder {
         }
         self.inner.args_len += 1;
         if let Some(span) = argument.span() {
-            self.inner.span = self.inner.span.append(span);
+            self.inner.span = self.inner.span.merge(span);
         }
         stack.arguments.push(argument);
         self

crates/nu_plugin_example/src/commands/call_decl.rs

@@ -0,0 +1,78 @@
+use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use nu_protocol::{
+    IntoSpanned, LabeledError, PipelineData, Record, Signature, Spanned, SyntaxShape, Value,
+};
+
+use crate::ExamplePlugin;
+
+pub struct CallDecl;
+
+impl PluginCommand for CallDecl {
+    type Plugin = ExamplePlugin;
+
+    fn name(&self) -> &str {
+        "example call-decl"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .required(
+                "name",
+                SyntaxShape::String,
+                "the name of the command to call",
+            )
+            .optional(
+                "named_args",
+                SyntaxShape::Record(vec![]),
+                "named arguments to pass to the command",
+            )
+            .rest(
+                "positional_args",
+                SyntaxShape::Any,
+                "positional arguments to pass to the command",
+            )
+    }
+
+    fn usage(&self) -> &str {
+        "Demonstrates calling other commands from plugins using `call_decl()`."
+    }
+
+    fn extra_usage(&self) -> &str {
+        "
+The arguments will not be typechecked at parse time. This command is for
+demonstration only, and should not be used for anything real.
+"
+        .trim()
+    }
+
+    fn run(
+        &self,
+        _plugin: &ExamplePlugin,
+        engine: &EngineInterface,
+        call: &EvaluatedCall,
+        input: PipelineData,
+    ) -> Result<PipelineData, LabeledError> {
+        let name: Spanned<String> = call.req(0)?;
+        let named_args: Option<Record> = call.opt(1)?;
+        let positional_args: Vec<Value> = call.rest(2)?;
+
+        let decl_id = engine.find_decl(&name.item)?.ok_or_else(|| {
+            LabeledError::new(format!("Can't find `{}`", name.item))
+                .with_label("not in scope", name.span)
+        })?;
+
+        let mut new_call = EvaluatedCall::new(call.head);
+
+        for (key, val) in named_args.into_iter().flatten() {
+            new_call.add_named(key.into_spanned(val.span()), val);
+        }
+
+        for val in positional_args {
+            new_call.add_positional(val);
+        }
+
+        let result = engine.call_decl(decl_id, new_call, input, true, false)?;
+
+        Ok(result)
+    }
+}

crates/nu_plugin_example/src/commands/mod.rs

@@ -13,11 +13,13 @@ pub use three::Three;
 pub use two::Two;
 
 // Engine interface demos
+mod call_decl;
 mod config;
 mod disable_gc;
 mod env;
 mod view_span;
 
+pub use call_decl::CallDecl;
 pub use config::Config;
 pub use disable_gc::DisableGc;
 pub use env::Env;

crates/nu_plugin_example/src/lib.rs

@@ -27,6 +27,7 @@ impl Plugin for ExamplePlugin {
             Box::new(Env),
             Box::new(ViewSpan),
             Box::new(DisableGc),
+            Box::new(CallDecl),
             // Stream demos
             Box::new(CollectBytes),
             Box::new(Echo),

tests/plugins/call_decl.rs

@@ -0,0 +1,42 @@
+use nu_test_support::nu_with_plugins;
+
+#[test]
+fn call_to_json() {
+    let result = nu_with_plugins!(
+        cwd: ".",
+        plugin: ("nu_plugin_example"),
+        r#"
+            [42] | example call-decl 'to json' {indent: 4}
+        "#
+    );
+    assert!(result.status.success());
+    // newlines are removed from test output
+    assert_eq!("[    42]", result.out);
+}
+
+#[test]
+fn call_reduce() {
+    let result = nu_with_plugins!(
+        cwd: ".",
+        plugin: ("nu_plugin_example"),
+        r#"
+            [1 2 3] | example call-decl 'reduce' {fold: 10} { |it, acc| $it + $acc }
+        "#
+    );
+    assert!(result.status.success());
+    assert_eq!("16", result.out);
+}
+
+#[test]
+fn call_scope_variables() {
+    let result = nu_with_plugins!(
+        cwd: ".",
+        plugin: ("nu_plugin_example"),
+        r#"
+            let test_var = 10
+            example call-decl 'scope variables' | where name == '$test_var' | length
+        "#
+    );
+    assert!(result.status.success());
+    assert_eq!("1", result.out);
+}

tests/plugins/mod.rs

@@ -1,3 +1,4 @@
+mod call_decl;
 mod config;
 mod core_inc;
 mod custom_values;