Overhaul the plugin cache file with a new msgpack+brotli format (#12579)

# Description

- Plugin signatures are now saved to `plugin.msgpackz`, which is
brotli-compressed MessagePack.
- The file is updated incrementally, rather than writing all plugin
commands in the engine every time.
- The file always contains the result of the `Signature` call to the
plugin, even if commands were removed.
- Invalid data for a particular plugin just causes an error to be
reported, but the rest of the plugins can still be parsed

# User-Facing Changes

- The plugin file has a different filename, and it's not a nushell
script.
- The default `plugin.nu` file will be automatically migrated the first
time, but not other plugin config files.
- We don't currently provide any utilities that could help edit this
file, beyond `plugin add` and `plugin rm`
  - `from msgpackz`, `to msgpackz` could also help
- New commands: `plugin add`, `plugin rm`

# Tests + Formatting

Tests added for the format and for the invalid handling.

- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

# After Submitting

- [ ] Check for documentation changes
- [ ] Definitely needs release notes

crates/nu-cmd-plugin/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+authors = ["The Nushell Project Developers"]
+description = "Commands for managing Nushell plugins."
+edition = "2021"
+license = "MIT"
+name = "nu-cmd-plugin"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-plugin"
+version = "0.92.3"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+nu-engine = { path = "../nu-engine", version = "0.92.3" }
+nu-path = { path = "../nu-path", version = "0.92.3" }
+nu-protocol = { path = "../nu-protocol", version = "0.92.3", features = ["plugin"] }
+nu-plugin = { path = "../nu-plugin", version = "0.92.3" }
+
+itertools = { workspace = true }
+
+[dev-dependencies]

crates/nu-cmd-plugin/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 - 2023 The Nushell Project Developers
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crates/nu-cmd-plugin/README.md

@@ -0,0 +1,3 @@
+# nu-cmd-plugin
+
+This crate implements Nushell commands related to plugin management.

crates/nu-cmd-plugin/src/commands/mod.rs

@@ -0,0 +1,5 @@
+mod plugin;
+mod register;
+
+pub use plugin::*;
+pub use register::Register;

crates/nu-cmd-plugin/src/commands/plugin/add.rs

@@ -0,0 +1,148 @@
+use std::sync::Arc;
+
+use nu_engine::{command_prelude::*, current_dir};
+use nu_plugin::{GetPlugin, PersistentPlugin};
+use nu_protocol::{PluginCacheItem, PluginGcConfig, PluginIdentity, RegisteredPlugin};
+
+use crate::util::modify_plugin_file;
+
+#[derive(Clone)]
+pub struct PluginAdd;
+
+impl Command for PluginAdd {
+    fn name(&self) -> &str {
+        "plugin add"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .input_output_type(Type::Nothing, Type::Nothing)
+            // This matches the option to `nu`
+            .named(
+                "plugin-config",
+                SyntaxShape::Filepath,
+                "Use a plugin cache file other than the one set in `$nu.plugin-path`",
+                None,
+            )
+            .named(
+                "shell",
+                SyntaxShape::Filepath,
+                "Use an additional shell program (cmd, sh, python, etc.) to run the plugin",
+                Some('s'),
+            )
+            .required(
+                "filename",
+                SyntaxShape::Filepath,
+                "Path to the executable for the plugin",
+            )
+            .category(Category::Plugin)
+    }
+
+    fn usage(&self) -> &str {
+        "Add a plugin to the plugin cache file."
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"
+This does not load the plugin commands into the scope - see `register` for that.
+
+Instead, it runs the plugin to get its command signatures, and then edits the
+plugin cache file (by default, `$nu.plugin-path`). The changes will be
+apparent the next time `nu` is next launched with that plugin cache file.
+"#
+        .trim()
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["plugin", "add", "register", "load", "signature"]
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                example: "plugin add nu_plugin_inc",
+                description: "Run the `nu_plugin_inc` plugin from the current directory or $env.NU_PLUGIN_DIRS and install its signatures.",
+                result: None,
+            },
+            Example {
+                example: "plugin add --plugin-config polars.msgpackz nu_plugin_polars",
+                description: "Run the `nu_plugin_polars` plugin from the current directory or $env.NU_PLUGIN_DIRS, and install its signatures to the \"polars.msgpackz\" plugin cache file.",
+                result: None,
+            },
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let filename: Spanned<String> = call.req(engine_state, stack, 0)?;
+        let shell: Option<Spanned<String>> = call.get_flag(engine_state, stack, "shell")?;
+
+        let cwd = current_dir(engine_state, stack)?;
+
+        // Check the current directory, or fall back to NU_PLUGIN_DIRS
+        let filename_expanded = match nu_path::canonicalize_with(&filename.item, &cwd) {
+            Ok(path) => path,
+            Err(err) => {
+                // Try to find it in NU_PLUGIN_DIRS first, before giving up
+                let mut found = None;
+                if let Some(nu_plugin_dirs) = stack.get_env_var(engine_state, "NU_PLUGIN_DIRS") {
+                    for dir in nu_plugin_dirs.into_list().unwrap_or(vec![]) {
+                        if let Ok(path) = nu_path::canonicalize_with(dir.as_str()?, &cwd)
+                            .and_then(|dir| nu_path::canonicalize_with(&filename.item, dir))
+                        {
+                            found = Some(path);
+                            break;
+                        }
+                    }
+                }
+                found.ok_or(err.into_spanned(filename.span))?
+            }
+        };
+
+        let shell_expanded = shell
+            .as_ref()
+            .map(|s| {
+                nu_path::canonicalize_with(&s.item, &cwd).map_err(|err| err.into_spanned(s.span))
+            })
+            .transpose()?;
+
+        // Parse the plugin filename so it can be used to spawn the plugin
+        let identity = PluginIdentity::new(filename_expanded, shell_expanded).map_err(|_| {
+            ShellError::GenericError {
+                error: "Plugin filename is invalid".into(),
+                msg: "plugin executable files must start with `nu_plugin_`".into(),
+                span: Some(filename.span),
+                help: None,
+                inner: vec![],
+            }
+        })?;
+
+        let custom_path = call.get_flag(engine_state, stack, "plugin-config")?;
+
+        // Start the plugin manually, to get the freshest signatures and to not affect engine
+        // state. Provide a GC config that will stop it ASAP
+        let plugin = Arc::new(PersistentPlugin::new(
+            identity,
+            PluginGcConfig {
+                enabled: true,
+                stop_after: 0,
+            },
+        ));
+        let interface = plugin.clone().get_plugin(Some((engine_state, stack)))?;
+        let commands = interface.get_signature()?;
+
+        modify_plugin_file(engine_state, stack, call.head, custom_path, |contents| {
+            // Update the file with the received signatures
+            let item = PluginCacheItem::new(plugin.identity(), commands);
+            contents.upsert_plugin(item);
+            Ok(())
+        })?;
+
+        Ok(Value::nothing(call.head).into_pipeline_data())
+    }
+}

crates/nu-cmd-plugin/src/commands/plugin/list.rs

@@ -0,0 +1,96 @@
+use itertools::Itertools;
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct PluginList;
+
+impl Command for PluginList {
+    fn name(&self) -> &str {
+        "plugin list"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("plugin list")
+            .input_output_type(
+                Type::Nothing,
+                Type::Table(vec![
+                    ("name".into(), Type::String),
+                    ("is_running".into(), Type::Bool),
+                    ("pid".into(), Type::Int),
+                    ("filename".into(), Type::String),
+                    ("shell".into(), Type::String),
+                    ("commands".into(), Type::List(Type::String.into())),
+                ]),
+            )
+            .category(Category::Plugin)
+    }
+
+    fn usage(&self) -> &str {
+        "List installed plugins."
+    }
+
+    fn examples(&self) -> Vec<nu_protocol::Example> {
+        vec![
+            Example {
+                example: "plugin list",
+                description: "List installed plugins.",
+                result: Some(Value::test_list(vec![Value::test_record(record! {
+                    "name" => Value::test_string("inc"),
+                    "is_running" => Value::test_bool(true),
+                    "pid" => Value::test_int(106480),
+                    "filename" => if cfg!(windows) {
+                        Value::test_string(r"C:\nu\plugins\nu_plugin_inc.exe")
+                    } else {
+                        Value::test_string("/opt/nu/plugins/nu_plugin_inc")
+                    },
+                    "shell" => Value::test_nothing(),
+                    "commands" => Value::test_list(vec![Value::test_string("inc")]),
+                })])),
+            },
+            Example {
+                example: "ps | where pid in (plugin list).pid",
+                description: "Get process information for running plugins.",
+                result: None,
+            },
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let span = call.span();
+        // Group plugin decls by plugin identity
+        let decls = engine_state.plugin_decls().into_group_map_by(|decl| {
+            decl.plugin_identity()
+                .expect("plugin decl should have identity")
+        });
+        // Build plugins list
+        let list = engine_state.plugins().iter().map(|plugin| {
+            // Find commands that belong to the plugin
+            let commands = decls.get(plugin.identity())
+                .into_iter()
+                .flat_map(|decls| {
+                    decls.iter().map(|decl| Value::string(decl.name(), span))
+                })
+                .collect();
+
+            Value::record(record! {
+                "name" => Value::string(plugin.identity().name(), span),
+                "is_running" => Value::bool(plugin.is_running(), span),
+                "pid" => plugin.pid()
+                    .map(|p| Value::int(p as i64, span))
+                    .unwrap_or(Value::nothing(span)),
+                "filename" => Value::string(plugin.identity().filename().to_string_lossy(), span),
+                "shell" => plugin.identity().shell()
+                    .map(|s| Value::string(s.to_string_lossy(), span))
+                    .unwrap_or(Value::nothing(span)),
+                "commands" => Value::list(commands, span),
+            }, span)
+        }).collect::<Vec<Value>>();
+        Ok(list.into_pipeline_data(engine_state.ctrlc.clone()))
+    }
+}

crates/nu-cmd-plugin/src/commands/plugin/mod.rs

@@ -0,0 +1,79 @@
+use nu_engine::{command_prelude::*, get_full_help};
+
+mod add;
+mod list;
+mod rm;
+mod stop;
+
+pub use add::PluginAdd;
+pub use list::PluginList;
+pub use rm::PluginRm;
+pub use stop::PluginStop;
+
+#[derive(Clone)]
+pub struct PluginCommand;
+
+impl Command for PluginCommand {
+    fn name(&self) -> &str {
+        "plugin"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("plugin")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .category(Category::Plugin)
+    }
+
+    fn usage(&self) -> &str {
+        "Commands for managing plugins."
+    }
+
+    fn extra_usage(&self) -> &str {
+        "To load a plugin, see `register`."
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(Value::string(
+            get_full_help(
+                &PluginCommand.signature(),
+                &PluginCommand.examples(),
+                engine_state,
+                stack,
+                self.is_parser_keyword(),
+            ),
+            call.head,
+        )
+        .into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                example: "plugin list",
+                description: "List installed plugins",
+                result: None,
+            },
+            Example {
+                example: "plugin stop inc",
+                description: "Stop the plugin named `inc`.",
+                result: None,
+            },
+            Example {
+                example: "plugin add nu_plugin_inc",
+                description: "Run the `nu_plugin_inc` plugin from the current directory and install its signatures.",
+                result: None,
+            },
+            Example {
+                example: "plugin rm inc",
+                description: "Remove the installed signatures for the `inc` plugin.",
+                result: None,
+            },
+        ]
+    }
+}

crates/nu-cmd-plugin/src/commands/plugin/rm.rs

@@ -0,0 +1,100 @@
+use nu_engine::command_prelude::*;
+
+use crate::util::modify_plugin_file;
+
+#[derive(Clone)]
+pub struct PluginRm;
+
+impl Command for PluginRm {
+    fn name(&self) -> &str {
+        "plugin rm"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .input_output_type(Type::Nothing, Type::Nothing)
+            // This matches the option to `nu`
+            .named(
+                "plugin-config",
+                SyntaxShape::Filepath,
+                "Use a plugin cache file other than the one set in `$nu.plugin-path`",
+                None,
+            )
+            .switch(
+                "force",
+                "Don't cause an error if the plugin name wasn't found in the file",
+                Some('f'),
+            )
+            .required(
+                "name",
+                SyntaxShape::String,
+                "The name of the plugin to remove (not the filename)",
+            )
+            .category(Category::Plugin)
+    }
+
+    fn usage(&self) -> &str {
+        "Remove a plugin from the plugin cache file."
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"
+This does not remove the plugin commands from the current scope or from `plugin
+list` in the current shell. It instead removes the plugin from the plugin
+cache file (by default, `$nu.plugin-path`). The changes will be apparent the
+next time `nu` is launched with that plugin cache file.
+
+This can be useful for removing an invalid plugin signature, if it can't be
+fixed with `plugin add`.
+"#
+        .trim()
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["plugin", "rm", "remove", "delete", "signature"]
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                example: "plugin rm inc",
+                description: "Remove the installed signatures for the `inc` plugin.",
+                result: None,
+            },
+            Example {
+                example: "plugin rm --plugin-config polars.msgpackz polars",
+                description: "Remove the installed signatures for the `polars` plugin from the \"polars.msgpackz\" plugin cache file.",
+                result: None,
+            },
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let name: Spanned<String> = call.req(engine_state, stack, 0)?;
+        let custom_path = call.get_flag(engine_state, stack, "plugin-config")?;
+        let force = call.has_flag(engine_state, stack, "force")?;
+
+        modify_plugin_file(engine_state, stack, call.head, custom_path, |contents| {
+            if !force && !contents.plugins.iter().any(|p| p.name == name.item) {
+                Err(ShellError::GenericError {
+                    error: format!("Failed to remove the `{}` plugin", name.item),
+                    msg: "couldn't find a plugin with this name in the cache file".into(),
+                    span: Some(name.span),
+                    help: None,
+                    inner: vec![],
+                })
+            } else {
+                contents.remove_plugin(&name.item);
+                Ok(())
+            }
+        })?;
+
+        Ok(Value::nothing(call.head).into_pipeline_data())
+    }
+}

crates/nu-cmd-plugin/src/commands/plugin/stop.rs

@@ -0,0 +1,70 @@
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct PluginStop;
+
+impl Command for PluginStop {
+    fn name(&self) -> &str {
+        "plugin stop"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("plugin stop")
+            .input_output_type(Type::Nothing, Type::Nothing)
+            .required(
+                "name",
+                SyntaxShape::String,
+                "The name of the plugin to stop.",
+            )
+            .category(Category::Plugin)
+    }
+
+    fn usage(&self) -> &str {
+        "Stop an installed plugin if it was running."
+    }
+
+    fn examples(&self) -> Vec<nu_protocol::Example> {
+        vec![
+            Example {
+                example: "plugin stop inc",
+                description: "Stop the plugin named `inc`.",
+                result: None,
+            },
+            Example {
+                example: "plugin list | each { |p| plugin stop $p.name }",
+                description: "Stop all plugins.",
+                result: None,
+            },
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let name: Spanned<String> = call.req(engine_state, stack, 0)?;
+
+        let mut found = false;
+        for plugin in engine_state.plugins() {
+            if plugin.identity().name() == name.item {
+                plugin.stop()?;
+                found = true;
+            }
+        }
+
+        if found {
+            Ok(PipelineData::Empty)
+        } else {
+            Err(ShellError::GenericError {
+                error: format!("Failed to stop the `{}` plugin", name.item),
+                msg: "couldn't find a plugin with this name".into(),
+                span: Some(name.span),
+                help: Some("you may need to `register` the plugin first".into()),
+                inner: vec![],
+            })
+        }
+    }
+}

crates/nu-cmd-plugin/src/commands/register.rs

@@ -0,0 +1,74 @@
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct Register;
+
+impl Command for Register {
+    fn name(&self) -> &str {
+        "register"
+    }
+
+    fn usage(&self) -> &str {
+        "Register a plugin."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("register")
+            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
+            .required(
+                "plugin",
+                SyntaxShape::Filepath,
+                "Path of executable for plugin.",
+            )
+            .optional(
+                "signature",
+                SyntaxShape::Any,
+                "Block with signature description as json object.",
+            )
+            .named(
+                "shell",
+                SyntaxShape::Filepath,
+                "path of shell used to run plugin (cmd, sh, python, etc)",
+                Some('s'),
+            )
+            .category(Category::Plugin)
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check:
+  https://www.nushell.sh/book/thinking_in_nu.html"#
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["plugin", "add", "register"]
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        _call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(PipelineData::empty())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Register `nu_plugin_query` plugin from ~/.cargo/bin/ dir",
+                example: r#"register ~/.cargo/bin/nu_plugin_query"#,
+                result: None,
+            },
+            Example {
+                description: "Register `nu_plugin_query` plugin from `nu -c` (writes/updates $nu.plugin-path)",
+                example: r#"let plugin = ((which nu).path.0 | path dirname | path join 'nu_plugin_query'); nu -c $'register ($plugin); version'"#,
+                result: None,
+            },
+        ]
+    }
+}

crates/nu-cmd-plugin/src/default_context.rs

@@ -0,0 +1,31 @@
+use crate::*;
+use nu_protocol::engine::{EngineState, StateWorkingSet};
+
+pub fn add_plugin_command_context(mut engine_state: EngineState) -> EngineState {
+    let delta = {
+        let mut working_set = StateWorkingSet::new(&engine_state);
+
+        macro_rules! bind_command {
+            ( $( $command:expr ),* $(,)? ) => {
+                $( working_set.add_decl(Box::new($command)); )*
+            };
+        }
+
+        bind_command!(
+            PluginCommand,
+            PluginAdd,
+            PluginList,
+            PluginRm,
+            PluginStop,
+            Register,
+        );
+
+        working_set.render()
+    };
+
+    if let Err(err) = engine_state.merge_delta(delta) {
+        eprintln!("Error creating default context: {err:?}");
+    }
+
+    engine_state
+}

crates/nu-cmd-plugin/src/lib.rs

@@ -0,0 +1,8 @@
+//! Nushell commands for managing plugins.
+
+mod commands;
+mod default_context;
+mod util;
+
+pub use commands::*;
+pub use default_context::*;

crates/nu-cmd-plugin/src/util.rs

@@ -0,0 +1,50 @@
+use std::fs::{self, File};
+
+use nu_engine::{command_prelude::*, current_dir};
+use nu_protocol::PluginCacheFile;
+
+pub(crate) fn modify_plugin_file(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    span: Span,
+    custom_path: Option<Spanned<String>>,
+    operate: impl FnOnce(&mut PluginCacheFile) -> Result<(), ShellError>,
+) -> Result<(), ShellError> {
+    let cwd = current_dir(engine_state, stack)?;
+
+    let plugin_cache_file_path = if let Some(ref custom_path) = custom_path {
+        nu_path::expand_path_with(&custom_path.item, cwd, true)
+    } else {
+        engine_state
+            .plugin_path
+            .clone()
+            .ok_or_else(|| ShellError::GenericError {
+                error: "Plugin cache file not set".into(),
+                msg: "pass --plugin-config explicitly here".into(),
+                span: Some(span),
+                help: Some("you may be running `nu` with --no-config-file".into()),
+                inner: vec![],
+            })?
+    };
+
+    // Try to read the plugin file if it exists
+    let mut contents = if fs::metadata(&plugin_cache_file_path).is_ok_and(|m| m.len() > 0) {
+        PluginCacheFile::read_from(
+            File::open(&plugin_cache_file_path).map_err(|err| err.into_spanned(span))?,
+            Some(span),
+        )?
+    } else {
+        PluginCacheFile::default()
+    };
+
+    // Do the operation
+    operate(&mut contents)?;
+
+    // Save the modified file on success
+    contents.write_to(
+        File::create(&plugin_cache_file_path).map_err(|err| err.into_spanned(span))?,
+        Some(span),
+    )?;
+
+    Ok(())
+}