Overhaul the plugin cache file with a new msgpack+brotli format (#12579)

# Description

- Plugin signatures are now saved to `plugin.msgpackz`, which is
brotli-compressed MessagePack.
- The file is updated incrementally, rather than writing all plugin
commands in the engine every time.
- The file always contains the result of the `Signature` call to the
plugin, even if commands were removed.
- Invalid data for a particular plugin just causes an error to be
reported, but the rest of the plugins can still be parsed

# User-Facing Changes

- The plugin file has a different filename, and it's not a nushell
script.
- The default `plugin.nu` file will be automatically migrated the first
time, but not other plugin config files.
- We don't currently provide any utilities that could help edit this
file, beyond `plugin add` and `plugin rm`
  - `from msgpackz`, `to msgpackz` could also help
- New commands: `plugin add`, `plugin rm`

# Tests + Formatting

Tests added for the format and for the invalid handling.

- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

# After Submitting

- [ ] Check for documentation changes
- [ ] Definitely needs release notes

crates/nu-plugin/src/lib.rs

@@ -78,10 +78,10 @@ pub use serializers::{json::JsonSerializer, msgpack::MsgPackSerializer};
 // Used by other nu crates.
 #[doc(hidden)]
 pub use plugin::{
-    create_plugin_signature, get_signature, serve_plugin_io, EngineInterfaceManager, GetPlugin,
-    Interface, InterfaceManager, PersistentPlugin, PluginDeclaration,
-    PluginExecutionCommandContext, PluginExecutionContext, PluginInterface, PluginInterfaceManager,
-    PluginSource, ServePluginError,
+    create_plugin_signature, get_signature, load_plugin_cache_item, load_plugin_file,
+    serve_plugin_io, EngineInterfaceManager, GetPlugin, Interface, InterfaceManager,
+    PersistentPlugin, PluginDeclaration, PluginExecutionCommandContext, PluginExecutionContext,
+    PluginInterface, PluginInterfaceManager, PluginSource, ServePluginError,
 };
 #[doc(hidden)]
 pub use protocol::{PluginCustomValue, PluginInput, PluginOutput};

crates/nu-plugin/src/plugin/interface/plugin.rs

@@ -750,15 +750,7 @@ impl PluginInterface {
                 help: Some(format!(
                     "the plugin may have experienced an error. Try registering the plugin again \
                         with `{}`",
-                    if let Some(shell) = self.state.source.shell() {
-                        format!(
-                            "register --shell '{}' '{}'",
-                            shell.display(),
-                            self.state.source.filename().display(),
-                        )
-                    } else {
-                        format!("register '{}'", self.state.source.filename().display())
-                    }
+                    self.state.source.identity.register_command(),
                 )),
                 inner: vec![],
             })?;

crates/nu-plugin/src/plugin/mod.rs

@@ -23,8 +23,9 @@ use std::{
 
 use nu_engine::documentation::get_flags_section;
 use nu_protocol::{
-    ast::Operator, CustomValue, IntoSpanned, LabeledError, PipelineData, PluginSignature,
-    ShellError, Spanned, Value,
+    ast::Operator, engine::StateWorkingSet, report_error_new, CustomValue, IntoSpanned,
+    LabeledError, PipelineData, PluginCacheFile, PluginCacheItem, PluginCacheItemData,
+    PluginIdentity, PluginSignature, ShellError, Span, Spanned, Value,
 };
 use thiserror::Error;
 
@@ -919,3 +920,79 @@ pub fn get_plugin_encoding(
         }
     })
 }
+
+/// Load the definitions from the plugin file into the engine state
+#[doc(hidden)]
+pub fn load_plugin_file(
+    working_set: &mut StateWorkingSet,
+    plugin_cache_file: &PluginCacheFile,
+    span: Option<Span>,
+) {
+    for plugin in &plugin_cache_file.plugins {
+        // Any errors encountered should just be logged.
+        if let Err(err) = load_plugin_cache_item(working_set, plugin, span) {
+            report_error_new(working_set.permanent_state, &err)
+        }
+    }
+}
+
+/// Load a definition from the plugin file into the engine state
+#[doc(hidden)]
+pub fn load_plugin_cache_item(
+    working_set: &mut StateWorkingSet,
+    plugin: &PluginCacheItem,
+    span: Option<Span>,
+) -> Result<(), ShellError> {
+    let identity =
+        PluginIdentity::new(plugin.filename.clone(), plugin.shell.clone()).map_err(|_| {
+            ShellError::GenericError {
+                error: "Invalid plugin filename in plugin cache file".into(),
+                msg: "loaded from here".into(),
+                span,
+                help: Some(format!(
+                    "the filename for `{}` is not a valid nushell plugin: {}",
+                    plugin.name,
+                    plugin.filename.display()
+                )),
+                inner: vec![],
+            }
+        })?;
+
+    match &plugin.data {
+        PluginCacheItemData::Valid { commands } => {
+            // Find garbage collection config for the plugin
+            let gc_config = working_set
+                .get_config()
+                .plugin_gc
+                .get(identity.name())
+                .clone();
+
+            // Add it to / get it from the working set
+            let plugin = working_set.find_or_create_plugin(&identity, || {
+                Arc::new(PersistentPlugin::new(identity.clone(), gc_config.clone()))
+            });
+
+            // Downcast the plugin to `PersistentPlugin` - we generally expect this to succeed.
+            // The trait object only exists so that nu-protocol can contain plugins without knowing
+            // anything about their implementation, but we only use `PersistentPlugin` in practice.
+            let plugin: Arc<PersistentPlugin> =
+                plugin
+                    .as_any()
+                    .downcast()
+                    .map_err(|_| ShellError::NushellFailed {
+                        msg: "encountered unexpected RegisteredPlugin type".into(),
+                    })?;
+
+            // Create the declarations from the commands
+            for signature in commands {
+                let decl = PluginDeclaration::new(plugin.clone(), signature.clone());
+                working_set.add_decl(Box::new(decl));
+            }
+            Ok(())
+        }
+        PluginCacheItemData::Invalid => Err(ShellError::PluginCacheDataInvalid {
+            plugin_name: identity.name().to_owned(),
+            register_command: identity.register_command(),
+        }),
+    }
+}