Overhaul the plugin cache file with a new msgpack+brotli format (#12579)

# Description

- Plugin signatures are now saved to `plugin.msgpackz`, which is
brotli-compressed MessagePack.
- The file is updated incrementally, rather than writing all plugin
commands in the engine every time.
- The file always contains the result of the `Signature` call to the
plugin, even if commands were removed.
- Invalid data for a particular plugin just causes an error to be
reported, but the rest of the plugins can still be parsed

# User-Facing Changes

- The plugin file has a different filename, and it's not a nushell
script.
- The default `plugin.nu` file will be automatically migrated the first
time, but not other plugin config files.
- We don't currently provide any utilities that could help edit this
file, beyond `plugin add` and `plugin rm`
  - `from msgpackz`, `to msgpackz` could also help
- New commands: `plugin add`, `plugin rm`

# Tests + Formatting

Tests added for the format and for the invalid handling.

- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

# After Submitting

- [ ] Check for documentation changes
- [ ] Definitely needs release notes

crates/nu-protocol/src/plugin/cache_file/mod.rs

@@ -0,0 +1,175 @@
+use std::{
+    io::{Read, Write},
+    path::PathBuf,
+};
+
+use serde::{Deserialize, Serialize};
+
+use crate::{PluginIdentity, PluginSignature, ShellError, Span};
+
+// This has a big impact on performance
+const BUFFER_SIZE: usize = 65536;
+
+// Chose settings at the low end, because we're just trying to get the maximum speed
+const COMPRESSION_QUALITY: u32 = 1;
+const WIN_SIZE: u32 = 20; // recommended 20-22
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct PluginCacheFile {
+    /// The Nushell version that last updated the file.
+    pub nushell_version: String,
+
+    /// The installed plugins.
+    pub plugins: Vec<PluginCacheItem>,
+}
+
+impl Default for PluginCacheFile {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl PluginCacheFile {
+    /// Create a new, empty plugin cache file.
+    pub fn new() -> PluginCacheFile {
+        PluginCacheFile {
+            nushell_version: env!("CARGO_PKG_VERSION").to_owned(),
+            plugins: vec![],
+        }
+    }
+
+    /// Read the plugin cache file from a reader, e.g. [`File`](std::fs::File).
+    pub fn read_from(
+        reader: impl Read,
+        error_span: Option<Span>,
+    ) -> Result<PluginCacheFile, ShellError> {
+        // Format is brotli compressed messagepack
+        let brotli_reader = brotli::Decompressor::new(reader, BUFFER_SIZE);
+
+        rmp_serde::from_read(brotli_reader).map_err(|err| ShellError::GenericError {
+            error: format!("Failed to load plugin file: {err}"),
+            msg: "plugin file load attempted here".into(),
+            span: error_span,
+            help: Some(
+                "it may be corrupt. Try deleting it and registering your plugins again".into(),
+            ),
+            inner: vec![],
+        })
+    }
+
+    /// Write the plugin cache file to a writer, e.g. [`File`](std::fs::File).
+    ///
+    /// The `nushell_version` will be updated to the current version before writing.
+    pub fn write_to(
+        &mut self,
+        writer: impl Write,
+        error_span: Option<Span>,
+    ) -> Result<(), ShellError> {
+        // Update the Nushell version before writing
+        self.nushell_version = env!("CARGO_PKG_VERSION").to_owned();
+
+        // Format is brotli compressed messagepack
+        let mut brotli_writer =
+            brotli::CompressorWriter::new(writer, BUFFER_SIZE, COMPRESSION_QUALITY, WIN_SIZE);
+
+        rmp_serde::encode::write_named(&mut brotli_writer, self)
+            .map_err(|err| err.to_string())
+            .and_then(|_| brotli_writer.flush().map_err(|err| err.to_string()))
+            .map_err(|err| ShellError::GenericError {
+                error: "Failed to save plugin file".into(),
+                msg: "plugin file save attempted here".into(),
+                span: error_span,
+                help: Some(err.to_string()),
+                inner: vec![],
+            })
+    }
+
+    /// Insert or update a plugin in the plugin cache file.
+    pub fn upsert_plugin(&mut self, item: PluginCacheItem) {
+        if let Some(existing_item) = self.plugins.iter_mut().find(|p| p.name == item.name) {
+            *existing_item = item;
+        } else {
+            self.plugins.push(item);
+
+            // Sort the plugins for consistency
+            self.plugins
+                .sort_by(|item1, item2| item1.name.cmp(&item2.name));
+        }
+    }
+
+    /// Remove a plugin from the plugin cache file by name.
+    pub fn remove_plugin(&mut self, name: &str) {
+        self.plugins.retain_mut(|item| item.name != name)
+    }
+}
+
+/// A single plugin definition from a [`PluginCacheFile`].
+///
+/// Contains the information necessary for the [`PluginIdentity`], as well as possibly valid data
+/// about the plugin including the cached command signatures.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct PluginCacheItem {
+    /// The name of the plugin, as would show in `plugin list`. This does not include the file
+    /// extension or the `nu_plugin_` prefix.
+    pub name: String,
+
+    /// The path to the file.
+    pub filename: PathBuf,
+
+    /// The shell program used to run the plugin, if applicable.
+    pub shell: Option<PathBuf>,
+
+    /// Additional data that might be invalid so that we don't fail to load the whole plugin file
+    /// if there's a deserialization error.
+    #[serde(flatten)]
+    pub data: PluginCacheItemData,
+}
+
+impl PluginCacheItem {
+    /// Create a [`PluginCacheItem`] from an identity and signatures.
+    pub fn new(identity: &PluginIdentity, mut commands: Vec<PluginSignature>) -> PluginCacheItem {
+        // Sort the commands for consistency
+        commands.sort_by(|cmd1, cmd2| cmd1.sig.name.cmp(&cmd2.sig.name));
+
+        PluginCacheItem {
+            name: identity.name().to_owned(),
+            filename: identity.filename().to_owned(),
+            shell: identity.shell().map(|p| p.to_owned()),
+            data: PluginCacheItemData::Valid { commands },
+        }
+    }
+}
+
+/// Possibly valid data about a plugin in a [`PluginCacheFile`]. If deserialization fails, it will
+/// be `Invalid`.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(untagged)]
+pub enum PluginCacheItemData {
+    Valid {
+        /// Signatures and examples for each command provided by the plugin.
+        commands: Vec<PluginSignature>,
+    },
+    #[serde(
+        serialize_with = "serialize_invalid",
+        deserialize_with = "deserialize_invalid"
+    )]
+    Invalid,
+}
+
+fn serialize_invalid<S>(serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    ().serialize(serializer)
+}
+
+fn deserialize_invalid<'de, D>(deserializer: D) -> Result<(), D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    serde::de::IgnoredAny::deserialize(deserializer)?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests;

crates/nu-protocol/src/plugin/cache_file/tests.rs

@@ -0,0 +1,120 @@
+use super::{PluginCacheFile, PluginCacheItem, PluginCacheItemData};
+use crate::{
+    Category, PluginExample, PluginSignature, ShellError, Signature, SyntaxShape, Type, Value,
+};
+use pretty_assertions::assert_eq;
+use std::io::Cursor;
+
+fn foo_plugin() -> PluginCacheItem {
+    PluginCacheItem {
+        name: "foo".into(),
+        filename: "/path/to/nu_plugin_foo".into(),
+        shell: None,
+        data: PluginCacheItemData::Valid {
+            commands: vec![PluginSignature {
+                sig: Signature::new("foo")
+                    .input_output_type(Type::Int, Type::List(Box::new(Type::Int)))
+                    .category(Category::Experimental),
+                examples: vec![PluginExample {
+                    example: "16 | foo".into(),
+                    description: "powers of two up to 16".into(),
+                    result: Some(Value::test_list(vec![
+                        Value::test_int(2),
+                        Value::test_int(4),
+                        Value::test_int(8),
+                        Value::test_int(16),
+                    ])),
+                }],
+            }],
+        },
+    }
+}
+
+fn bar_plugin() -> PluginCacheItem {
+    PluginCacheItem {
+        name: "bar".into(),
+        filename: "/path/to/nu_plugin_bar".into(),
+        shell: None,
+        data: PluginCacheItemData::Valid {
+            commands: vec![PluginSignature {
+                sig: Signature::new("bar")
+                    .usage("overwrites files with random data")
+                    .switch("force", "ignore errors", Some('f'))
+                    .required(
+                        "path",
+                        SyntaxShape::Filepath,
+                        "file to overwrite with random data",
+                    )
+                    .category(Category::Experimental),
+                examples: vec![],
+            }],
+        },
+    }
+}
+
+#[test]
+fn roundtrip() -> Result<(), ShellError> {
+    let mut plugin_cache_file = PluginCacheFile {
+        nushell_version: env!("CARGO_PKG_VERSION").to_owned(),
+        plugins: vec![foo_plugin(), bar_plugin()],
+    };
+
+    let mut output = vec![];
+
+    plugin_cache_file.write_to(&mut output, None)?;
+
+    let read_file = PluginCacheFile::read_from(Cursor::new(&output[..]), None)?;
+
+    assert_eq!(plugin_cache_file, read_file);
+
+    Ok(())
+}
+
+#[test]
+fn roundtrip_invalid() -> Result<(), ShellError> {
+    let mut plugin_cache_file = PluginCacheFile {
+        nushell_version: env!("CARGO_PKG_VERSION").to_owned(),
+        plugins: vec![PluginCacheItem {
+            name: "invalid".into(),
+            filename: "/path/to/nu_plugin_invalid".into(),
+            shell: None,
+            data: PluginCacheItemData::Invalid,
+        }],
+    };
+
+    let mut output = vec![];
+
+    plugin_cache_file.write_to(&mut output, None)?;
+
+    let read_file = PluginCacheFile::read_from(Cursor::new(&output[..]), None)?;
+
+    assert_eq!(plugin_cache_file, read_file);
+
+    Ok(())
+}
+
+#[test]
+fn upsert_new() {
+    let mut file = PluginCacheFile::new();
+
+    file.plugins.push(foo_plugin());
+
+    file.upsert_plugin(bar_plugin());
+
+    assert_eq!(2, file.plugins.len());
+}
+
+#[test]
+fn upsert_replace() {
+    let mut file = PluginCacheFile::new();
+
+    file.plugins.push(foo_plugin());
+
+    let mut mutated_foo = foo_plugin();
+    mutated_foo.shell = Some("/bin/sh".into());
+
+    file.upsert_plugin(mutated_foo);
+
+    assert_eq!(1, file.plugins.len());
+    assert_eq!(Some("/bin/sh".into()), file.plugins[0].shell);
+}

crates/nu-protocol/src/plugin/identity.rs

@@ -88,6 +88,19 @@ impl PluginIdentity {
         PluginIdentity::new(format!(r"/fake/path/nu_plugin_{name}"), None)
             .expect("fake plugin identity path is invalid")
     }
+
+    /// A command that could be used to register the plugin, for suggesting in errors.
+    pub fn register_command(&self) -> String {
+        if let Some(shell) = self.shell() {
+            format!(
+                "register --shell '{}' '{}'",
+                shell.display(),
+                self.filename().display(),
+            )
+        } else {
+            format!("register '{}'", self.filename().display())
+        }
+    }
 }
 
 #[test]

crates/nu-protocol/src/plugin/mod.rs

@@ -1,7 +1,9 @@
+mod cache_file;
 mod identity;
 mod registered;
 mod signature;
 
+pub use cache_file::*;
 pub use identity::*;
 pub use registered::*;
 pub use signature::*;

crates/nu-protocol/src/plugin/signature.rs

@@ -2,7 +2,7 @@ use crate::{PluginExample, Signature};
 use serde::{Deserialize, Serialize};
 
 /// A simple wrapper for Signature that includes examples.
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct PluginSignature {
     pub sig: Signature,
     pub examples: Vec<PluginExample>,