Overhaul the plugin cache file with a new msgpack+brotli format (#12579)

# Description

- Plugin signatures are now saved to `plugin.msgpackz`, which is
brotli-compressed MessagePack.
- The file is updated incrementally, rather than writing all plugin
commands in the engine every time.
- The file always contains the result of the `Signature` call to the
plugin, even if commands were removed.
- Invalid data for a particular plugin just causes an error to be
reported, but the rest of the plugins can still be parsed

# User-Facing Changes

- The plugin file has a different filename, and it's not a nushell
script.
- The default `plugin.nu` file will be automatically migrated the first
time, but not other plugin config files.
- We don't currently provide any utilities that could help edit this
file, beyond `plugin add` and `plugin rm`
  - `from msgpackz`, `to msgpackz` could also help
- New commands: `plugin add`, `plugin rm`

# Tests + Formatting

Tests added for the format and for the invalid handling.

- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

# After Submitting

- [ ] Check for documentation changes
- [ ] Definitely needs release notes
This commit is contained in:
Devyn Cairns
2024-04-21 05:36:26 -07:00
committed by GitHub
parent 6cba7c6b40
commit 2595f31541
45 changed files with 1462 additions and 211 deletions

View File

@ -24,7 +24,7 @@ use std::{
type PoisonDebuggerError<'a> = PoisonError<MutexGuard<'a, Box<dyn Debugger>>>;
#[cfg(feature = "plugin")]
use crate::RegisteredPlugin;
use crate::{PluginCacheFile, PluginCacheItem, RegisteredPlugin};
pub static PWD_ENV: &str = "PWD";
@ -92,7 +92,7 @@ pub struct EngineState {
pub repl_state: Arc<Mutex<ReplState>>,
pub table_decl_id: Option<usize>,
#[cfg(feature = "plugin")]
pub plugin_signatures: Option<PathBuf>,
pub plugin_path: Option<PathBuf>,
#[cfg(feature = "plugin")]
plugins: Vec<Arc<dyn RegisteredPlugin>>,
config_path: HashMap<String, PathBuf>,
@ -155,7 +155,7 @@ impl EngineState {
})),
table_decl_id: None,
#[cfg(feature = "plugin")]
plugin_signatures: None,
plugin_path: None,
#[cfg(feature = "plugin")]
plugins: vec![],
config_path: HashMap::new(),
@ -255,7 +255,7 @@ impl EngineState {
if let Some(existing) = self
.plugins
.iter_mut()
.find(|p| p.identity() == plugin.identity())
.find(|p| p.identity().name() == plugin.identity().name())
{
// Stop the existing plugin, so that the new plugin definitely takes over
existing.stop()?;
@ -267,10 +267,10 @@ impl EngineState {
}
#[cfg(feature = "plugin")]
if delta.plugins_changed {
if !delta.plugin_cache_items.is_empty() {
// Update the plugin file with the new signatures.
if self.plugin_signatures.is_some() {
self.update_plugin_file()?;
if self.plugin_path.is_some() {
self.update_plugin_file(std::mem::take(&mut delta.plugin_cache_items))?;
}
}
@ -480,93 +480,58 @@ impl EngineState {
}
#[cfg(feature = "plugin")]
pub fn update_plugin_file(&self) -> Result<(), ShellError> {
use std::io::Write;
use crate::{PluginExample, PluginSignature};
pub fn update_plugin_file(
&self,
updated_items: Vec<PluginCacheItem>,
) -> Result<(), ShellError> {
// Updating the signatures plugin file with the added signatures
self.plugin_signatures
use std::fs::File;
let plugin_path = self
.plugin_path
.as_ref()
.ok_or_else(|| ShellError::PluginFailedToLoad {
msg: "Plugin file not found".into(),
})
.and_then(|plugin_path| {
// Always create the file, which will erase previous signatures
std::fs::File::create(plugin_path.as_path()).map_err(|err| {
ShellError::PluginFailedToLoad {
msg: err.to_string(),
}
})
})
.and_then(|mut plugin_file| {
// Plugin definitions with parsed signature
self.plugin_decls().try_for_each(|decl| {
// A successful plugin registration already includes the plugin filename
// No need to check the None option
let identity = decl.plugin_identity().expect("plugin should have identity");
let mut file_name = identity
.filename()
.to_str()
.expect("path was checked during registration as a str")
.to_string();
.ok_or_else(|| ShellError::GenericError {
error: "Plugin file path not set".into(),
msg: "".into(),
span: None,
help: Some("you may be running nu with --no-config-file".into()),
inner: vec![],
})?;
// Fix files or folders with quotes
if file_name.contains('\'')
|| file_name.contains('"')
|| file_name.contains(' ')
{
file_name = format!("`{file_name}`");
}
// Read the current contents of the plugin file if it exists
let mut contents = match File::open(plugin_path.as_path()) {
Ok(mut plugin_file) => PluginCacheFile::read_from(&mut plugin_file, None),
Err(err) => {
if err.kind() == std::io::ErrorKind::NotFound {
Ok(PluginCacheFile::default())
} else {
Err(ShellError::GenericError {
error: "Failed to open plugin file".into(),
msg: "".into(),
span: None,
help: None,
inner: vec![err.into()],
})
}
}
}?;
let sig = decl.signature();
let examples = decl
.examples()
.into_iter()
.map(PluginExample::from)
.collect();
let sig_with_examples = PluginSignature::new(sig, examples);
serde_json::to_string_pretty(&sig_with_examples)
.map(|signature| {
// Extracting the possible path to the shell used to load the plugin
let shell_str = identity
.shell()
.map(|path| {
format!(
"-s {}",
path.to_str().expect(
"shell path was checked during registration as a str"
)
)
})
.unwrap_or_default();
// Update the given signatures
for item in updated_items {
contents.upsert_plugin(item);
}
// Each signature is stored in the plugin file with the shell and signature
// This information will be used when loading the plugin
// information when nushell starts
format!("register {file_name} {shell_str} {signature}\n\n")
})
.map_err(|err| ShellError::PluginFailedToLoad {
msg: err.to_string(),
})
.and_then(|line| {
plugin_file.write_all(line.as_bytes()).map_err(|err| {
ShellError::PluginFailedToLoad {
msg: err.to_string(),
}
})
})
.and_then(|_| {
plugin_file.flush().map_err(|err| ShellError::GenericError {
error: "Error flushing plugin file".into(),
msg: format! {"{err}"},
span: None,
help: None,
inner: vec![],
})
})
})
})
// Write it to the same path
let plugin_file =
File::create(plugin_path.as_path()).map_err(|err| ShellError::GenericError {
error: "Failed to write plugin file".into(),
msg: "".into(),
span: None,
help: None,
inner: vec![err.into()],
})?;
contents.write_to(plugin_file, None)
}
/// Update plugins with new garbage collection config

View File

@ -9,7 +9,7 @@ use crate::{
use std::sync::Arc;
#[cfg(feature = "plugin")]
use crate::RegisteredPlugin;
use crate::{PluginCacheItem, RegisteredPlugin};
/// A delta (or change set) between the current global state and a possible future global state. Deltas
/// can be applied to the global state to update it to contain both previous state and the state held
@ -24,9 +24,9 @@ pub struct StateDelta {
pub(super) usage: Usage,
pub scope: Vec<ScopeFrame>,
#[cfg(feature = "plugin")]
pub(super) plugins_changed: bool, // marks whether plugin file should be updated
#[cfg(feature = "plugin")]
pub(super) plugins: Vec<Arc<dyn RegisteredPlugin>>,
#[cfg(feature = "plugin")]
pub(super) plugin_cache_items: Vec<PluginCacheItem>,
}
impl StateDelta {
@ -48,9 +48,9 @@ impl StateDelta {
scope: vec![scope_frame],
usage: Usage::new(),
#[cfg(feature = "plugin")]
plugins_changed: false,
#[cfg(feature = "plugin")]
plugins: vec![],
#[cfg(feature = "plugin")]
plugin_cache_items: vec![],
}
}

View File

@ -15,7 +15,7 @@ use std::{
};
#[cfg(feature = "plugin")]
use crate::{PluginIdentity, RegisteredPlugin};
use crate::{PluginCacheItem, PluginIdentity, RegisteredPlugin};
/// A temporary extension to the global state. This handles bridging between the global state and the
/// additional declarations and scope changes that are not yet part of the global scope.
@ -159,11 +159,6 @@ impl<'a> StateWorkingSet<'a> {
.insert(name, decl_id)
}
#[cfg(feature = "plugin")]
pub fn mark_plugins_file_dirty(&mut self) {
self.delta.plugins_changed = true;
}
#[cfg(feature = "plugin")]
pub fn find_or_create_plugin(
&mut self,
@ -186,6 +181,11 @@ impl<'a> StateWorkingSet<'a> {
}
}
#[cfg(feature = "plugin")]
pub fn update_plugin_cache(&mut self, item: PluginCacheItem) {
self.delta.plugin_cache_items.push(item);
}
pub fn merge_predecl(&mut self, name: &[u8]) -> Option<DeclId> {
self.move_predecls_to_overlay();

View File

@ -750,6 +750,19 @@ pub enum ShellError {
span: Span,
},
/// The cached plugin data (in `$nu.plugin-path`) for a plugin is invalid.
///
/// ## Resolution
///
/// `register` the plugin again to update the data, or remove it.
#[error("The cached plugin data for `{plugin_name}` is invalid")]
#[diagnostic(code(nu::shell::plugin_cache_data_invalid))]
PluginCacheDataInvalid {
plugin_name: String,
#[help("try registering the plugin again with `{}`")]
register_command: String,
},
/// A plugin failed to load.
///
/// ## Resolution

View File

@ -116,7 +116,7 @@ pub fn create_nu_constant(engine_state: &EngineState, span: Span) -> Result<Valu
{
record.push(
"plugin-path",
if let Some(path) = &engine_state.plugin_signatures {
if let Some(path) = &engine_state.plugin_path {
let canon_plugin_path = canonicalize_path(engine_state, path);
Value::string(canon_plugin_path.to_string_lossy(), span)
} else {
@ -124,7 +124,7 @@ pub fn create_nu_constant(engine_state: &EngineState, span: Span) -> Result<Valu
config_path.clone().map_or_else(
|e| e,
|mut path| {
path.push("plugin.nu");
path.push("plugin.msgpackz");
let canonical_plugin_path = canonicalize_path(engine_state, &path);
Value::string(canonical_plugin_path.to_string_lossy(), span)
},

View File

@ -13,7 +13,7 @@ pub struct Example<'a> {
// and `description` fields, because these information is fetched from plugin, a third party
// binary, nushell have no way to construct it directly.
#[cfg(feature = "plugin")]
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PluginExample {
pub example: String,
pub description: String,

View File

@ -0,0 +1,175 @@
use std::{
io::{Read, Write},
path::PathBuf,
};
use serde::{Deserialize, Serialize};
use crate::{PluginIdentity, PluginSignature, ShellError, Span};
// This has a big impact on performance
const BUFFER_SIZE: usize = 65536;
// Chose settings at the low end, because we're just trying to get the maximum speed
const COMPRESSION_QUALITY: u32 = 1;
const WIN_SIZE: u32 = 20; // recommended 20-22
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PluginCacheFile {
/// The Nushell version that last updated the file.
pub nushell_version: String,
/// The installed plugins.
pub plugins: Vec<PluginCacheItem>,
}
impl Default for PluginCacheFile {
fn default() -> Self {
Self::new()
}
}
impl PluginCacheFile {
/// Create a new, empty plugin cache file.
pub fn new() -> PluginCacheFile {
PluginCacheFile {
nushell_version: env!("CARGO_PKG_VERSION").to_owned(),
plugins: vec![],
}
}
/// Read the plugin cache file from a reader, e.g. [`File`](std::fs::File).
pub fn read_from(
reader: impl Read,
error_span: Option<Span>,
) -> Result<PluginCacheFile, ShellError> {
// Format is brotli compressed messagepack
let brotli_reader = brotli::Decompressor::new(reader, BUFFER_SIZE);
rmp_serde::from_read(brotli_reader).map_err(|err| ShellError::GenericError {
error: format!("Failed to load plugin file: {err}"),
msg: "plugin file load attempted here".into(),
span: error_span,
help: Some(
"it may be corrupt. Try deleting it and registering your plugins again".into(),
),
inner: vec![],
})
}
/// Write the plugin cache file to a writer, e.g. [`File`](std::fs::File).
///
/// The `nushell_version` will be updated to the current version before writing.
pub fn write_to(
&mut self,
writer: impl Write,
error_span: Option<Span>,
) -> Result<(), ShellError> {
// Update the Nushell version before writing
self.nushell_version = env!("CARGO_PKG_VERSION").to_owned();
// Format is brotli compressed messagepack
let mut brotli_writer =
brotli::CompressorWriter::new(writer, BUFFER_SIZE, COMPRESSION_QUALITY, WIN_SIZE);
rmp_serde::encode::write_named(&mut brotli_writer, self)
.map_err(|err| err.to_string())
.and_then(|_| brotli_writer.flush().map_err(|err| err.to_string()))
.map_err(|err| ShellError::GenericError {
error: "Failed to save plugin file".into(),
msg: "plugin file save attempted here".into(),
span: error_span,
help: Some(err.to_string()),
inner: vec![],
})
}
/// Insert or update a plugin in the plugin cache file.
pub fn upsert_plugin(&mut self, item: PluginCacheItem) {
if let Some(existing_item) = self.plugins.iter_mut().find(|p| p.name == item.name) {
*existing_item = item;
} else {
self.plugins.push(item);
// Sort the plugins for consistency
self.plugins
.sort_by(|item1, item2| item1.name.cmp(&item2.name));
}
}
/// Remove a plugin from the plugin cache file by name.
pub fn remove_plugin(&mut self, name: &str) {
self.plugins.retain_mut(|item| item.name != name)
}
}
/// A single plugin definition from a [`PluginCacheFile`].
///
/// Contains the information necessary for the [`PluginIdentity`], as well as possibly valid data
/// about the plugin including the cached command signatures.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PluginCacheItem {
/// The name of the plugin, as would show in `plugin list`. This does not include the file
/// extension or the `nu_plugin_` prefix.
pub name: String,
/// The path to the file.
pub filename: PathBuf,
/// The shell program used to run the plugin, if applicable.
pub shell: Option<PathBuf>,
/// Additional data that might be invalid so that we don't fail to load the whole plugin file
/// if there's a deserialization error.
#[serde(flatten)]
pub data: PluginCacheItemData,
}
impl PluginCacheItem {
/// Create a [`PluginCacheItem`] from an identity and signatures.
pub fn new(identity: &PluginIdentity, mut commands: Vec<PluginSignature>) -> PluginCacheItem {
// Sort the commands for consistency
commands.sort_by(|cmd1, cmd2| cmd1.sig.name.cmp(&cmd2.sig.name));
PluginCacheItem {
name: identity.name().to_owned(),
filename: identity.filename().to_owned(),
shell: identity.shell().map(|p| p.to_owned()),
data: PluginCacheItemData::Valid { commands },
}
}
}
/// Possibly valid data about a plugin in a [`PluginCacheFile`]. If deserialization fails, it will
/// be `Invalid`.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(untagged)]
pub enum PluginCacheItemData {
Valid {
/// Signatures and examples for each command provided by the plugin.
commands: Vec<PluginSignature>,
},
#[serde(
serialize_with = "serialize_invalid",
deserialize_with = "deserialize_invalid"
)]
Invalid,
}
fn serialize_invalid<S>(serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
().serialize(serializer)
}
fn deserialize_invalid<'de, D>(deserializer: D) -> Result<(), D::Error>
where
D: serde::Deserializer<'de>,
{
serde::de::IgnoredAny::deserialize(deserializer)?;
Ok(())
}
#[cfg(test)]
mod tests;

View File

@ -0,0 +1,120 @@
use super::{PluginCacheFile, PluginCacheItem, PluginCacheItemData};
use crate::{
Category, PluginExample, PluginSignature, ShellError, Signature, SyntaxShape, Type, Value,
};
use pretty_assertions::assert_eq;
use std::io::Cursor;
fn foo_plugin() -> PluginCacheItem {
PluginCacheItem {
name: "foo".into(),
filename: "/path/to/nu_plugin_foo".into(),
shell: None,
data: PluginCacheItemData::Valid {
commands: vec![PluginSignature {
sig: Signature::new("foo")
.input_output_type(Type::Int, Type::List(Box::new(Type::Int)))
.category(Category::Experimental),
examples: vec![PluginExample {
example: "16 | foo".into(),
description: "powers of two up to 16".into(),
result: Some(Value::test_list(vec![
Value::test_int(2),
Value::test_int(4),
Value::test_int(8),
Value::test_int(16),
])),
}],
}],
},
}
}
fn bar_plugin() -> PluginCacheItem {
PluginCacheItem {
name: "bar".into(),
filename: "/path/to/nu_plugin_bar".into(),
shell: None,
data: PluginCacheItemData::Valid {
commands: vec![PluginSignature {
sig: Signature::new("bar")
.usage("overwrites files with random data")
.switch("force", "ignore errors", Some('f'))
.required(
"path",
SyntaxShape::Filepath,
"file to overwrite with random data",
)
.category(Category::Experimental),
examples: vec![],
}],
},
}
}
#[test]
fn roundtrip() -> Result<(), ShellError> {
let mut plugin_cache_file = PluginCacheFile {
nushell_version: env!("CARGO_PKG_VERSION").to_owned(),
plugins: vec![foo_plugin(), bar_plugin()],
};
let mut output = vec![];
plugin_cache_file.write_to(&mut output, None)?;
let read_file = PluginCacheFile::read_from(Cursor::new(&output[..]), None)?;
assert_eq!(plugin_cache_file, read_file);
Ok(())
}
#[test]
fn roundtrip_invalid() -> Result<(), ShellError> {
let mut plugin_cache_file = PluginCacheFile {
nushell_version: env!("CARGO_PKG_VERSION").to_owned(),
plugins: vec![PluginCacheItem {
name: "invalid".into(),
filename: "/path/to/nu_plugin_invalid".into(),
shell: None,
data: PluginCacheItemData::Invalid,
}],
};
let mut output = vec![];
plugin_cache_file.write_to(&mut output, None)?;
let read_file = PluginCacheFile::read_from(Cursor::new(&output[..]), None)?;
assert_eq!(plugin_cache_file, read_file);
Ok(())
}
#[test]
fn upsert_new() {
let mut file = PluginCacheFile::new();
file.plugins.push(foo_plugin());
file.upsert_plugin(bar_plugin());
assert_eq!(2, file.plugins.len());
}
#[test]
fn upsert_replace() {
let mut file = PluginCacheFile::new();
file.plugins.push(foo_plugin());
let mut mutated_foo = foo_plugin();
mutated_foo.shell = Some("/bin/sh".into());
file.upsert_plugin(mutated_foo);
assert_eq!(1, file.plugins.len());
assert_eq!(Some("/bin/sh".into()), file.plugins[0].shell);
}

View File

@ -88,6 +88,19 @@ impl PluginIdentity {
PluginIdentity::new(format!(r"/fake/path/nu_plugin_{name}"), None)
.expect("fake plugin identity path is invalid")
}
/// A command that could be used to register the plugin, for suggesting in errors.
pub fn register_command(&self) -> String {
if let Some(shell) = self.shell() {
format!(
"register --shell '{}' '{}'",
shell.display(),
self.filename().display(),
)
} else {
format!("register '{}'", self.filename().display())
}
}
}
#[test]

View File

@ -1,7 +1,9 @@
mod cache_file;
mod identity;
mod registered;
mod signature;
pub use cache_file::*;
pub use identity::*;
pub use registered::*;
pub use signature::*;

View File

@ -2,7 +2,7 @@ use crate::{PluginExample, Signature};
use serde::{Deserialize, Serialize};
/// A simple wrapper for Signature that includes examples.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PluginSignature {
pub sig: Signature,
pub examples: Vec<PluginExample>,

View File

@ -56,6 +56,7 @@ pub enum Category {
Network,
Path,
Platform,
Plugin,
Random,
Shells,
Strings,
@ -90,6 +91,7 @@ impl std::fmt::Display for Category {
Category::Network => "network",
Category::Path => "path",
Category::Platform => "platform",
Category::Plugin => "plugin",
Category::Random => "random",
Category::Shells => "shells",
Category::Strings => "strings",