Overhaul the plugin cache file with a new msgpack+brotli format (#12579)

# Description

- Plugin signatures are now saved to `plugin.msgpackz`, which is
brotli-compressed MessagePack.
- The file is updated incrementally, rather than writing all plugin
commands in the engine every time.
- The file always contains the result of the `Signature` call to the
plugin, even if commands were removed.
- Invalid data for a particular plugin just causes an error to be
reported, but the rest of the plugins can still be parsed

# User-Facing Changes

- The plugin file has a different filename, and it's not a nushell
script.
- The default `plugin.nu` file will be automatically migrated the first
time, but not other plugin config files.
- We don't currently provide any utilities that could help edit this
file, beyond `plugin add` and `plugin rm`
  - `from msgpackz`, `to msgpackz` could also help
- New commands: `plugin add`, `plugin rm`

# Tests + Formatting

Tests added for the format and for the invalid handling.

- 🟢 `toolkit fmt`
- 🟢 `toolkit clippy`
- 🟢 `toolkit test`
- 🟢 `toolkit test stdlib`

# After Submitting

- [ ] Check for documentation changes
- [ ] Definitely needs release notes
This commit is contained in:
Devyn Cairns
2024-04-21 05:36:26 -07:00
committed by GitHub
parent 6cba7c6b40
commit 2595f31541
45 changed files with 1462 additions and 211 deletions

View File

@ -0,0 +1,5 @@
mod plugin;
mod register;
pub use plugin::*;
pub use register::Register;

View File

@ -0,0 +1,148 @@
use std::sync::Arc;
use nu_engine::{command_prelude::*, current_dir};
use nu_plugin::{GetPlugin, PersistentPlugin};
use nu_protocol::{PluginCacheItem, PluginGcConfig, PluginIdentity, RegisteredPlugin};
use crate::util::modify_plugin_file;
#[derive(Clone)]
pub struct PluginAdd;
impl Command for PluginAdd {
fn name(&self) -> &str {
"plugin add"
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(Type::Nothing, Type::Nothing)
// This matches the option to `nu`
.named(
"plugin-config",
SyntaxShape::Filepath,
"Use a plugin cache file other than the one set in `$nu.plugin-path`",
None,
)
.named(
"shell",
SyntaxShape::Filepath,
"Use an additional shell program (cmd, sh, python, etc.) to run the plugin",
Some('s'),
)
.required(
"filename",
SyntaxShape::Filepath,
"Path to the executable for the plugin",
)
.category(Category::Plugin)
}
fn usage(&self) -> &str {
"Add a plugin to the plugin cache file."
}
fn extra_usage(&self) -> &str {
r#"
This does not load the plugin commands into the scope - see `register` for that.
Instead, it runs the plugin to get its command signatures, and then edits the
plugin cache file (by default, `$nu.plugin-path`). The changes will be
apparent the next time `nu` is next launched with that plugin cache file.
"#
.trim()
}
fn search_terms(&self) -> Vec<&str> {
vec!["plugin", "add", "register", "load", "signature"]
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
example: "plugin add nu_plugin_inc",
description: "Run the `nu_plugin_inc` plugin from the current directory or $env.NU_PLUGIN_DIRS and install its signatures.",
result: None,
},
Example {
example: "plugin add --plugin-config polars.msgpackz nu_plugin_polars",
description: "Run the `nu_plugin_polars` plugin from the current directory or $env.NU_PLUGIN_DIRS, and install its signatures to the \"polars.msgpackz\" plugin cache file.",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let filename: Spanned<String> = call.req(engine_state, stack, 0)?;
let shell: Option<Spanned<String>> = call.get_flag(engine_state, stack, "shell")?;
let cwd = current_dir(engine_state, stack)?;
// Check the current directory, or fall back to NU_PLUGIN_DIRS
let filename_expanded = match nu_path::canonicalize_with(&filename.item, &cwd) {
Ok(path) => path,
Err(err) => {
// Try to find it in NU_PLUGIN_DIRS first, before giving up
let mut found = None;
if let Some(nu_plugin_dirs) = stack.get_env_var(engine_state, "NU_PLUGIN_DIRS") {
for dir in nu_plugin_dirs.into_list().unwrap_or(vec![]) {
if let Ok(path) = nu_path::canonicalize_with(dir.as_str()?, &cwd)
.and_then(|dir| nu_path::canonicalize_with(&filename.item, dir))
{
found = Some(path);
break;
}
}
}
found.ok_or(err.into_spanned(filename.span))?
}
};
let shell_expanded = shell
.as_ref()
.map(|s| {
nu_path::canonicalize_with(&s.item, &cwd).map_err(|err| err.into_spanned(s.span))
})
.transpose()?;
// Parse the plugin filename so it can be used to spawn the plugin
let identity = PluginIdentity::new(filename_expanded, shell_expanded).map_err(|_| {
ShellError::GenericError {
error: "Plugin filename is invalid".into(),
msg: "plugin executable files must start with `nu_plugin_`".into(),
span: Some(filename.span),
help: None,
inner: vec![],
}
})?;
let custom_path = call.get_flag(engine_state, stack, "plugin-config")?;
// Start the plugin manually, to get the freshest signatures and to not affect engine
// state. Provide a GC config that will stop it ASAP
let plugin = Arc::new(PersistentPlugin::new(
identity,
PluginGcConfig {
enabled: true,
stop_after: 0,
},
));
let interface = plugin.clone().get_plugin(Some((engine_state, stack)))?;
let commands = interface.get_signature()?;
modify_plugin_file(engine_state, stack, call.head, custom_path, |contents| {
// Update the file with the received signatures
let item = PluginCacheItem::new(plugin.identity(), commands);
contents.upsert_plugin(item);
Ok(())
})?;
Ok(Value::nothing(call.head).into_pipeline_data())
}
}

View File

@ -0,0 +1,96 @@
use itertools::Itertools;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct PluginList;
impl Command for PluginList {
fn name(&self) -> &str {
"plugin list"
}
fn signature(&self) -> Signature {
Signature::build("plugin list")
.input_output_type(
Type::Nothing,
Type::Table(vec![
("name".into(), Type::String),
("is_running".into(), Type::Bool),
("pid".into(), Type::Int),
("filename".into(), Type::String),
("shell".into(), Type::String),
("commands".into(), Type::List(Type::String.into())),
]),
)
.category(Category::Plugin)
}
fn usage(&self) -> &str {
"List installed plugins."
}
fn examples(&self) -> Vec<nu_protocol::Example> {
vec![
Example {
example: "plugin list",
description: "List installed plugins.",
result: Some(Value::test_list(vec![Value::test_record(record! {
"name" => Value::test_string("inc"),
"is_running" => Value::test_bool(true),
"pid" => Value::test_int(106480),
"filename" => if cfg!(windows) {
Value::test_string(r"C:\nu\plugins\nu_plugin_inc.exe")
} else {
Value::test_string("/opt/nu/plugins/nu_plugin_inc")
},
"shell" => Value::test_nothing(),
"commands" => Value::test_list(vec![Value::test_string("inc")]),
})])),
},
Example {
example: "ps | where pid in (plugin list).pid",
description: "Get process information for running plugins.",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let span = call.span();
// Group plugin decls by plugin identity
let decls = engine_state.plugin_decls().into_group_map_by(|decl| {
decl.plugin_identity()
.expect("plugin decl should have identity")
});
// Build plugins list
let list = engine_state.plugins().iter().map(|plugin| {
// Find commands that belong to the plugin
let commands = decls.get(plugin.identity())
.into_iter()
.flat_map(|decls| {
decls.iter().map(|decl| Value::string(decl.name(), span))
})
.collect();
Value::record(record! {
"name" => Value::string(plugin.identity().name(), span),
"is_running" => Value::bool(plugin.is_running(), span),
"pid" => plugin.pid()
.map(|p| Value::int(p as i64, span))
.unwrap_or(Value::nothing(span)),
"filename" => Value::string(plugin.identity().filename().to_string_lossy(), span),
"shell" => plugin.identity().shell()
.map(|s| Value::string(s.to_string_lossy(), span))
.unwrap_or(Value::nothing(span)),
"commands" => Value::list(commands, span),
}, span)
}).collect::<Vec<Value>>();
Ok(list.into_pipeline_data(engine_state.ctrlc.clone()))
}
}

View File

@ -0,0 +1,79 @@
use nu_engine::{command_prelude::*, get_full_help};
mod add;
mod list;
mod rm;
mod stop;
pub use add::PluginAdd;
pub use list::PluginList;
pub use rm::PluginRm;
pub use stop::PluginStop;
#[derive(Clone)]
pub struct PluginCommand;
impl Command for PluginCommand {
fn name(&self) -> &str {
"plugin"
}
fn signature(&self) -> Signature {
Signature::build("plugin")
.input_output_types(vec![(Type::Nothing, Type::Nothing)])
.category(Category::Plugin)
}
fn usage(&self) -> &str {
"Commands for managing plugins."
}
fn extra_usage(&self) -> &str {
"To load a plugin, see `register`."
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
Ok(Value::string(
get_full_help(
&PluginCommand.signature(),
&PluginCommand.examples(),
engine_state,
stack,
self.is_parser_keyword(),
),
call.head,
)
.into_pipeline_data())
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
example: "plugin list",
description: "List installed plugins",
result: None,
},
Example {
example: "plugin stop inc",
description: "Stop the plugin named `inc`.",
result: None,
},
Example {
example: "plugin add nu_plugin_inc",
description: "Run the `nu_plugin_inc` plugin from the current directory and install its signatures.",
result: None,
},
Example {
example: "plugin rm inc",
description: "Remove the installed signatures for the `inc` plugin.",
result: None,
},
]
}
}

View File

@ -0,0 +1,100 @@
use nu_engine::command_prelude::*;
use crate::util::modify_plugin_file;
#[derive(Clone)]
pub struct PluginRm;
impl Command for PluginRm {
fn name(&self) -> &str {
"plugin rm"
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(Type::Nothing, Type::Nothing)
// This matches the option to `nu`
.named(
"plugin-config",
SyntaxShape::Filepath,
"Use a plugin cache file other than the one set in `$nu.plugin-path`",
None,
)
.switch(
"force",
"Don't cause an error if the plugin name wasn't found in the file",
Some('f'),
)
.required(
"name",
SyntaxShape::String,
"The name of the plugin to remove (not the filename)",
)
.category(Category::Plugin)
}
fn usage(&self) -> &str {
"Remove a plugin from the plugin cache file."
}
fn extra_usage(&self) -> &str {
r#"
This does not remove the plugin commands from the current scope or from `plugin
list` in the current shell. It instead removes the plugin from the plugin
cache file (by default, `$nu.plugin-path`). The changes will be apparent the
next time `nu` is launched with that plugin cache file.
This can be useful for removing an invalid plugin signature, if it can't be
fixed with `plugin add`.
"#
.trim()
}
fn search_terms(&self) -> Vec<&str> {
vec!["plugin", "rm", "remove", "delete", "signature"]
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
example: "plugin rm inc",
description: "Remove the installed signatures for the `inc` plugin.",
result: None,
},
Example {
example: "plugin rm --plugin-config polars.msgpackz polars",
description: "Remove the installed signatures for the `polars` plugin from the \"polars.msgpackz\" plugin cache file.",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let name: Spanned<String> = call.req(engine_state, stack, 0)?;
let custom_path = call.get_flag(engine_state, stack, "plugin-config")?;
let force = call.has_flag(engine_state, stack, "force")?;
modify_plugin_file(engine_state, stack, call.head, custom_path, |contents| {
if !force && !contents.plugins.iter().any(|p| p.name == name.item) {
Err(ShellError::GenericError {
error: format!("Failed to remove the `{}` plugin", name.item),
msg: "couldn't find a plugin with this name in the cache file".into(),
span: Some(name.span),
help: None,
inner: vec![],
})
} else {
contents.remove_plugin(&name.item);
Ok(())
}
})?;
Ok(Value::nothing(call.head).into_pipeline_data())
}
}

View File

@ -0,0 +1,70 @@
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct PluginStop;
impl Command for PluginStop {
fn name(&self) -> &str {
"plugin stop"
}
fn signature(&self) -> Signature {
Signature::build("plugin stop")
.input_output_type(Type::Nothing, Type::Nothing)
.required(
"name",
SyntaxShape::String,
"The name of the plugin to stop.",
)
.category(Category::Plugin)
}
fn usage(&self) -> &str {
"Stop an installed plugin if it was running."
}
fn examples(&self) -> Vec<nu_protocol::Example> {
vec![
Example {
example: "plugin stop inc",
description: "Stop the plugin named `inc`.",
result: None,
},
Example {
example: "plugin list | each { |p| plugin stop $p.name }",
description: "Stop all plugins.",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let name: Spanned<String> = call.req(engine_state, stack, 0)?;
let mut found = false;
for plugin in engine_state.plugins() {
if plugin.identity().name() == name.item {
plugin.stop()?;
found = true;
}
}
if found {
Ok(PipelineData::Empty)
} else {
Err(ShellError::GenericError {
error: format!("Failed to stop the `{}` plugin", name.item),
msg: "couldn't find a plugin with this name".into(),
span: Some(name.span),
help: Some("you may need to `register` the plugin first".into()),
inner: vec![],
})
}
}
}

View File

@ -0,0 +1,74 @@
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct Register;
impl Command for Register {
fn name(&self) -> &str {
"register"
}
fn usage(&self) -> &str {
"Register a plugin."
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build("register")
.input_output_types(vec![(Type::Nothing, Type::Nothing)])
.required(
"plugin",
SyntaxShape::Filepath,
"Path of executable for plugin.",
)
.optional(
"signature",
SyntaxShape::Any,
"Block with signature description as json object.",
)
.named(
"shell",
SyntaxShape::Filepath,
"path of shell used to run plugin (cmd, sh, python, etc)",
Some('s'),
)
.category(Category::Plugin)
}
fn extra_usage(&self) -> &str {
r#"This command is a parser keyword. For details, check:
https://www.nushell.sh/book/thinking_in_nu.html"#
}
fn search_terms(&self) -> Vec<&str> {
vec!["plugin", "add", "register"]
}
fn is_parser_keyword(&self) -> bool {
true
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
_call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
Ok(PipelineData::empty())
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Register `nu_plugin_query` plugin from ~/.cargo/bin/ dir",
example: r#"register ~/.cargo/bin/nu_plugin_query"#,
result: None,
},
Example {
description: "Register `nu_plugin_query` plugin from `nu -c` (writes/updates $nu.plugin-path)",
example: r#"let plugin = ((which nu).path.0 | path dirname | path join 'nu_plugin_query'); nu -c $'register ($plugin); version'"#,
result: None,
},
]
}
}

View File

@ -0,0 +1,31 @@
use crate::*;
use nu_protocol::engine::{EngineState, StateWorkingSet};
pub fn add_plugin_command_context(mut engine_state: EngineState) -> EngineState {
let delta = {
let mut working_set = StateWorkingSet::new(&engine_state);
macro_rules! bind_command {
( $( $command:expr ),* $(,)? ) => {
$( working_set.add_decl(Box::new($command)); )*
};
}
bind_command!(
PluginCommand,
PluginAdd,
PluginList,
PluginRm,
PluginStop,
Register,
);
working_set.render()
};
if let Err(err) = engine_state.merge_delta(delta) {
eprintln!("Error creating default context: {err:?}");
}
engine_state
}

View File

@ -0,0 +1,8 @@
//! Nushell commands for managing plugins.
mod commands;
mod default_context;
mod util;
pub use commands::*;
pub use default_context::*;

View File

@ -0,0 +1,50 @@
use std::fs::{self, File};
use nu_engine::{command_prelude::*, current_dir};
use nu_protocol::PluginCacheFile;
pub(crate) fn modify_plugin_file(
engine_state: &EngineState,
stack: &mut Stack,
span: Span,
custom_path: Option<Spanned<String>>,
operate: impl FnOnce(&mut PluginCacheFile) -> Result<(), ShellError>,
) -> Result<(), ShellError> {
let cwd = current_dir(engine_state, stack)?;
let plugin_cache_file_path = if let Some(ref custom_path) = custom_path {
nu_path::expand_path_with(&custom_path.item, cwd, true)
} else {
engine_state
.plugin_path
.clone()
.ok_or_else(|| ShellError::GenericError {
error: "Plugin cache file not set".into(),
msg: "pass --plugin-config explicitly here".into(),
span: Some(span),
help: Some("you may be running `nu` with --no-config-file".into()),
inner: vec![],
})?
};
// Try to read the plugin file if it exists
let mut contents = if fs::metadata(&plugin_cache_file_path).is_ok_and(|m| m.len() > 0) {
PluginCacheFile::read_from(
File::open(&plugin_cache_file_path).map_err(|err| err.into_spanned(span))?,
Some(span),
)?
} else {
PluginCacheFile::default()
};
// Do the operation
operate(&mut contents)?;
// Save the modified file on success
contents.write_to(
File::create(&plugin_cache_file_path).map_err(|err| err.into_spanned(span))?,
Some(span),
)?;
Ok(())
}