forked from extern/nushell
Overhaul the plugin cache file with a new msgpack+brotli format (#12579)
# Description - Plugin signatures are now saved to `plugin.msgpackz`, which is brotli-compressed MessagePack. - The file is updated incrementally, rather than writing all plugin commands in the engine every time. - The file always contains the result of the `Signature` call to the plugin, even if commands were removed. - Invalid data for a particular plugin just causes an error to be reported, but the rest of the plugins can still be parsed # User-Facing Changes - The plugin file has a different filename, and it's not a nushell script. - The default `plugin.nu` file will be automatically migrated the first time, but not other plugin config files. - We don't currently provide any utilities that could help edit this file, beyond `plugin add` and `plugin rm` - `from msgpackz`, `to msgpackz` could also help - New commands: `plugin add`, `plugin rm` # Tests + Formatting Tests added for the format and for the invalid handling. - 🟢 `toolkit fmt` - 🟢 `toolkit clippy` - 🟢 `toolkit test` - 🟢 `toolkit test stdlib` # After Submitting - [ ] Check for documentation changes - [ ] Definitely needs release notes
This commit is contained in:
Devyn Cairns
GitHub
@ -6,13 +6,15 @@ use nu_protocol::{
|
||||
report_error, HistoryFileFormat, PipelineData,
|
||||
};
|
||||
#[cfg(feature = "plugin")]
|
||||
use nu_protocol::{ParseError, Spanned};
|
||||
use nu_protocol::{ParseError, PluginCacheFile, Spanned};
|
||||
#[cfg(feature = "plugin")]
|
||||
use nu_utils::utils::perf;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[cfg(feature = "plugin")]
|
||||
const PLUGIN_FILE: &str = "plugin.nu";
|
||||
const PLUGIN_FILE: &str = "plugin.msgpackz";
|
||||
#[cfg(feature = "plugin")]
|
||||
const OLD_PLUGIN_FILE: &str = "plugin.nu";
|
||||
|
||||
const HISTORY_FILE_TXT: &str = "history.txt";
|
||||
const HISTORY_FILE_SQLITE: &str = "history.sqlite3";
|
||||
@ -20,14 +22,38 @@ const HISTORY_FILE_SQLITE: &str = "history.sqlite3";
|
||||
#[cfg(feature = "plugin")]
|
||||
pub fn read_plugin_file(
|
||||
engine_state: &mut EngineState,
|
||||
stack: &mut Stack,
|
||||
plugin_file: Option<Spanned<String>>,
|
||||
storage_path: &str,
|
||||
) {
|
||||
use std::path::Path;
|
||||
|
||||
use nu_protocol::{report_error_new, ShellError};
|
||||
|
||||
let span = plugin_file.as_ref().map(|s| s.span);
|
||||
|
||||
// Check and warn + abort if this is a .nu plugin file
|
||||
if plugin_file
|
||||
.as_ref()
|
||||
.and_then(|p| Path::new(&p.item).extension())
|
||||
.is_some_and(|ext| ext == "nu")
|
||||
{
|
||||
report_error_new(
|
||||
engine_state,
|
||||
&ShellError::GenericError {
|
||||
error: "Wrong plugin file format".into(),
|
||||
msg: ".nu plugin files are no longer supported".into(),
|
||||
span,
|
||||
help: Some("please recreate this file in the new .msgpackz format".into()),
|
||||
inner: vec![],
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let mut start_time = std::time::Instant::now();
|
||||
// Reading signatures from signature file
|
||||
// The plugin.nu file stores the parsed signature collected from each registered plugin
|
||||
add_plugin_file(engine_state, plugin_file, storage_path);
|
||||
// Reading signatures from plugin cache file
|
||||
// The plugin.msgpackz file stores the parsed signature collected from each registered plugin
|
||||
add_plugin_file(engine_state, plugin_file.clone(), storage_path);
|
||||
perf(
|
||||
"add plugin file to engine_state",
|
||||
start_time,
|
||||
@ -38,38 +64,107 @@ pub fn read_plugin_file(
|
||||
);
|
||||
|
||||
start_time = std::time::Instant::now();
|
||||
let plugin_path = engine_state.plugin_signatures.clone();
|
||||
let plugin_path = engine_state.plugin_path.clone();
|
||||
if let Some(plugin_path) = plugin_path {
|
||||
let plugin_filename = plugin_path.to_string_lossy();
|
||||
let plug_path = plugin_filename.to_string();
|
||||
// Open the plugin file
|
||||
let mut file = match std::fs::File::open(&plugin_path) {
|
||||
Ok(file) => file,
|
||||
Err(err) => {
|
||||
if err.kind() == std::io::ErrorKind::NotFound {
|
||||
log::warn!("Plugin file not found: {}", plugin_path.display());
|
||||
|
||||
if let Ok(contents) = std::fs::read(&plugin_path) {
|
||||
perf(
|
||||
&format!("read plugin file {}", &plug_path),
|
||||
start_time,
|
||||
file!(),
|
||||
line!(),
|
||||
column!(),
|
||||
engine_state.get_config().use_ansi_coloring,
|
||||
);
|
||||
start_time = std::time::Instant::now();
|
||||
eval_source(
|
||||
engine_state,
|
||||
stack,
|
||||
&contents,
|
||||
&plugin_filename,
|
||||
PipelineData::empty(),
|
||||
false,
|
||||
);
|
||||
perf(
|
||||
&format!("eval_source plugin file {}", &plug_path),
|
||||
start_time,
|
||||
file!(),
|
||||
line!(),
|
||||
column!(),
|
||||
engine_state.get_config().use_ansi_coloring,
|
||||
// Try migration of an old plugin file if this wasn't a custom plugin file
|
||||
if plugin_file.is_none() && migrate_old_plugin_file(engine_state, storage_path)
|
||||
{
|
||||
let Ok(file) = std::fs::File::open(&plugin_path) else {
|
||||
log::warn!("Failed to load newly migrated plugin file");
|
||||
return;
|
||||
};
|
||||
file
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
report_error_new(
|
||||
engine_state,
|
||||
&ShellError::GenericError {
|
||||
error: format!(
|
||||
"Error while opening plugin cache file: {}",
|
||||
plugin_path.display()
|
||||
),
|
||||
msg: "plugin path defined here".into(),
|
||||
span,
|
||||
help: None,
|
||||
inner: vec![err.into()],
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Abort if the file is empty.
|
||||
if file.metadata().is_ok_and(|m| m.len() == 0) {
|
||||
log::warn!(
|
||||
"Not reading plugin file because it's empty: {}",
|
||||
plugin_path.display()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Read the contents of the plugin file
|
||||
let contents = match PluginCacheFile::read_from(&mut file, span) {
|
||||
Ok(contents) => contents,
|
||||
Err(err) => {
|
||||
log::warn!("Failed to read plugin cache file: {err:?}");
|
||||
report_error_new(
|
||||
engine_state,
|
||||
&ShellError::GenericError {
|
||||
error: format!(
|
||||
"Error while reading plugin cache file: {}",
|
||||
plugin_path.display()
|
||||
),
|
||||
msg: "plugin path defined here".into(),
|
||||
span,
|
||||
help: Some(
|
||||
"you might try deleting the file and registering all of your \
|
||||
plugins again"
|
||||
.into(),
|
||||
),
|
||||
inner: vec![],
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
perf(
|
||||
&format!("read plugin file {}", plugin_path.display()),
|
||||
start_time,
|
||||
file!(),
|
||||
line!(),
|
||||
column!(),
|
||||
engine_state.get_config().use_ansi_coloring,
|
||||
);
|
||||
start_time = std::time::Instant::now();
|
||||
|
||||
let mut working_set = StateWorkingSet::new(engine_state);
|
||||
|
||||
nu_plugin::load_plugin_file(&mut working_set, &contents, span);
|
||||
|
||||
if let Err(err) = engine_state.merge_delta(working_set.render()) {
|
||||
report_error_new(engine_state, &err);
|
||||
return;
|
||||
}
|
||||
|
||||
perf(
|
||||
&format!("load plugin file {}", plugin_path.display()),
|
||||
start_time,
|
||||
file!(),
|
||||
line!(),
|
||||
column!(),
|
||||
engine_state.get_config().use_ansi_coloring,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -79,15 +174,30 @@ pub fn add_plugin_file(
|
||||
plugin_file: Option<Spanned<String>>,
|
||||
storage_path: &str,
|
||||
) {
|
||||
use std::path::Path;
|
||||
|
||||
let working_set = StateWorkingSet::new(engine_state);
|
||||
let cwd = working_set.get_cwd();
|
||||
|
||||
if let Some(plugin_file) = plugin_file {
|
||||
if let Ok(path) = canonicalize_with(&plugin_file.item, cwd) {
|
||||
engine_state.plugin_signatures = Some(path)
|
||||
let path = Path::new(&plugin_file.item);
|
||||
let path_dir = path.parent().unwrap_or(path);
|
||||
// Just try to canonicalize the directory of the plugin file first.
|
||||
if let Ok(path_dir) = canonicalize_with(path_dir, &cwd) {
|
||||
// Try to canonicalize the actual filename, but it's ok if that fails. The file doesn't
|
||||
// have to exist.
|
||||
let path = path_dir.join(path.file_name().unwrap_or(path.as_os_str()));
|
||||
let path = canonicalize_with(&path, &cwd).unwrap_or(path);
|
||||
engine_state.plugin_path = Some(path)
|
||||
} else {
|
||||
let e = ParseError::FileNotFound(plugin_file.item, plugin_file.span);
|
||||
report_error(&working_set, &e);
|
||||
// It's an error if the directory for the plugin file doesn't exist.
|
||||
report_error(
|
||||
&working_set,
|
||||
&ParseError::FileNotFound(
|
||||
path_dir.to_string_lossy().into_owned(),
|
||||
plugin_file.span,
|
||||
),
|
||||
);
|
||||
}
|
||||
} else if let Some(mut plugin_path) = nu_path::config_dir() {
|
||||
// Path to store plugins signatures
|
||||
@ -95,7 +205,7 @@ pub fn add_plugin_file(
|
||||
let mut plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
|
||||
plugin_path.push(PLUGIN_FILE);
|
||||
let plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
|
||||
engine_state.plugin_signatures = Some(plugin_path);
|
||||
engine_state.plugin_path = Some(plugin_path);
|
||||
}
|
||||
}
|
||||
|
||||
@ -151,3 +261,129 @@ pub(crate) fn get_history_path(storage_path: &str, mode: HistoryFileFormat) -> O
|
||||
history_path
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "plugin")]
|
||||
pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -> bool {
|
||||
use nu_protocol::{
|
||||
report_error_new, PluginCacheItem, PluginCacheItemData, PluginExample, PluginIdentity,
|
||||
PluginSignature, ShellError,
|
||||
};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
let cwd = engine_state.current_work_dir();
|
||||
|
||||
let Some(config_dir) = nu_path::config_dir().and_then(|mut dir| {
|
||||
dir.push(storage_path);
|
||||
nu_path::canonicalize_with(dir, &cwd).ok()
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let Ok(old_plugin_file_path) = nu_path::canonicalize_with(OLD_PLUGIN_FILE, &config_dir) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let old_contents = match std::fs::read(&old_plugin_file_path) {
|
||||
Ok(old_contents) => old_contents,
|
||||
Err(err) => {
|
||||
report_error_new(
|
||||
engine_state,
|
||||
&ShellError::GenericError {
|
||||
error: "Can't read old plugin file to migrate".into(),
|
||||
msg: "".into(),
|
||||
span: None,
|
||||
help: Some(err.to_string()),
|
||||
inner: vec![],
|
||||
},
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// Make a copy of the engine state, because we'll read the newly generated file
|
||||
let mut engine_state = engine_state.clone();
|
||||
let mut stack = Stack::new();
|
||||
|
||||
if !eval_source(
|
||||
&mut engine_state,
|
||||
&mut stack,
|
||||
&old_contents,
|
||||
&old_plugin_file_path.to_string_lossy(),
|
||||
PipelineData::Empty,
|
||||
false,
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Now that the plugin commands are loaded, we just have to generate the file
|
||||
let mut contents = PluginCacheFile::new();
|
||||
|
||||
let mut groups = BTreeMap::<PluginIdentity, Vec<PluginSignature>>::new();
|
||||
|
||||
for decl in engine_state.plugin_decls() {
|
||||
if let Some(identity) = decl.plugin_identity() {
|
||||
groups
|
||||
.entry(identity.clone())
|
||||
.or_default()
|
||||
.push(PluginSignature {
|
||||
sig: decl.signature(),
|
||||
examples: decl
|
||||
.examples()
|
||||
.into_iter()
|
||||
.map(PluginExample::from)
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (identity, commands) in groups {
|
||||
contents.upsert_plugin(PluginCacheItem {
|
||||
name: identity.name().to_owned(),
|
||||
filename: identity.filename().to_owned(),
|
||||
shell: identity.shell().map(|p| p.to_owned()),
|
||||
data: PluginCacheItemData::Valid { commands },
|
||||
});
|
||||
}
|
||||
|
||||
// Write the new file
|
||||
let new_plugin_file_path = config_dir.join(PLUGIN_FILE);
|
||||
if let Err(err) = std::fs::File::create(&new_plugin_file_path)
|
||||
.map_err(|e| e.into())
|
||||
.and_then(|file| contents.write_to(file, None))
|
||||
{
|
||||
report_error_new(
|
||||
&engine_state,
|
||||
&ShellError::GenericError {
|
||||
error: "Failed to save migrated plugin file".into(),
|
||||
msg: "".into(),
|
||||
span: None,
|
||||
help: Some("ensure `$nu.plugin-path` is writable".into()),
|
||||
inner: vec![err],
|
||||
},
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
if engine_state.is_interactive {
|
||||
eprintln!(
|
||||
"Your old plugin.nu file has been migrated to the new format: {}",
|
||||
new_plugin_file_path.display()
|
||||
);
|
||||
eprintln!(
|
||||
"The plugin.nu file has not been removed. If `plugin list` looks okay, \
|
||||
you may do so manually."
|
||||
);
|
||||
}
|
||||
|
||||
perf(
|
||||
"migrate old plugin file",
|
||||
start_time,
|
||||
file!(),
|
||||
line!(),
|
||||
column!(),
|
||||
engine_state.get_config().use_ansi_coloring,
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
@ -32,4 +32,6 @@ pub use validation::NuValidator;
|
||||
#[cfg(feature = "plugin")]
|
||||
pub use config_files::add_plugin_file;
|
||||
#[cfg(feature = "plugin")]
|
||||
pub use config_files::migrate_old_plugin_file;
|
||||
#[cfg(feature = "plugin")]
|
||||
pub use config_files::read_plugin_file;
|
||||
|
||||
Reference in New Issue
Block a user