2023-03-20 05:05:22 +01:00
|
|
|
use crate::util::eval_source;
|
2022-03-16 19:17:06 +01:00
|
|
|
#[cfg(feature = "plugin")]
|
2022-07-17 20:29:19 +02:00
|
|
|
use nu_path::canonicalize_with;
|
2024-05-10 01:29:27 +02:00
|
|
|
#[cfg(feature = "plugin")]
|
|
|
|
use nu_protocol::{engine::StateWorkingSet, report_error, ParseError, PluginRegistryFile, Spanned};
|
Add `command_prelude` module (#12291)
# Description
When implementing a `Command`, one must also import all the types
present in the function signatures for `Command`. This makes it so that
we often import the same set of types in each command implementation
file. E.g., something like this:
```rust
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
record, Category, Example, IntoInterruptiblePipelineData, IntoPipelineData, PipelineData,
ShellError, Signature, Span, Type, Value,
};
```
This PR adds the `nu_engine::command_prelude` module which contains the
necessary and commonly used types to implement a `Command`:
```rust
// command_prelude.rs
pub use crate::CallExt;
pub use nu_protocol::{
ast::{Call, CellPath},
engine::{Command, EngineState, Stack},
record, Category, Example, IntoInterruptiblePipelineData, IntoPipelineData, IntoSpanned,
PipelineData, Record, ShellError, Signature, Span, Spanned, SyntaxShape, Type, Value,
};
```
This should reduce the boilerplate needed to implement a command and
also gives us a place to track the breadth of the `Command` API. I tried
to be conservative with what went into the prelude modules, since it
might be hard/annoying to remove items from the prelude in the future.
Let me know if something should be included or excluded.
2024-03-26 22:17:30 +01:00
|
|
|
use nu_protocol::{
|
2024-05-10 01:29:27 +02:00
|
|
|
engine::{EngineState, Stack},
|
|
|
|
report_error_new, HistoryFileFormat, PipelineData,
|
Add `command_prelude` module (#12291)
# Description
When implementing a `Command`, one must also import all the types
present in the function signatures for `Command`. This makes it so that
we often import the same set of types in each command implementation
file. E.g., something like this:
```rust
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
record, Category, Example, IntoInterruptiblePipelineData, IntoPipelineData, PipelineData,
ShellError, Signature, Span, Type, Value,
};
```
This PR adds the `nu_engine::command_prelude` module which contains the
necessary and commonly used types to implement a `Command`:
```rust
// command_prelude.rs
pub use crate::CallExt;
pub use nu_protocol::{
ast::{Call, CellPath},
engine::{Command, EngineState, Stack},
record, Category, Example, IntoInterruptiblePipelineData, IntoPipelineData, IntoSpanned,
PipelineData, Record, ShellError, Signature, Span, Spanned, SyntaxShape, Type, Value,
};
```
This should reduce the boilerplate needed to implement a command and
also gives us a place to track the breadth of the `Command` API. I tried
to be conservative with what went into the prelude modules, since it
might be hard/annoying to remove items from the prelude in the future.
Let me know if something should be included or excluded.
2024-03-26 22:17:30 +01:00
|
|
|
};
|
2023-01-24 21:28:59 +01:00
|
|
|
#[cfg(feature = "plugin")]
|
2024-06-28 01:56:56 +02:00
|
|
|
use nu_utils::perf;
|
2022-03-16 19:17:06 +01:00
|
|
|
use std::path::PathBuf;
|
|
|
|
|
|
|
|
#[cfg(feature = "plugin")]
|
2024-04-21 14:36:26 +02:00
|
|
|
const PLUGIN_FILE: &str = "plugin.msgpackz";
|
|
|
|
#[cfg(feature = "plugin")]
|
|
|
|
const OLD_PLUGIN_FILE: &str = "plugin.nu";
|
2022-03-16 19:17:06 +01:00
|
|
|
|
2022-06-14 22:53:33 +02:00
|
|
|
const HISTORY_FILE_TXT: &str = "history.txt";
|
|
|
|
const HISTORY_FILE_SQLITE: &str = "history.sqlite3";
|
|
|
|
|
2022-03-16 19:17:06 +01:00
|
|
|
#[cfg(feature = "plugin")]
|
|
|
|
pub fn read_plugin_file(
|
|
|
|
engine_state: &mut EngineState,
|
2022-07-17 20:29:19 +02:00
|
|
|
plugin_file: Option<Spanned<String>>,
|
2022-03-16 19:17:06 +01:00
|
|
|
storage_path: &str,
|
|
|
|
) {
|
2024-05-10 01:29:27 +02:00
|
|
|
use nu_protocol::ShellError;
|
2024-04-21 14:36:26 +02:00
|
|
|
use std::path::Path;
|
|
|
|
|
|
|
|
let span = plugin_file.as_ref().map(|s| s.span);
|
|
|
|
|
|
|
|
// Check and warn + abort if this is a .nu plugin file
|
|
|
|
if plugin_file
|
|
|
|
.as_ref()
|
|
|
|
.and_then(|p| Path::new(&p.item).extension())
|
|
|
|
.is_some_and(|ext| ext == "nu")
|
|
|
|
{
|
|
|
|
report_error_new(
|
|
|
|
engine_state,
|
|
|
|
&ShellError::GenericError {
|
|
|
|
error: "Wrong plugin file format".into(),
|
|
|
|
msg: ".nu plugin files are no longer supported".into(),
|
|
|
|
span,
|
|
|
|
help: Some("please recreate this file in the new .msgpackz format".into()),
|
|
|
|
inner: vec![],
|
|
|
|
},
|
|
|
|
);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2024-04-12 17:11:41 +02:00
|
|
|
let mut start_time = std::time::Instant::now();
|
2024-04-25 00:40:39 +02:00
|
|
|
// Reading signatures from plugin registry file
|
2024-04-21 14:36:26 +02:00
|
|
|
// The plugin.msgpackz file stores the parsed signature collected from each registered plugin
|
|
|
|
add_plugin_file(engine_state, plugin_file.clone(), storage_path);
|
2024-06-28 01:56:56 +02:00
|
|
|
perf!(
|
2024-04-12 17:11:41 +02:00
|
|
|
"add plugin file to engine_state",
|
|
|
|
start_time,
|
2024-06-28 01:56:56 +02:00
|
|
|
engine_state.get_config().use_ansi_coloring
|
2024-04-12 17:11:41 +02:00
|
|
|
);
|
2022-03-16 19:17:06 +01:00
|
|
|
|
2024-04-12 17:11:41 +02:00
|
|
|
start_time = std::time::Instant::now();
|
2024-04-21 14:36:26 +02:00
|
|
|
let plugin_path = engine_state.plugin_path.clone();
|
2022-03-16 19:17:06 +01:00
|
|
|
if let Some(plugin_path) = plugin_path {
|
2024-04-21 14:36:26 +02:00
|
|
|
// Open the plugin file
|
|
|
|
let mut file = match std::fs::File::open(&plugin_path) {
|
|
|
|
Ok(file) => file,
|
|
|
|
Err(err) => {
|
|
|
|
if err.kind() == std::io::ErrorKind::NotFound {
|
|
|
|
log::warn!("Plugin file not found: {}", plugin_path.display());
|
|
|
|
|
|
|
|
// Try migration of an old plugin file if this wasn't a custom plugin file
|
|
|
|
if plugin_file.is_none() && migrate_old_plugin_file(engine_state, storage_path)
|
|
|
|
{
|
|
|
|
let Ok(file) = std::fs::File::open(&plugin_path) else {
|
|
|
|
log::warn!("Failed to load newly migrated plugin file");
|
|
|
|
return;
|
|
|
|
};
|
|
|
|
file
|
|
|
|
} else {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
report_error_new(
|
|
|
|
engine_state,
|
|
|
|
&ShellError::GenericError {
|
|
|
|
error: format!(
|
2024-04-25 00:40:39 +02:00
|
|
|
"Error while opening plugin registry file: {}",
|
2024-04-21 14:36:26 +02:00
|
|
|
plugin_path.display()
|
|
|
|
),
|
|
|
|
msg: "plugin path defined here".into(),
|
|
|
|
span,
|
|
|
|
help: None,
|
|
|
|
inner: vec![err.into()],
|
|
|
|
},
|
|
|
|
);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
// Abort if the file is empty.
|
|
|
|
if file.metadata().is_ok_and(|m| m.len() == 0) {
|
|
|
|
log::warn!(
|
|
|
|
"Not reading plugin file because it's empty: {}",
|
|
|
|
plugin_path.display()
|
2024-04-12 17:11:41 +02:00
|
|
|
);
|
2024-04-21 14:36:26 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Read the contents of the plugin file
|
2024-04-25 00:40:39 +02:00
|
|
|
let contents = match PluginRegistryFile::read_from(&mut file, span) {
|
2024-04-21 14:36:26 +02:00
|
|
|
Ok(contents) => contents,
|
|
|
|
Err(err) => {
|
2024-04-25 00:40:39 +02:00
|
|
|
log::warn!("Failed to read plugin registry file: {err:?}");
|
2024-04-21 14:36:26 +02:00
|
|
|
report_error_new(
|
|
|
|
engine_state,
|
|
|
|
&ShellError::GenericError {
|
|
|
|
error: format!(
|
2024-04-25 00:40:39 +02:00
|
|
|
"Error while reading plugin registry file: {}",
|
2024-04-21 14:36:26 +02:00
|
|
|
plugin_path.display()
|
|
|
|
),
|
|
|
|
msg: "plugin path defined here".into(),
|
|
|
|
span,
|
|
|
|
help: Some(
|
|
|
|
"you might try deleting the file and registering all of your \
|
|
|
|
plugins again"
|
|
|
|
.into(),
|
|
|
|
),
|
|
|
|
inner: vec![],
|
|
|
|
},
|
|
|
|
);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2024-06-28 01:56:56 +02:00
|
|
|
perf!(
|
2024-04-21 14:36:26 +02:00
|
|
|
&format!("read plugin file {}", plugin_path.display()),
|
|
|
|
start_time,
|
2024-06-28 01:56:56 +02:00
|
|
|
engine_state.get_config().use_ansi_coloring
|
2024-04-21 14:36:26 +02:00
|
|
|
);
|
|
|
|
start_time = std::time::Instant::now();
|
|
|
|
|
|
|
|
let mut working_set = StateWorkingSet::new(engine_state);
|
|
|
|
|
2024-04-27 19:08:12 +02:00
|
|
|
nu_plugin_engine::load_plugin_file(&mut working_set, &contents, span);
|
2024-04-21 14:36:26 +02:00
|
|
|
|
|
|
|
if let Err(err) = engine_state.merge_delta(working_set.render()) {
|
|
|
|
report_error_new(engine_state, &err);
|
|
|
|
return;
|
2022-03-16 19:17:06 +01:00
|
|
|
}
|
2024-04-21 14:36:26 +02:00
|
|
|
|
2024-06-28 01:56:56 +02:00
|
|
|
perf!(
|
2024-04-21 14:36:26 +02:00
|
|
|
&format!("load plugin file {}", plugin_path.display()),
|
|
|
|
start_time,
|
2024-06-28 01:56:56 +02:00
|
|
|
engine_state.get_config().use_ansi_coloring
|
2024-04-21 14:36:26 +02:00
|
|
|
);
|
2022-03-16 19:17:06 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(feature = "plugin")]
|
2022-07-17 20:29:19 +02:00
|
|
|
pub fn add_plugin_file(
|
|
|
|
engine_state: &mut EngineState,
|
|
|
|
plugin_file: Option<Spanned<String>>,
|
|
|
|
storage_path: &str,
|
|
|
|
) {
|
2024-04-21 14:36:26 +02:00
|
|
|
use std::path::Path;
|
|
|
|
|
Canonicalize default-config-dir and plugin-path (#11999)
<!--
if this PR closes one or more issues, you can automatically link the PR
with
them by using one of the [*linking
keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword),
e.g.
- this PR should close #xxxx
- fixes #xxxx
you can also mention related issues, PRs or discussions!
-->
# Description
<!--
Thank you for improving Nushell. Please, check our [contributing
guide](../CONTRIBUTING.md) and talk to the core team before making major
changes.
Description of your pull request goes here. **Provide examples and/or
screenshots** if your changes affect the user experience.
-->
This PR makes sure `$nu.default-config-dir` and `$nu.plugin-path` are
canonicalized.
# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->
`$nu.default-config-dir` (and `$nu.plugin-path`) will now give canonical
paths, with symlinks and whatnot resolved.
# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.
Make sure you've run and fixed any issues with these commands:
- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to
check that you're using the standard code style
- `cargo test --workspace` to check that all tests pass (on Windows make
sure to [enable developer
mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
- `cargo run -- -c "use std testing; testing run-tests --path
crates/nu-std"` to run the tests for the standard library
> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->
I've added a couple of tests to check that even if the config folder
and/or any of the config files within are symlinks, the `$nu.*`
variables are properly canonicalized. These tests unfortunately only run
on Linux and MacOS, because I couldn't figure out how to change the
config directory on Windows. Also, given that they involve creating
files, I'm not sure if they're excessive, so I could remove one or two
of them.
# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->
2024-03-02 18:15:31 +01:00
|
|
|
let working_set = StateWorkingSet::new(engine_state);
|
2022-07-17 20:29:19 +02:00
|
|
|
|
2024-05-07 17:17:49 +02:00
|
|
|
if let Ok(cwd) = engine_state.cwd_as_string(None) {
|
|
|
|
if let Some(plugin_file) = plugin_file {
|
|
|
|
let path = Path::new(&plugin_file.item);
|
|
|
|
let path_dir = path.parent().unwrap_or(path);
|
|
|
|
// Just try to canonicalize the directory of the plugin file first.
|
|
|
|
if let Ok(path_dir) = canonicalize_with(path_dir, &cwd) {
|
|
|
|
// Try to canonicalize the actual filename, but it's ok if that fails. The file doesn't
|
|
|
|
// have to exist.
|
|
|
|
let path = path_dir.join(path.file_name().unwrap_or(path.as_os_str()));
|
|
|
|
let path = canonicalize_with(&path, &cwd).unwrap_or(path);
|
|
|
|
engine_state.plugin_path = Some(path)
|
|
|
|
} else {
|
|
|
|
// It's an error if the directory for the plugin file doesn't exist.
|
|
|
|
report_error(
|
|
|
|
&working_set,
|
|
|
|
&ParseError::FileNotFound(
|
|
|
|
path_dir.to_string_lossy().into_owned(),
|
|
|
|
plugin_file.span,
|
|
|
|
),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
} else if let Some(mut plugin_path) = nu_path::config_dir() {
|
|
|
|
// Path to store plugins signatures
|
|
|
|
plugin_path.push(storage_path);
|
|
|
|
let mut plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
|
|
|
|
plugin_path.push(PLUGIN_FILE);
|
|
|
|
let plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
|
|
|
|
engine_state.plugin_path = Some(plugin_path);
|
2022-07-17 20:29:19 +02:00
|
|
|
}
|
2022-03-16 19:17:06 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn eval_config_contents(
|
|
|
|
config_path: PathBuf,
|
|
|
|
engine_state: &mut EngineState,
|
|
|
|
stack: &mut Stack,
|
|
|
|
) {
|
|
|
|
if config_path.exists() & config_path.is_file() {
|
2022-11-04 21:11:17 +01:00
|
|
|
let config_filename = config_path.to_string_lossy();
|
2022-03-16 19:17:06 +01:00
|
|
|
|
|
|
|
if let Ok(contents) = std::fs::read(&config_path) {
|
2024-04-19 08:38:08 +02:00
|
|
|
// Set the current active file to the config file.
|
|
|
|
let prev_file = engine_state.file.take();
|
|
|
|
engine_state.file = Some(config_path.clone());
|
2024-04-09 16:06:41 +02:00
|
|
|
|
2022-03-16 19:17:06 +01:00
|
|
|
eval_source(
|
|
|
|
engine_state,
|
|
|
|
stack,
|
|
|
|
&contents,
|
|
|
|
&config_filename,
|
2022-12-07 19:31:57 +01:00
|
|
|
PipelineData::empty(),
|
2023-02-02 00:02:27 +01:00
|
|
|
false,
|
2022-03-16 19:17:06 +01:00
|
|
|
);
|
|
|
|
|
2024-04-19 08:38:08 +02:00
|
|
|
// Restore the current active file.
|
|
|
|
engine_state.file = prev_file;
|
2024-04-09 16:06:41 +02:00
|
|
|
|
2022-07-14 16:09:27 +02:00
|
|
|
// Merge the environment in case env vars changed in the config
|
2024-05-24 18:09:59 +02:00
|
|
|
match engine_state.cwd(Some(stack)) {
|
|
|
|
Ok(cwd) => {
|
|
|
|
if let Err(e) = engine_state.merge_env(stack, cwd) {
|
|
|
|
report_error_new(engine_state, &e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Err(e) => {
|
|
|
|
report_error_new(engine_state, &e);
|
|
|
|
}
|
2022-03-16 19:17:06 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-06-14 22:53:33 +02:00
|
|
|
|
|
|
|
pub(crate) fn get_history_path(storage_path: &str, mode: HistoryFileFormat) -> Option<PathBuf> {
|
|
|
|
nu_path::config_dir().map(|mut history_path| {
|
|
|
|
history_path.push(storage_path);
|
|
|
|
history_path.push(match mode {
|
|
|
|
HistoryFileFormat::PlainText => HISTORY_FILE_TXT,
|
|
|
|
HistoryFileFormat::Sqlite => HISTORY_FILE_SQLITE,
|
|
|
|
});
|
|
|
|
history_path
|
|
|
|
})
|
|
|
|
}
|
2024-04-21 14:36:26 +02:00
|
|
|
|
|
|
|
#[cfg(feature = "plugin")]
|
|
|
|
pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -> bool {
|
|
|
|
use nu_protocol::{
|
2024-05-10 01:29:27 +02:00
|
|
|
PluginExample, PluginIdentity, PluginRegistryItem, PluginRegistryItemData, PluginSignature,
|
|
|
|
ShellError,
|
2024-04-21 14:36:26 +02:00
|
|
|
};
|
|
|
|
use std::collections::BTreeMap;
|
|
|
|
|
|
|
|
let start_time = std::time::Instant::now();
|
|
|
|
|
2024-05-07 17:17:49 +02:00
|
|
|
let Ok(cwd) = engine_state.cwd_as_string(None) else {
|
|
|
|
return false;
|
|
|
|
};
|
2024-04-21 14:36:26 +02:00
|
|
|
|
|
|
|
let Some(config_dir) = nu_path::config_dir().and_then(|mut dir| {
|
|
|
|
dir.push(storage_path);
|
|
|
|
nu_path::canonicalize_with(dir, &cwd).ok()
|
|
|
|
}) else {
|
|
|
|
return false;
|
|
|
|
};
|
|
|
|
|
|
|
|
let Ok(old_plugin_file_path) = nu_path::canonicalize_with(OLD_PLUGIN_FILE, &config_dir) else {
|
|
|
|
return false;
|
|
|
|
};
|
|
|
|
|
|
|
|
let old_contents = match std::fs::read(&old_plugin_file_path) {
|
|
|
|
Ok(old_contents) => old_contents,
|
|
|
|
Err(err) => {
|
|
|
|
report_error_new(
|
|
|
|
engine_state,
|
|
|
|
&ShellError::GenericError {
|
|
|
|
error: "Can't read old plugin file to migrate".into(),
|
|
|
|
msg: "".into(),
|
|
|
|
span: None,
|
|
|
|
help: Some(err.to_string()),
|
|
|
|
inner: vec![],
|
|
|
|
},
|
|
|
|
);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
// Make a copy of the engine state, because we'll read the newly generated file
|
|
|
|
let mut engine_state = engine_state.clone();
|
|
|
|
let mut stack = Stack::new();
|
|
|
|
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
if eval_source(
|
2024-04-21 14:36:26 +02:00
|
|
|
&mut engine_state,
|
|
|
|
&mut stack,
|
|
|
|
&old_contents,
|
|
|
|
&old_plugin_file_path.to_string_lossy(),
|
|
|
|
PipelineData::Empty,
|
|
|
|
false,
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
) != 0
|
|
|
|
{
|
2024-04-21 14:36:26 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Now that the plugin commands are loaded, we just have to generate the file
|
2024-04-25 00:40:39 +02:00
|
|
|
let mut contents = PluginRegistryFile::new();
|
2024-04-21 14:36:26 +02:00
|
|
|
|
|
|
|
let mut groups = BTreeMap::<PluginIdentity, Vec<PluginSignature>>::new();
|
|
|
|
|
|
|
|
for decl in engine_state.plugin_decls() {
|
|
|
|
if let Some(identity) = decl.plugin_identity() {
|
|
|
|
groups
|
|
|
|
.entry(identity.clone())
|
|
|
|
.or_default()
|
|
|
|
.push(PluginSignature {
|
|
|
|
sig: decl.signature(),
|
|
|
|
examples: decl
|
|
|
|
.examples()
|
|
|
|
.into_iter()
|
|
|
|
.map(PluginExample::from)
|
|
|
|
.collect(),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (identity, commands) in groups {
|
2024-04-25 00:40:39 +02:00
|
|
|
contents.upsert_plugin(PluginRegistryItem {
|
2024-04-21 14:36:26 +02:00
|
|
|
name: identity.name().to_owned(),
|
|
|
|
filename: identity.filename().to_owned(),
|
|
|
|
shell: identity.shell().map(|p| p.to_owned()),
|
2024-06-21 13:27:09 +02:00
|
|
|
data: PluginRegistryItemData::Valid {
|
|
|
|
metadata: Default::default(),
|
|
|
|
commands,
|
|
|
|
},
|
2024-04-21 14:36:26 +02:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
// Write the new file
|
|
|
|
let new_plugin_file_path = config_dir.join(PLUGIN_FILE);
|
|
|
|
if let Err(err) = std::fs::File::create(&new_plugin_file_path)
|
|
|
|
.map_err(|e| e.into())
|
|
|
|
.and_then(|file| contents.write_to(file, None))
|
|
|
|
{
|
|
|
|
report_error_new(
|
|
|
|
&engine_state,
|
|
|
|
&ShellError::GenericError {
|
|
|
|
error: "Failed to save migrated plugin file".into(),
|
|
|
|
msg: "".into(),
|
|
|
|
span: None,
|
|
|
|
help: Some("ensure `$nu.plugin-path` is writable".into()),
|
|
|
|
inner: vec![err],
|
|
|
|
},
|
|
|
|
);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if engine_state.is_interactive {
|
|
|
|
eprintln!(
|
|
|
|
"Your old plugin.nu file has been migrated to the new format: {}",
|
|
|
|
new_plugin_file_path.display()
|
|
|
|
);
|
|
|
|
eprintln!(
|
|
|
|
"The plugin.nu file has not been removed. If `plugin list` looks okay, \
|
|
|
|
you may do so manually."
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2024-06-28 01:56:56 +02:00
|
|
|
perf!(
|
2024-04-21 14:36:26 +02:00
|
|
|
"migrate old plugin file",
|
|
|
|
start_time,
|
2024-06-28 01:56:56 +02:00
|
|
|
engine_state.get_config().use_ansi_coloring
|
2024-04-21 14:36:26 +02:00
|
|
|
);
|
|
|
|
true
|
|
|
|
}
|