mirror of
https://github.com/nushell/nushell.git
synced 2025-06-30 06:30:08 +02:00
Replace ExternalStream
with new ByteStream
type (#12774)
# Description This PR introduces a `ByteStream` type which is a `Read`-able stream of bytes. Internally, it has an enum over three different byte stream sources: ```rust pub enum ByteStreamSource { Read(Box<dyn Read + Send + 'static>), File(File), Child(ChildProcess), } ``` This is in comparison to the current `RawStream` type, which is an `Iterator<Item = Vec<u8>>` and has to allocate for each read chunk. Currently, `PipelineData::ExternalStream` serves a weird dual role where it is either external command output or a wrapper around `RawStream`. `ByteStream` makes this distinction more clear (via `ByteStreamSource`) and replaces `PipelineData::ExternalStream` in this PR: ```rust pub enum PipelineData { Empty, Value(Value, Option<PipelineMetadata>), ListStream(ListStream, Option<PipelineMetadata>), ByteStream(ByteStream, Option<PipelineMetadata>), } ``` The PR is relatively large, but a decent amount of it is just repetitive changes. This PR fixes #7017, fixes #10763, and fixes #12369. This PR also improves performance when piping external commands. Nushell should, in most cases, have competitive pipeline throughput compared to, e.g., bash. | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | -------------------------------------------------- | -------------:| ------------:| -----------:| | `throughput \| rg 'x'` | 3059 | 3744 | 3739 | | `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 | # User-Facing Changes - This is a breaking change for the plugin communication protocol, because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`. Plugins now only have to deal with a single input stream, as opposed to the previous three streams: stdout, stderr, and exit code. - The output of `describe` has been changed for external/byte streams. - Temporary breaking change: `bytes starts-with` no longer works with byte streams. This is to keep the PR smaller, and `bytes ends-with` already does not work on byte streams. - If a process core dumped, then instead of having a `Value::Error` in the `exit_code` column of the output returned from `complete`, it now is a `Value::Int` with the negation of the signal number. # After Submitting - Update docs and book as necessary - Release notes (e.g., plugin protocol changes) - Adapt/convert commands to work with byte streams (high priority is `str length`, `bytes starts-with`, and maybe `bytes ends-with`). - Refactor the `tee` code, Devyn has already done some work on this. --------- Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
This commit is contained in:
@ -313,7 +313,7 @@ where
|
||||
// Unwrap the PipelineData from input, consuming the potential stream, and pass it to the
|
||||
// simpler signature in Plugin
|
||||
let span = input.span().unwrap_or(call.head);
|
||||
let input_value = input.into_value(span);
|
||||
let input_value = input.into_value(span)?;
|
||||
// Wrap the output in PipelineData::Value
|
||||
<Self as SimplePluginCommand>::run(self, plugin, engine, call, &input_value)
|
||||
.map(|value| PipelineData::Value(value, None))
|
||||
|
@ -345,7 +345,7 @@ impl InterfaceManager for EngineInterfaceManager {
|
||||
});
|
||||
Ok(PipelineData::ListStream(stream, meta))
|
||||
}
|
||||
PipelineData::Empty | PipelineData::ExternalStream { .. } => Ok(data),
|
||||
PipelineData::Empty | PipelineData::ByteStream(..) => Ok(data),
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -850,7 +850,7 @@ impl EngineInterface {
|
||||
let input = input.map_or_else(|| PipelineData::Empty, |v| PipelineData::Value(v, None));
|
||||
let output = self.eval_closure_with_stream(closure, positional, input, true, false)?;
|
||||
// Unwrap an error value
|
||||
match output.into_value(closure.span) {
|
||||
match output.into_value(closure.span)? {
|
||||
Value::Error { error, .. } => Err(*error),
|
||||
value => Ok(value),
|
||||
}
|
||||
@ -920,7 +920,7 @@ impl Interface for EngineInterface {
|
||||
});
|
||||
Ok(PipelineData::ListStream(stream, meta))
|
||||
}
|
||||
PipelineData::Empty | PipelineData::ExternalStream { .. } => Ok(data),
|
||||
PipelineData::Empty | PipelineData::ByteStream(..) => Ok(data),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -4,10 +4,9 @@ use super::{EngineInterfaceManager, ReceivedPluginCall};
|
||||
use nu_plugin_core::{interface_test_util::TestCase, Interface, InterfaceManager};
|
||||
use nu_plugin_protocol::{
|
||||
test_util::{expected_test_custom_value, test_plugin_custom_value, TestCustomValue},
|
||||
CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, EvaluatedCall,
|
||||
ExternalStreamInfo, ListStreamInfo, PipelineDataHeader, PluginCall, PluginCallResponse,
|
||||
PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, RawStreamInfo,
|
||||
StreamData,
|
||||
ByteStreamInfo, CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse,
|
||||
EvaluatedCall, ListStreamInfo, PipelineDataHeader, PluginCall, PluginCallResponse,
|
||||
PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, StreamData,
|
||||
};
|
||||
use nu_protocol::{
|
||||
engine::Closure, Config, CustomValue, IntoInterruptiblePipelineData, LabeledError,
|
||||
@ -158,16 +157,9 @@ fn manager_consume_all_propagates_message_error_to_readers() -> Result<(), Shell
|
||||
test.add(invalid_input());
|
||||
|
||||
let stream = manager.read_pipeline_data(
|
||||
PipelineDataHeader::ExternalStream(ExternalStreamInfo {
|
||||
PipelineDataHeader::ByteStream(ByteStreamInfo {
|
||||
id: 0,
|
||||
span: Span::test_data(),
|
||||
stdout: Some(RawStreamInfo {
|
||||
id: 0,
|
||||
is_binary: false,
|
||||
known_size: None,
|
||||
}),
|
||||
stderr: None,
|
||||
exit_code: None,
|
||||
trim_end_newline: false,
|
||||
}),
|
||||
None,
|
||||
)?;
|
||||
@ -1046,7 +1038,7 @@ fn interface_eval_closure_with_stream() -> Result<(), ShellError> {
|
||||
true,
|
||||
false,
|
||||
)?
|
||||
.into_value(Span::test_data());
|
||||
.into_value(Span::test_data())?;
|
||||
|
||||
assert_eq!(Value::test_int(2), result);
|
||||
|
||||
|
@ -30,7 +30,7 @@ pub use interface::{EngineInterface, EngineInterfaceManager};
|
||||
|
||||
/// This should be larger than the largest commonly sent message to avoid excessive fragmentation.
|
||||
///
|
||||
/// The buffers coming from external streams are typically each 8192 bytes, so double that.
|
||||
/// The buffers coming from byte streams are typically each 8192 bytes, so double that.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) const OUTPUT_BUFFER_SIZE: usize = 16384;
|
||||
|
||||
|
Reference in New Issue
Block a user