extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-06-30 22:50:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Replace ExternalStream with new ByteStream type (#12774)

Browse Source 
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
    Read(Box<dyn Read + Send + 'static>),
    File(File),
    Child(ChildProcess),
}
```

This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.

Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
    Empty,
    Value(Value, Option<PipelineMetadata>),
    ListStream(ListStream, Option<PipelineMetadata>),
    ByteStream(ByteStream, Option<PipelineMetadata>),
}
```

The PR is relatively large, but a decent amount of it is just repetitive
changes.

This PR fixes #7017, fixes #10763, and fixes #12369.

This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |

# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.

# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.

---------

Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
This commit is contained in:
Ian Manske
2024-05-16 14:11:18 +00:00
committed by GitHub
parent 1b8eb23785
commit 6fd854ed9f
210 changed files with 3955 additions and 4012 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
crates/nu-cmd-extra/src/extra/bits/into.rs
View File

@ -118,22 +118,12 @@ fn into_bits(
let cell_paths = call.rest(engine_state, stack, 0)?;
let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths);
match input {
PipelineData::ExternalStream { stdout: None, .. } => {
Ok(Value::binary(vec![], head).into_pipeline_data())
}
PipelineData::ExternalStream {
stdout: Some(stream),
..
} => {
// TODO: in the future, we may want this to stream out, converting each to bytes
let output = stream.into_bytes()?;
Ok(Value::binary(output.item, head).into_pipeline_data())
}
_ => {
let args = Arguments { cell_paths };
operate(action, args, input, call.head, engine_state.ctrlc.clone())
}
if let PipelineData::ByteStream(stream, ..) = input {
// TODO: in the future, we may want this to stream out, converting each to bytes
Ok(Value::binary(stream.into_bytes()?, head).into_pipeline_data())
} else {
let args = Arguments { cell_paths };
operate(action, args, input, call.head, engine_state.ctrlc.clone())
}
}

50
crates/nu-cmd-extra/src/extra/filters/each_while.rs
View File

@ -78,38 +78,40 @@ impl Command for EachWhile {
| PipelineData::ListStream(..) => {
let mut closure = ClosureEval::new(engine_state, stack, closure);
Ok(input
.into_iter()
.map_while(move |value| match closure.run_with_value(value) {
Ok(data) => {
let value = data.into_value(head);
(!value.is_nothing()).then_some(value)
}
Err(_) => None,
})
.fuse()
.into_pipeline_data(head, engine_state.ctrlc.clone()))
}
PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()),
PipelineData::ExternalStream {
stdout: Some(stream),
..
} => {
let mut closure = ClosureEval::new(engine_state, stack, closure);
Ok(stream
.into_iter()
.map_while(move |value| {
let value = value.ok()?;
match closure.run_with_value(value) {
Ok(data) => {
let value = data.into_value(head);
(!value.is_nothing()).then_some(value)
}
match closure
.run_with_value(value)
.and_then(|data| data.into_value(head))
{
Ok(value) => (!value.is_nothing()).then_some(value),
Err(_) => None,
}
})
.fuse()
.into_pipeline_data(head, engine_state.ctrlc.clone()))
}
PipelineData::ByteStream(stream, ..) => {
let span = stream.span();
if let Some(chunks) = stream.chunks() {
let mut closure = ClosureEval::new(engine_state, stack, closure);
Ok(chunks
.map_while(move |value| {
let value = value.ok()?;
match closure
.run_with_value(value)
.and_then(|data| data.into_value(span))
{
Ok(value) => (!value.is_nothing()).then_some(value),
Err(_) => None,
}
})
.fuse()
.into_pipeline_data(head, engine_state.ctrlc.clone()))
} else {
Ok(PipelineData::Empty)
}
}
// This match allows non-iterables to be accepted,
// which is currently considered undesirable (Nov 2022).
PipelineData::Value(value, ..) => {

2
crates/nu-cmd-extra/src/extra/filters/roll/roll_down.rs
View File

@ -56,7 +56,7 @@ impl Command for RollDown {
let by: Option<usize> = call.get_flag(engine_state, stack, "by")?;
let metadata = input.metadata();
let value = input.into_value(call.head);
let value = input.into_value(call.head)?;
let rotated_value = vertical_rotate_value(value, by, VerticalDirection::Down)?;
Ok(rotated_value.into_pipeline_data().set_metadata(metadata))

2
crates/nu-cmd-extra/src/extra/filters/roll/roll_left.rs
View File

@ -94,7 +94,7 @@ impl Command for RollLeft {
let metadata = input.metadata();
let cells_only = call.has_flag(engine_state, stack, "cells-only")?;
let value = input.into_value(call.head);
let value = input.into_value(call.head)?;
let rotated_value =
horizontal_rotate_value(value, by, cells_only, &HorizontalDirection::Left)?;

2
crates/nu-cmd-extra/src/extra/filters/roll/roll_right.rs
View File

@ -94,7 +94,7 @@ impl Command for RollRight {
let metadata = input.metadata();
let cells_only = call.has_flag(engine_state, stack, "cells-only")?;
let value = input.into_value(call.head);
let value = input.into_value(call.head)?;
let rotated_value =
horizontal_rotate_value(value, by, cells_only, &HorizontalDirection::Right)?;

2
crates/nu-cmd-extra/src/extra/filters/roll/roll_up.rs
View File

@ -56,7 +56,7 @@ impl Command for RollUp {
let by: Option<usize> = call.get_flag(engine_state, stack, "by")?;
let metadata = input.metadata();
let value = input.into_value(call.head);
let value = input.into_value(call.head)?;
let rotated_value = vertical_rotate_value(value, by, VerticalDirection::Up)?;
Ok(rotated_value.into_pipeline_data().set_metadata(metadata))

2
crates/nu-cmd-extra/src/extra/filters/update_cells.rs
View File

@ -152,7 +152,7 @@ impl Iterator for UpdateCellIterator {
fn eval_value(closure: &mut ClosureEval, span: Span, value: Value) -> Value {
closure
.run_with_value(value)
.map(|data| data.into_value(span))
.and_then(|data| data.into_value(span))
.unwrap_or_else(|err| Value::error(err, span))
}

2
crates/nu-cmd-extra/src/extra/strings/format/command.rs
View File

@ -39,7 +39,7 @@ impl Command for FormatPattern {
let mut working_set = StateWorkingSet::new(engine_state);
let specified_pattern: Result<Value, ShellError> = call.req(engine_state, stack, 0);
let input_val = input.into_value(call.head);
let input_val = input.into_value(call.head)?;
// add '$it' variable to support format like this: $it.column1.column2.
let it_id = working_set.add_variable(b"$it".to_vec(), call.head, Type::Any, false);
stack.add_var(it_id, input_val.clone());

160
crates/nu-cmd-extra/tests/commands/bytes/starts_with.rs
View File

@ -19,102 +19,102 @@ fn basic_string_fails() {
assert_eq!(actual.out, "");
}
#[test]
fn short_stream_binary() {
let actual = nu!(r#"
nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101]
"#);
// #[test]
// fn short_stream_binary() {
// let actual = nu!(r#"
// nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101]
// "#);
assert_eq!(actual.out, "true");
}
// assert_eq!(actual.out, "true");
// }
#[test]
fn short_stream_mismatch() {
let actual = nu!(r#"
nu --testbin repeater (0x[010203]) 5 | bytes starts-with 0x[010204]
"#);
// #[test]
// fn short_stream_mismatch() {
// let actual = nu!(r#"
// nu --testbin repeater (0x[010203]) 5 | bytes starts-with 0x[010204]
// "#);
assert_eq!(actual.out, "false");
}
// assert_eq!(actual.out, "false");
// }
#[test]
fn short_stream_binary_overflow() {
let actual = nu!(r#"
nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101010101]
"#);
// #[test]
// fn short_stream_binary_overflow() {
// let actual = nu!(r#"
// nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101010101]
// "#);
assert_eq!(actual.out, "false");
}
// assert_eq!(actual.out, "false");
// }
#[test]
fn long_stream_binary() {
let actual = nu!(r#"
nu --testbin repeater (0x[01]) 32768 | bytes starts-with 0x[010101]
"#);
// #[test]
// fn long_stream_binary() {
// let actual = nu!(r#"
// nu --testbin repeater (0x[01]) 32768 | bytes starts-with 0x[010101]
// "#);
assert_eq!(actual.out, "true");
}
// assert_eq!(actual.out, "true");
// }
#[test]
fn long_stream_binary_overflow() {
// .. ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow
let actual = nu!(r#"
nu --testbin repeater (0x[01]) 32768 | bytes starts-with (0..32768 | each {|| 0x[01] } | bytes collect)
"#);
// #[test]
// fn long_stream_binary_overflow() {
// // .. ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow
// let actual = nu!(r#"
// nu --testbin repeater (0x[01]) 32768 | bytes starts-with (0..32768 | each {|| 0x[01] } | bytes collect)
// "#);
assert_eq!(actual.out, "false");
}
// assert_eq!(actual.out, "false");
// }
#[test]
fn long_stream_binary_exact() {
// ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow
let actual = nu!(r#"
nu --testbin repeater (0x[01020304]) 8192 | bytes starts-with (0..<8192 | each {|| 0x[01020304] } | bytes collect)
"#);
// #[test]
// fn long_stream_binary_exact() {
// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow
// let actual = nu!(r#"
// nu --testbin repeater (0x[01020304]) 8192 | bytes starts-with (0..<8192 | each {|| 0x[01020304] } | bytes collect)
// "#);
assert_eq!(actual.out, "true");
}
// assert_eq!(actual.out, "true");
// }
#[test]
fn long_stream_string_exact() {
// ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow
let actual = nu!(r#"
nu --testbin repeater hell 8192 | bytes starts-with (0..<8192 | each {|| "hell" | into binary } | bytes collect)
"#);
// #[test]
// fn long_stream_string_exact() {
// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow
// let actual = nu!(r#"
// nu --testbin repeater hell 8192 | bytes starts-with (0..<8192 | each {|| "hell" | into binary } | bytes collect)
// "#);
assert_eq!(actual.out, "true");
}
// assert_eq!(actual.out, "true");
// }
#[test]
fn long_stream_mixed_exact() {
// ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow
let actual = nu!(r#"
let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect)
let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect)
// #[test]
// fn long_stream_mixed_exact() {
// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow
// let actual = nu!(r#"
// let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect)
// let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect)
nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg)
"#);
// nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg)
// "#);
assert_eq!(
actual.err, "",
"invocation failed. command line limit likely reached"
);
assert_eq!(actual.out, "true");
}
// assert_eq!(
// actual.err, "",
// "invocation failed. command line limit likely reached"
// );
// assert_eq!(actual.out, "true");
// }
#[test]
fn long_stream_mixed_overflow() {
// ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow
let actual = nu!(r#"
let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect)
let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect)
// #[test]
// fn long_stream_mixed_overflow() {
// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow
// let actual = nu!(r#"
// let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect)
// let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect)
nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg 0x[01])
"#);
// nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg 0x[01])
// "#);
assert_eq!(
actual.err, "",
"invocation failed. command line limit likely reached"
);
assert_eq!(actual.out, "false");
}
// assert_eq!(
// actual.err, "",
// "invocation failed. command line limit likely reached"
// );
// assert_eq!(actual.out, "false");
// }