2024-05-15 03:48:27 +02:00
|
|
|
use rstest::rstest;
|
|
|
|
|
2024-02-25 23:32:50 +01:00
|
|
|
use nu_test_support::nu_with_plugins;
|
|
|
|
use pretty_assertions::assert_eq;
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn seq_produces_stream() {
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"example seq 1 5 | describe"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "list<int> (stream)");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn seq_describe_no_collect_succeeds_without_error() {
|
|
|
|
// This tests to ensure that there's no error if the stream is suddenly closed
|
2024-03-13 12:49:53 +01:00
|
|
|
// Test several times, because this can cause different errors depending on what is written
|
|
|
|
// when the engine stops running, especially if there's partial output
|
|
|
|
for _ in 0..10 {
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"example seq 1 5 | describe --no-collect"
|
2024-03-13 12:49:53 +01:00
|
|
|
);
|
2024-02-25 23:32:50 +01:00
|
|
|
|
2024-03-13 12:49:53 +01:00
|
|
|
assert_eq!(actual.out, "stream");
|
|
|
|
assert_eq!(actual.err, "");
|
|
|
|
}
|
2024-02-25 23:32:50 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn seq_stream_collects_to_correct_list() {
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"example seq 1 5 | to json --raw"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "[1,2,3,4,5]");
|
|
|
|
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"example seq 1 0 | to json --raw"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "[]");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn seq_big_stream() {
|
|
|
|
// Testing big streams helps to ensure there are no deadlocking bugs
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"example seq 1 100000 | length"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "100000");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn sum_accepts_list_of_int() {
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"[1 2 3] | example sum"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "6");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn sum_accepts_list_of_float() {
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"[1.0 2.0 3.5] | example sum"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "6.5");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn sum_accepts_stream_of_int() {
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"seq 1 5 | example sum"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "15");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn sum_accepts_stream_of_float() {
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"seq 1 5 | into float | example sum"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "15");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn sum_big_stream() {
|
|
|
|
// Testing big streams helps to ensure there are no deadlocking bugs
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"seq 1 100000 | example sum"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "5000050000");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
fn collect_bytes_accepts_list_of_string() {
|
2024-02-25 23:32:50 +01:00
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
"[a b] | example collect-bytes"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "ab");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
fn collect_bytes_accepts_list_of_binary() {
|
2024-02-25 23:32:50 +01:00
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
"[0x[41] 0x[42]] | example collect-bytes"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "AB");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
fn collect_bytes_produces_byte_stream() {
|
2024-02-25 23:32:50 +01:00
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
"[a b c] | example collect-bytes | describe"
|
2024-02-25 23:32:50 +01:00
|
|
|
);
|
|
|
|
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
assert_eq!(actual.out, "byte stream");
|
2024-02-25 23:32:50 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
fn collect_bytes_big_stream() {
|
2024-02-25 23:32:50 +01:00
|
|
|
// This in particular helps to ensure that a big stream can be both read and written at the same
|
|
|
|
// time without deadlocking
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
2024-02-25 23:32:50 +01:00
|
|
|
r#"(
|
|
|
|
seq 1 10000 |
|
Replace `ExternalStream` with new `ByteStream` type (#12774)
# Description
This PR introduces a `ByteStream` type which is a `Read`-able stream of
bytes. Internally, it has an enum over three different byte stream
sources:
```rust
pub enum ByteStreamSource {
Read(Box<dyn Read + Send + 'static>),
File(File),
Child(ChildProcess),
}
```
This is in comparison to the current `RawStream` type, which is an
`Iterator<Item = Vec<u8>>` and has to allocate for each read chunk.
Currently, `PipelineData::ExternalStream` serves a weird dual role where
it is either external command output or a wrapper around `RawStream`.
`ByteStream` makes this distinction more clear (via `ByteStreamSource`)
and replaces `PipelineData::ExternalStream` in this PR:
```rust
pub enum PipelineData {
Empty,
Value(Value, Option<PipelineMetadata>),
ListStream(ListStream, Option<PipelineMetadata>),
ByteStream(ByteStream, Option<PipelineMetadata>),
}
```
The PR is relatively large, but a decent amount of it is just repetitive
changes.
This PR fixes #7017, fixes #10763, and fixes #12369.
This PR also improves performance when piping external commands. Nushell
should, in most cases, have competitive pipeline throughput compared to,
e.g., bash.
| Command | Before (MB/s) | After (MB/s) | Bash (MB/s) |
| -------------------------------------------------- | -------------:|
------------:| -----------:|
| `throughput \| rg 'x'` | 3059 | 3744 | 3739 |
| `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 |
# User-Facing Changes
- This is a breaking change for the plugin communication protocol,
because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`.
Plugins now only have to deal with a single input stream, as opposed to
the previous three streams: stdout, stderr, and exit code.
- The output of `describe` has been changed for external/byte streams.
- Temporary breaking change: `bytes starts-with` no longer works with
byte streams. This is to keep the PR smaller, and `bytes ends-with`
already does not work on byte streams.
- If a process core dumped, then instead of having a `Value::Error` in
the `exit_code` column of the output returned from `complete`, it now is
a `Value::Int` with the negation of the signal number.
# After Submitting
- Update docs and book as necessary
- Release notes (e.g., plugin protocol changes)
- Adapt/convert commands to work with byte streams (high priority is
`str length`, `bytes starts-with`, and maybe `bytes ends-with`).
- Refactor the `tee` code, Devyn has already done some work on this.
---------
Co-authored-by: Devyn Cairns <devyn.cairns@gmail.com>
2024-05-16 16:11:18 +02:00
|
|
|
each {|i| ($i | into string) ++ (char newline) } |
|
|
|
|
example collect-bytes |
|
2024-02-25 23:32:50 +01:00
|
|
|
lines |
|
|
|
|
length
|
|
|
|
)"#
|
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "10000");
|
|
|
|
}
|
Add support for engine calls from plugins (#12029)
# Description
This allows plugins to make calls back to the engine to get config,
evaluate closures, and do other things that must be done within the
engine process.
Engine calls can both produce and consume streams as necessary. Closures
passed to plugins can both accept stream input and produce stream output
sent back to the plugin.
Engine calls referring to a plugin call's context can be processed as
long either the response hasn't been received, or the response created
streams that haven't ended yet.
This is a breaking API change for plugins. There are some pretty major
changes to the interface that plugins must implement, including:
1. Plugins now run with `&self` and must be `Sync`. Executing multiple
plugin calls in parallel is supported, and there's a chance that a
closure passed to a plugin could invoke the same plugin. Supporting
state across plugin invocations is left up to the plugin author to do in
whichever way they feel best, but the plugin object itself is still
shared. Even though the engine doesn't run multiple plugin calls through
the same process yet, I still considered it important to break the API
in this way at this stage. We might want to consider an optional
threadpool feature for performance.
2. Plugins take a reference to `EngineInterface`, which can be cloned.
This interface allows plugins to make calls back to the engine,
including for getting config and running closures.
3. Plugins no longer take the `config` parameter. This can be accessed
from the interface via the `.get_plugin_config()` engine call.
# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->
Not only does this have plugin protocol changes, it will require plugins
to make some code changes before they will work again. But on the plus
side, the engine call feature is extensible, and we can add more things
to it as needed.
Plugin maintainers will have to change the trait signature at the very
least. If they were using `config`, they will have to call
`engine.get_plugin_config()` instead.
If they were using the mutable reference to the plugin, they will have
to come up with some strategy to work around it (for example, for `Inc`
I just cloned it). This shouldn't be such a big deal at the moment as
it's not like plugins have ever run as daemons with persistent state in
the past, and they don't in this PR either. But I thought it was
important to make the change before we support plugins as daemons, as an
exclusive mutable reference is not compatible with parallel plugin
calls.
I suggest this gets merged sometime *after* the current pending release,
so that we have some time to adjust to the previous plugin protocol
changes that don't require code changes before making ones that do.
# Tests + Formatting
- :green_circle: `toolkit fmt`
- :green_circle: `toolkit clippy`
- :green_circle: `toolkit test`
- :green_circle: `toolkit test stdlib`
# After Submitting
I will document the additional protocol features (`EngineCall`,
`EngineCallResponse`), and constraints on plugin call processing if
engine calls are used - basically, to be aware that an engine call could
result in a nested plugin call, so the plugin should be able to handle
that.
2024-03-09 18:26:30 +01:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn for_each_prints_on_stderr() {
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"[a b c] | example for-each { $in }"
|
Add support for engine calls from plugins (#12029)
# Description
This allows plugins to make calls back to the engine to get config,
evaluate closures, and do other things that must be done within the
engine process.
Engine calls can both produce and consume streams as necessary. Closures
passed to plugins can both accept stream input and produce stream output
sent back to the plugin.
Engine calls referring to a plugin call's context can be processed as
long either the response hasn't been received, or the response created
streams that haven't ended yet.
This is a breaking API change for plugins. There are some pretty major
changes to the interface that plugins must implement, including:
1. Plugins now run with `&self` and must be `Sync`. Executing multiple
plugin calls in parallel is supported, and there's a chance that a
closure passed to a plugin could invoke the same plugin. Supporting
state across plugin invocations is left up to the plugin author to do in
whichever way they feel best, but the plugin object itself is still
shared. Even though the engine doesn't run multiple plugin calls through
the same process yet, I still considered it important to break the API
in this way at this stage. We might want to consider an optional
threadpool feature for performance.
2. Plugins take a reference to `EngineInterface`, which can be cloned.
This interface allows plugins to make calls back to the engine,
including for getting config and running closures.
3. Plugins no longer take the `config` parameter. This can be accessed
from the interface via the `.get_plugin_config()` engine call.
# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->
Not only does this have plugin protocol changes, it will require plugins
to make some code changes before they will work again. But on the plus
side, the engine call feature is extensible, and we can add more things
to it as needed.
Plugin maintainers will have to change the trait signature at the very
least. If they were using `config`, they will have to call
`engine.get_plugin_config()` instead.
If they were using the mutable reference to the plugin, they will have
to come up with some strategy to work around it (for example, for `Inc`
I just cloned it). This shouldn't be such a big deal at the moment as
it's not like plugins have ever run as daemons with persistent state in
the past, and they don't in this PR either. But I thought it was
important to make the change before we support plugins as daemons, as an
exclusive mutable reference is not compatible with parallel plugin
calls.
I suggest this gets merged sometime *after* the current pending release,
so that we have some time to adjust to the previous plugin protocol
changes that don't require code changes before making ones that do.
# Tests + Formatting
- :green_circle: `toolkit fmt`
- :green_circle: `toolkit clippy`
- :green_circle: `toolkit test`
- :green_circle: `toolkit test stdlib`
# After Submitting
I will document the additional protocol features (`EngineCall`,
`EngineCallResponse`), and constraints on plugin call processing if
engine calls are used - basically, to be aware that an engine call could
result in a nested plugin call, so the plugin should be able to handle
that.
2024-03-09 18:26:30 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.err, "a\nb\nc\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn generate_sequence() {
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
2024-03-19 18:36:46 +01:00
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
"example generate 0 { |i| if $i <= 10 { {out: $i, next: ($i + 2)} } } | to json --raw"
|
Add support for engine calls from plugins (#12029)
# Description
This allows plugins to make calls back to the engine to get config,
evaluate closures, and do other things that must be done within the
engine process.
Engine calls can both produce and consume streams as necessary. Closures
passed to plugins can both accept stream input and produce stream output
sent back to the plugin.
Engine calls referring to a plugin call's context can be processed as
long either the response hasn't been received, or the response created
streams that haven't ended yet.
This is a breaking API change for plugins. There are some pretty major
changes to the interface that plugins must implement, including:
1. Plugins now run with `&self` and must be `Sync`. Executing multiple
plugin calls in parallel is supported, and there's a chance that a
closure passed to a plugin could invoke the same plugin. Supporting
state across plugin invocations is left up to the plugin author to do in
whichever way they feel best, but the plugin object itself is still
shared. Even though the engine doesn't run multiple plugin calls through
the same process yet, I still considered it important to break the API
in this way at this stage. We might want to consider an optional
threadpool feature for performance.
2. Plugins take a reference to `EngineInterface`, which can be cloned.
This interface allows plugins to make calls back to the engine,
including for getting config and running closures.
3. Plugins no longer take the `config` parameter. This can be accessed
from the interface via the `.get_plugin_config()` engine call.
# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->
Not only does this have plugin protocol changes, it will require plugins
to make some code changes before they will work again. But on the plus
side, the engine call feature is extensible, and we can add more things
to it as needed.
Plugin maintainers will have to change the trait signature at the very
least. If they were using `config`, they will have to call
`engine.get_plugin_config()` instead.
If they were using the mutable reference to the plugin, they will have
to come up with some strategy to work around it (for example, for `Inc`
I just cloned it). This shouldn't be such a big deal at the moment as
it's not like plugins have ever run as daemons with persistent state in
the past, and they don't in this PR either. But I thought it was
important to make the change before we support plugins as daemons, as an
exclusive mutable reference is not compatible with parallel plugin
calls.
I suggest this gets merged sometime *after* the current pending release,
so that we have some time to adjust to the previous plugin protocol
changes that don't require code changes before making ones that do.
# Tests + Formatting
- :green_circle: `toolkit fmt`
- :green_circle: `toolkit clippy`
- :green_circle: `toolkit test`
- :green_circle: `toolkit test stdlib`
# After Submitting
I will document the additional protocol features (`EngineCall`,
`EngineCallResponse`), and constraints on plugin call processing if
engine calls are used - basically, to be aware that an engine call could
result in a nested plugin call, so the plugin should be able to handle
that.
2024-03-09 18:26:30 +01:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(actual.out, "[0,2,4,6,8,10]");
|
|
|
|
}
|
2024-05-15 03:48:27 +02:00
|
|
|
|
|
|
|
#[rstest]
|
|
|
|
#[timeout(std::time::Duration::from_secs(6))]
|
|
|
|
fn echo_interactivity_on_slow_pipelines() {
|
|
|
|
// This test works by putting 0 on the upstream immediately, followed by 1 after 10 seconds.
|
|
|
|
// If values aren't streamed to the plugin as they become available, `example echo` won't emit
|
|
|
|
// anything until both 0 and 1 are available. The desired behavior is that `example echo` gets
|
|
|
|
// the 0 immediately, which is consumed by `first`, allowing the pipeline to terminate early.
|
|
|
|
let actual = nu_with_plugins!(
|
|
|
|
cwd: "tests/fixtures/formats",
|
|
|
|
plugin: ("nu_plugin_example"),
|
|
|
|
r#"[1] | each { |n| sleep 10sec; $n } | prepend 0 | example echo | first"#
|
|
|
|
);
|
|
|
|
assert_eq!(actual.out, "0");
|
|
|
|
}
|