From 6fd854ed9f25070e1f5456b4c96d76d5283f188d Mon Sep 17 00:00:00 2001 From: Ian Manske Date: Thu, 16 May 2024 14:11:18 +0000 Subject: [PATCH] Replace `ExternalStream` with new `ByteStream` type (#12774) # Description This PR introduces a `ByteStream` type which is a `Read`-able stream of bytes. Internally, it has an enum over three different byte stream sources: ```rust pub enum ByteStreamSource { Read(Box), File(File), Child(ChildProcess), } ``` This is in comparison to the current `RawStream` type, which is an `Iterator>` and has to allocate for each read chunk. Currently, `PipelineData::ExternalStream` serves a weird dual role where it is either external command output or a wrapper around `RawStream`. `ByteStream` makes this distinction more clear (via `ByteStreamSource`) and replaces `PipelineData::ExternalStream` in this PR: ```rust pub enum PipelineData { Empty, Value(Value, Option), ListStream(ListStream, Option), ByteStream(ByteStream, Option), } ``` The PR is relatively large, but a decent amount of it is just repetitive changes. This PR fixes #7017, fixes #10763, and fixes #12369. This PR also improves performance when piping external commands. Nushell should, in most cases, have competitive pipeline throughput compared to, e.g., bash. | Command | Before (MB/s) | After (MB/s) | Bash (MB/s) | | -------------------------------------------------- | -------------:| ------------:| -----------:| | `throughput \| rg 'x'` | 3059 | 3744 | 3739 | | `throughput \| nu --testbin relay o> /dev/null` | 3508 | 8087 | 8136 | # User-Facing Changes - This is a breaking change for the plugin communication protocol, because the `ExternalStreamInfo` was replaced with `ByteStreamInfo`. Plugins now only have to deal with a single input stream, as opposed to the previous three streams: stdout, stderr, and exit code. - The output of `describe` has been changed for external/byte streams. - Temporary breaking change: `bytes starts-with` no longer works with byte streams. This is to keep the PR smaller, and `bytes ends-with` already does not work on byte streams. - If a process core dumped, then instead of having a `Value::Error` in the `exit_code` column of the output returned from `complete`, it now is a `Value::Int` with the negation of the signal number. # After Submitting - Update docs and book as necessary - Release notes (e.g., plugin protocol changes) - Adapt/convert commands to work with byte streams (high priority is `str length`, `bytes starts-with`, and maybe `bytes ends-with`). - Refactor the `tee` code, Devyn has already done some work on this. --------- Co-authored-by: Devyn Cairns --- Cargo.lock | 1 + crates/nu-cli/src/completions/completer.rs | 5 +- .../src/completions/custom_completions.rs | 84 +- crates/nu-cli/src/config_files.rs | 5 +- crates/nu-cli/src/eval_cmds.rs | 8 +- crates/nu-cli/src/eval_file.rs | 23 +- crates/nu-cli/src/menus/menu_completions.rs | 3 +- crates/nu-cli/src/util.rs | 144 +- .../src/dataframe/eager/cast.rs | 2 +- .../src/dataframe/eager/filter_with.rs | 3 +- .../src/dataframe/eager/first.rs | 2 +- .../src/dataframe/eager/last.rs | 2 +- .../src/dataframe/eager/rename.rs | 3 +- .../src/dataframe/eager/to_nu.rs | 2 +- .../src/dataframe/eager/with_column.rs | 3 +- .../expressions/expressions_macro.rs | 4 +- .../src/dataframe/expressions/otherwise.rs | 2 +- .../src/dataframe/expressions/quantile.rs | 2 +- .../src/dataframe/expressions/when.rs | 2 +- .../src/dataframe/lazy/explode.rs | 2 +- .../src/dataframe/lazy/fill_nan.rs | 2 +- .../src/dataframe/lazy/fill_null.rs | 2 +- .../src/dataframe/lazy/join.rs | 2 +- .../src/dataframe/lazy/quantile.rs | 2 +- .../src/dataframe/series/masks/is_not_null.rs | 2 +- .../src/dataframe/series/masks/is_null.rs | 2 +- .../src/dataframe/series/n_unique.rs | 2 +- .../src/dataframe/series/shift.rs | 3 +- .../src/dataframe/series/unique.rs | 3 +- .../src/dataframe/test_dataframe.rs | 3 +- .../src/dataframe/values/nu_dataframe/mod.rs | 2 +- .../src/dataframe/values/nu_expression/mod.rs | 2 +- .../src/dataframe/values/nu_lazyframe/mod.rs | 2 +- .../dataframe/values/nu_lazygroupby/mod.rs | 2 +- crates/nu-cmd-extra/src/extra/bits/into.rs | 22 +- .../src/extra/filters/each_while.rs | 50 +- .../src/extra/filters/roll/roll_down.rs | 2 +- .../src/extra/filters/roll/roll_left.rs | 2 +- .../src/extra/filters/roll/roll_right.rs | 2 +- .../src/extra/filters/roll/roll_up.rs | 2 +- .../src/extra/filters/update_cells.rs | 2 +- .../src/extra/strings/format/command.rs | 2 +- .../tests/commands/bytes/starts_with.rs | 160 +-- .../nu-cmd-lang/src/core_commands/collect.rs | 2 +- .../nu-cmd-lang/src/core_commands/describe.rs | 77 +- crates/nu-cmd-lang/src/core_commands/do_.rs | 181 ++- crates/nu-cmd-lang/src/core_commands/for_.rs | 30 +- crates/nu-cmd-lang/src/core_commands/let_.rs | 2 +- crates/nu-cmd-lang/src/core_commands/loop_.rs | 12 +- crates/nu-cmd-lang/src/core_commands/mut_.rs | 2 +- crates/nu-cmd-lang/src/core_commands/try_.rs | 7 +- .../nu-cmd-lang/src/core_commands/while_.rs | 18 +- crates/nu-cmd-lang/src/example_support.rs | 7 +- crates/nu-color-config/src/style_computer.rs | 6 +- crates/nu-command/src/bytes/starts_with.rs | 64 +- crates/nu-command/src/charting/histogram.rs | 2 +- .../nu-command/src/conversions/into/binary.rs | 28 +- .../src/conversions/into/cell_path.rs | 6 +- .../nu-command/src/conversions/into/glob.rs | 20 +- .../nu-command/src/conversions/into/record.rs | 2 +- .../nu-command/src/conversions/into/string.rs | 30 +- .../nu-command/src/database/values/sqlite.rs | 2 +- crates/nu-command/src/debug/inspect.rs | 2 +- crates/nu-command/src/debug/timeit.rs | 5 +- crates/nu-command/src/filesystem/open.rs | 30 +- crates/nu-command/src/filesystem/save.rs | 240 ++-- crates/nu-command/src/filters/columns.rs | 8 +- crates/nu-command/src/filters/drop/column.rs | 6 +- crates/nu-command/src/filters/each.rs | 64 +- crates/nu-command/src/filters/empty.rs | 38 +- crates/nu-command/src/filters/filter.rs | 63 +- crates/nu-command/src/filters/find.rs | 72 +- crates/nu-command/src/filters/first.rs | 6 +- crates/nu-command/src/filters/get.rs | 2 +- crates/nu-command/src/filters/group_by.rs | 2 +- crates/nu-command/src/filters/headers.rs | 2 +- crates/nu-command/src/filters/insert.rs | 10 +- crates/nu-command/src/filters/items.rs | 25 +- crates/nu-command/src/filters/join.rs | 2 +- crates/nu-command/src/filters/last.rs | 14 +- crates/nu-command/src/filters/lines.rs | 175 +-- crates/nu-command/src/filters/par_each.rs | 102 +- crates/nu-command/src/filters/reduce.rs | 2 +- crates/nu-command/src/filters/reject.rs | 2 +- crates/nu-command/src/filters/skip/skip_.rs | 7 +- .../nu-command/src/filters/skip/skip_until.rs | 3 +- .../nu-command/src/filters/skip/skip_while.rs | 3 +- crates/nu-command/src/filters/take/take_.rs | 14 +- .../nu-command/src/filters/take/take_until.rs | 3 +- .../nu-command/src/filters/take/take_while.rs | 3 +- crates/nu-command/src/filters/tee.rs | 460 +++++-- crates/nu-command/src/filters/update.rs | 8 +- crates/nu-command/src/filters/upsert.rs | 18 +- crates/nu-command/src/filters/utils.rs | 2 +- crates/nu-command/src/filters/values.rs | 8 +- crates/nu-command/src/filters/where_.rs | 11 +- crates/nu-command/src/filters/wrap.rs | 4 +- crates/nu-command/src/formats/from/json.rs | 2 +- crates/nu-command/src/formats/from/msgpack.rs | 73 +- .../nu-command/src/formats/from/msgpackz.rs | 22 +- crates/nu-command/src/formats/from/ods.rs | 40 +- crates/nu-command/src/formats/from/xlsx.rs | 38 +- crates/nu-command/src/formats/to/delimited.rs | 2 +- crates/nu-command/src/formats/to/json.rs | 2 +- crates/nu-command/src/formats/to/msgpack.rs | 2 +- crates/nu-command/src/formats/to/msgpackz.rs | 2 +- crates/nu-command/src/formats/to/nuon.rs | 2 +- crates/nu-command/src/formats/to/text.rs | 79 +- crates/nu-command/src/formats/to/toml.rs | 2 +- crates/nu-command/src/formats/to/xml.rs | 2 +- crates/nu-command/src/formats/to/yaml.rs | 2 +- crates/nu-command/src/generators/generate.rs | 18 +- crates/nu-command/src/hash/generic_digest.rs | 60 +- crates/nu-command/src/misc/tutor.rs | 10 +- crates/nu-command/src/network/http/client.rs | 39 +- crates/nu-command/src/network/url/parse.rs | 2 +- crates/nu-command/src/path/join.rs | 4 +- crates/nu-command/src/progress_bar.rs | 12 - .../src/strings/encode_decode/decode.rs | 12 +- .../src/strings/encode_decode/encode.rs | 9 +- crates/nu-command/src/strings/parse.rs | 37 +- crates/nu-command/src/system/complete.rs | 98 +- crates/nu-command/src/system/nu_check.rs | 16 +- crates/nu-command/src/system/run_external.rs | 369 ++--- crates/nu-command/src/viewers/table.rs | 47 +- .../tests/format_conversions/csv.rs | 1 + .../tests/format_conversions/tsv.rs | 1 + crates/nu-engine/src/documentation.rs | 4 +- crates/nu-engine/src/env.rs | 17 +- crates/nu-engine/src/eval.rs | 118 +- crates/nu-explore/src/nu_common/value.rs | 78 +- crates/nu-plugin-core/src/interface/mod.rs | 164 +-- crates/nu-plugin-core/src/interface/tests.rs | 209 +-- crates/nu-plugin-engine/src/context.rs | 2 +- crates/nu-plugin-engine/src/init.rs | 2 +- crates/nu-plugin-engine/src/interface/mod.rs | 10 +- .../nu-plugin-engine/src/interface/tests.rs | 61 +- crates/nu-plugin-protocol/src/lib.rs | 59 +- crates/nu-plugin-test-support/src/lib.rs | 2 +- .../nu-plugin-test-support/src/plugin_test.rs | 65 +- .../tests/custom_value/mod.rs | 2 +- .../nu-plugin-test-support/tests/hello/mod.rs | 2 +- .../tests/lowercase/mod.rs | 2 +- crates/nu-plugin/src/plugin/command.rs | 2 +- crates/nu-plugin/src/plugin/interface/mod.rs | 6 +- .../nu-plugin/src/plugin/interface/tests.rs | 20 +- crates/nu-plugin/src/plugin/mod.rs | 2 +- crates/nu-protocol/Cargo.toml | 4 + .../src/debugger/debugger_trait.rs | 6 +- crates/nu-protocol/src/debugger/profiler.rs | 8 +- crates/nu-protocol/src/errors/shell_error.rs | 70 +- crates/nu-protocol/src/eval_const.rs | 7 +- crates/nu-protocol/src/lib.rs | 7 +- .../nu-protocol/src/pipeline/byte_stream.rs | 822 ++++++++++++ .../list_stream.rs | 0 .../{pipeline_data => pipeline}/metadata.rs | 0 crates/nu-protocol/src/pipeline/mod.rs | 11 + .../{pipeline_data => pipeline}/out_dest.rs | 8 +- .../nu-protocol/src/pipeline/pipeline_data.rs | 725 ++++++++++ crates/nu-protocol/src/pipeline_data/mod.rs | 1185 ----------------- .../src/pipeline_data/raw_stream.rs | 176 --- crates/nu-protocol/src/process/child.rs | 294 ++++ crates/nu-protocol/src/process/exit_status.rs | 64 + crates/nu-protocol/src/process/mod.rs | 5 + crates/nu-protocol/src/util.rs | 52 - .../nu-protocol/tests/test_pipeline_data.rs | 2 +- crates/nu-system/src/foreground.rs | 6 +- .../{collect_external.rs => collect_bytes.rs} | 39 +- crates/nu_plugin_example/src/commands/mod.rs | 4 +- crates/nu_plugin_example/src/lib.rs | 2 +- crates/nu_plugin_polars/src/cache/rm.rs | 2 +- .../src/dataframe/eager/to_arrow.rs | 2 +- .../src/dataframe/eager/to_avro.rs | 2 +- .../src/dataframe/eager/to_csv.rs | 2 +- .../src/dataframe/eager/to_json_lines.rs | 2 +- .../src/dataframe/eager/to_nu.rs | 2 +- .../src/dataframe/eager/to_parquet.rs | 2 +- .../expressions/expressions_macro.rs | 4 +- .../src/dataframe/expressions/is_in.rs | 3 +- .../src/dataframe/expressions/otherwise.rs | 2 +- .../src/dataframe/expressions/when.rs | 2 +- .../src/dataframe/lazy/cast.rs | 2 +- .../src/dataframe/lazy/collect.rs | 2 +- .../src/dataframe/lazy/explode.rs | 11 +- .../src/dataframe/lazy/fetch.rs | 2 +- .../src/dataframe/lazy/fill_nan.rs | 2 +- .../src/dataframe/lazy/fill_null.rs | 2 +- .../src/dataframe/lazy/filter.rs | 2 +- .../src/dataframe/lazy/filter_with.rs | 2 +- .../src/dataframe/lazy/first.rs | 2 +- .../src/dataframe/lazy/groupby.rs | 2 +- .../src/dataframe/lazy/join.rs | 2 +- .../src/dataframe/lazy/last.rs | 2 +- .../src/dataframe/lazy/median.rs | 2 +- .../src/dataframe/lazy/quantile.rs | 2 +- .../src/dataframe/lazy/rename.rs | 2 +- .../src/dataframe/lazy/select.rs | 2 +- .../src/dataframe/lazy/sort_by_expr.rs | 2 +- .../src/dataframe/lazy/with_column.rs | 2 +- .../src/dataframe/series/masks/is_not_null.rs | 3 +- .../src/dataframe/series/masks/is_null.rs | 3 +- .../src/dataframe/series/n_unique.rs | 3 +- .../src/dataframe/series/shift.rs | 2 +- .../src/dataframe/series/unique.rs | 2 +- .../src/dataframe/values/mod.rs | 6 +- .../src/dataframe/values/nu_dataframe/mod.rs | 2 +- .../src/dataframe/values/nu_lazyframe/mod.rs | 2 +- src/main.rs | 29 +- tests/plugins/stream.rs | 21 +- tests/shell/pipeline/commands/internal.rs | 4 +- 210 files changed, 3955 insertions(+), 4012 deletions(-) create mode 100644 crates/nu-protocol/src/pipeline/byte_stream.rs rename crates/nu-protocol/src/{pipeline_data => pipeline}/list_stream.rs (100%) rename crates/nu-protocol/src/{pipeline_data => pipeline}/metadata.rs (100%) create mode 100644 crates/nu-protocol/src/pipeline/mod.rs rename crates/nu-protocol/src/{pipeline_data => pipeline}/out_dest.rs (81%) create mode 100644 crates/nu-protocol/src/pipeline/pipeline_data.rs delete mode 100644 crates/nu-protocol/src/pipeline_data/mod.rs delete mode 100644 crates/nu-protocol/src/pipeline_data/raw_stream.rs create mode 100644 crates/nu-protocol/src/process/child.rs create mode 100644 crates/nu-protocol/src/process/exit_status.rs create mode 100644 crates/nu-protocol/src/process/mod.rs delete mode 100644 crates/nu-protocol/src/util.rs rename crates/nu_plugin_example/src/commands/{collect_external.rs => collect_bytes.rs} (56%) diff --git a/Cargo.lock b/Cargo.lock index 656f859a63..5a9b95cebd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3254,6 +3254,7 @@ dependencies = [ "indexmap", "lru", "miette", + "nix", "nu-path", "nu-system", "nu-test-support", diff --git a/crates/nu-cli/src/completions/completer.rs b/crates/nu-cli/src/completions/completer.rs index 348111f009..007a0e288a 100644 --- a/crates/nu-cli/src/completions/completer.rs +++ b/crates/nu-cli/src/completions/completer.rs @@ -103,9 +103,8 @@ impl NuCompleter { PipelineData::empty(), ); - match result { - Ok(pd) => { - let value = pd.into_value(span); + match result.and_then(|data| data.into_value(span)) { + Ok(value) => { if let Value::List { vals, .. } = value { let result = map_value_completions(vals.iter(), Span::new(span.start, span.end), offset); diff --git a/crates/nu-cli/src/completions/custom_completions.rs b/crates/nu-cli/src/completions/custom_completions.rs index d2ccd5191d..17c8e6a924 100644 --- a/crates/nu-cli/src/completions/custom_completions.rs +++ b/crates/nu-cli/src/completions/custom_completions.rs @@ -74,55 +74,53 @@ impl Completer for CustomCompletion { // Parse result let suggestions = result - .map(|pd| { - let value = pd.into_value(span); - match &value { - Value::Record { val, .. } => { - let completions = val - .get("completions") - .and_then(|val| { - val.as_list() - .ok() - .map(|it| map_value_completions(it.iter(), span, offset)) - }) - .unwrap_or_default(); - let options = val.get("options"); + .and_then(|data| data.into_value(span)) + .map(|value| match &value { + Value::Record { val, .. } => { + let completions = val + .get("completions") + .and_then(|val| { + val.as_list() + .ok() + .map(|it| map_value_completions(it.iter(), span, offset)) + }) + .unwrap_or_default(); + let options = val.get("options"); - if let Some(Value::Record { val: options, .. }) = &options { - let should_sort = options - .get("sort") - .and_then(|val| val.as_bool().ok()) - .unwrap_or(false); + if let Some(Value::Record { val: options, .. }) = &options { + let should_sort = options + .get("sort") + .and_then(|val| val.as_bool().ok()) + .unwrap_or(false); - if should_sort { - self.sort_by = SortBy::Ascending; - } - - custom_completion_options = Some(CompletionOptions { - case_sensitive: options - .get("case_sensitive") - .and_then(|val| val.as_bool().ok()) - .unwrap_or(true), - positional: options - .get("positional") - .and_then(|val| val.as_bool().ok()) - .unwrap_or(true), - match_algorithm: match options.get("completion_algorithm") { - Some(option) => option - .coerce_string() - .ok() - .and_then(|option| option.try_into().ok()) - .unwrap_or(MatchAlgorithm::Prefix), - None => completion_options.match_algorithm, - }, - }); + if should_sort { + self.sort_by = SortBy::Ascending; } - completions + custom_completion_options = Some(CompletionOptions { + case_sensitive: options + .get("case_sensitive") + .and_then(|val| val.as_bool().ok()) + .unwrap_or(true), + positional: options + .get("positional") + .and_then(|val| val.as_bool().ok()) + .unwrap_or(true), + match_algorithm: match options.get("completion_algorithm") { + Some(option) => option + .coerce_string() + .ok() + .and_then(|option| option.try_into().ok()) + .unwrap_or(MatchAlgorithm::Prefix), + None => completion_options.match_algorithm, + }, + }); } - Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset), - _ => vec![], + + completions } + Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset), + _ => vec![], }) .unwrap_or_default(); diff --git a/crates/nu-cli/src/config_files.rs b/crates/nu-cli/src/config_files.rs index e89fa6c1d1..ec7ad2f412 100644 --- a/crates/nu-cli/src/config_files.rs +++ b/crates/nu-cli/src/config_files.rs @@ -306,14 +306,15 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) - let mut engine_state = engine_state.clone(); let mut stack = Stack::new(); - if !eval_source( + if eval_source( &mut engine_state, &mut stack, &old_contents, &old_plugin_file_path.to_string_lossy(), PipelineData::Empty, false, - ) { + ) != 0 + { return false; } diff --git a/crates/nu-cli/src/eval_cmds.rs b/crates/nu-cli/src/eval_cmds.rs index 0b0b5f8ddf..8fa3bf30e5 100644 --- a/crates/nu-cli/src/eval_cmds.rs +++ b/crates/nu-cli/src/eval_cmds.rs @@ -1,5 +1,4 @@ use log::info; -use miette::Result; use nu_engine::{convert_env_values, eval_block}; use nu_parser::parse; use nu_protocol::{ @@ -59,9 +58,10 @@ pub fn evaluate_commands( t_mode.coerce_str()?.parse().unwrap_or_default(); } - let exit_code = pipeline.print(engine_state, stack, no_newline, false)?; - if exit_code != 0 { - std::process::exit(exit_code as i32); + if let Some(status) = pipeline.print(engine_state, stack, no_newline, false)? { + if status.code() != 0 { + std::process::exit(status.code()) + } } info!("evaluate {}:{}:{}", file!(), line!(), column!()); diff --git a/crates/nu-cli/src/eval_file.rs b/crates/nu-cli/src/eval_file.rs index 7483c6bc33..ff6ba36fe3 100644 --- a/crates/nu-cli/src/eval_file.rs +++ b/crates/nu-cli/src/eval_file.rs @@ -96,7 +96,7 @@ pub fn evaluate_file( engine_state.merge_delta(working_set.delta)?; // Check if the file contains a main command. - if engine_state.find_decl(b"main", &[]).is_some() { + let exit_code = if engine_state.find_decl(b"main", &[]).is_some() { // Evaluate the file, but don't run main yet. let pipeline = match eval_block::(engine_state, stack, &block, PipelineData::empty()) { @@ -109,26 +109,29 @@ pub fn evaluate_file( }; // Print the pipeline output of the last command of the file. - let exit_code = pipeline.print(engine_state, stack, true, false)?; - if exit_code != 0 { - std::process::exit(exit_code as i32); + if let Some(status) = pipeline.print(engine_state, stack, true, false)? { + if status.code() != 0 { + std::process::exit(status.code()) + } } // Invoke the main command with arguments. // Arguments with whitespaces are quoted, thus can be safely concatenated by whitespace. let args = format!("main {}", args.join(" ")); - if !eval_source( + eval_source( engine_state, stack, args.as_bytes(), "", input, true, - ) { - std::process::exit(1); - } - } else if !eval_source(engine_state, stack, &file, file_path_str, input, true) { - std::process::exit(1); + ) + } else { + eval_source(engine_state, stack, &file, file_path_str, input, true) + }; + + if exit_code != 0 { + std::process::exit(exit_code) } info!("evaluate {}:{}:{}", file!(), line!(), column!()); diff --git a/crates/nu-cli/src/menus/menu_completions.rs b/crates/nu-cli/src/menus/menu_completions.rs index fbfc598225..c65f0bd100 100644 --- a/crates/nu-cli/src/menus/menu_completions.rs +++ b/crates/nu-cli/src/menus/menu_completions.rs @@ -59,8 +59,7 @@ impl Completer for NuMenuCompleter { let res = eval_block::(&self.engine_state, &mut self.stack, block, input); - if let Ok(values) = res { - let values = values.into_value(self.span); + if let Ok(values) = res.and_then(|data| data.into_value(self.span)) { convert_to_suggestions(values, line, pos, self.only_buffer_difference) } else { Vec::new() diff --git a/crates/nu-cli/src/util.rs b/crates/nu-cli/src/util.rs index 2f996691c9..7ebea0deb2 100644 --- a/crates/nu-cli/src/util.rs +++ b/crates/nu-cli/src/util.rs @@ -4,7 +4,7 @@ use nu_parser::{escape_quote_string, lex, parse, unescape_unquote_string, Token, use nu_protocol::{ debugger::WithoutDebug, engine::{EngineState, Stack, StateWorkingSet}, - print_if_stream, report_error, report_error_new, PipelineData, ShellError, Span, Value, + report_error, report_error_new, PipelineData, ShellError, Span, Value, }; #[cfg(windows)] use nu_utils::enable_vt_processing; @@ -206,9 +206,48 @@ pub fn eval_source( fname: &str, input: PipelineData, allow_return: bool, -) -> bool { +) -> i32 { let start_time = std::time::Instant::now(); + let exit_code = match evaluate_source(engine_state, stack, source, fname, input, allow_return) { + Ok(code) => code.unwrap_or(0), + Err(err) => { + report_error_new(engine_state, &err); + 1 + } + }; + + stack.add_env_var( + "LAST_EXIT_CODE".to_string(), + Value::int(exit_code.into(), Span::unknown()), + ); + + // reset vt processing, aka ansi because illbehaved externals can break it + #[cfg(windows)] + { + let _ = enable_vt_processing(); + } + + perf( + &format!("eval_source {}", &fname), + start_time, + file!(), + line!(), + column!(), + engine_state.get_config().use_ansi_coloring, + ); + + exit_code +} + +fn evaluate_source( + engine_state: &mut EngineState, + stack: &mut Stack, + source: &[u8], + fname: &str, + input: PipelineData, + allow_return: bool, +) -> Result, ShellError> { let (block, delta) = { let mut working_set = StateWorkingSet::new(engine_state); let output = parse( @@ -222,97 +261,40 @@ pub fn eval_source( } if let Some(err) = working_set.parse_errors.first() { - set_last_exit_code(stack, 1); report_error(&working_set, err); - return false; + return Ok(Some(1)); } (output, working_set.render()) }; - if let Err(err) = engine_state.merge_delta(delta) { - set_last_exit_code(stack, 1); - report_error_new(engine_state, &err); - return false; - } + engine_state.merge_delta(delta)?; - let b = if allow_return { + let pipeline = if allow_return { eval_block_with_early_return::(engine_state, stack, &block, input) } else { eval_block::(engine_state, stack, &block, input) + }?; + + let status = if let PipelineData::ByteStream(stream, ..) = pipeline { + stream.print(false)? + } else { + if let Some(hook) = engine_state.get_config().hooks.display_output.clone() { + let pipeline = eval_hook( + engine_state, + stack, + Some(pipeline), + vec![], + &hook, + "display_output", + )?; + pipeline.print(engine_state, stack, false, false) + } else { + pipeline.print(engine_state, stack, true, false) + }? }; - match b { - Ok(pipeline_data) => { - let config = engine_state.get_config(); - let result; - if let PipelineData::ExternalStream { - stdout: stream, - stderr: stderr_stream, - exit_code, - .. - } = pipeline_data - { - result = print_if_stream(stream, stderr_stream, false, exit_code); - } else if let Some(hook) = config.hooks.display_output.clone() { - match eval_hook( - engine_state, - stack, - Some(pipeline_data), - vec![], - &hook, - "display_output", - ) { - Err(err) => { - result = Err(err); - } - Ok(val) => { - result = val.print(engine_state, stack, false, false); - } - } - } else { - result = pipeline_data.print(engine_state, stack, true, false); - } - - match result { - Err(err) => { - report_error_new(engine_state, &err); - return false; - } - Ok(exit_code) => { - set_last_exit_code(stack, exit_code); - } - } - - // reset vt processing, aka ansi because illbehaved externals can break it - #[cfg(windows)] - { - let _ = enable_vt_processing(); - } - } - Err(err) => { - set_last_exit_code(stack, 1); - report_error_new(engine_state, &err); - return false; - } - } - perf( - &format!("eval_source {}", &fname), - start_time, - file!(), - line!(), - column!(), - engine_state.get_config().use_ansi_coloring, - ); - - true -} - -fn set_last_exit_code(stack: &mut Stack, exit_code: i64) { - stack.add_env_var( - "LAST_EXIT_CODE".to_string(), - Value::int(exit_code, Span::unknown()), - ); + Ok(status.map(|status| status.code())) } #[cfg(test)] diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs index c170f8db8a..be9c33a229 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs @@ -79,7 +79,7 @@ impl Command for CastDF { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let (dtype, column_nm) = df_args(engine_state, stack, call)?; let df = NuLazyFrame::try_from_value(value)?; diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs index 3793945181..e0e94d10a0 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs @@ -72,8 +72,7 @@ impl Command for FilterWith { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; command_lazy(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs index 70160b3005..14c86e8c40 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs @@ -86,7 +86,7 @@ impl Command for FirstDF { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuDataFrame::try_from_value(value)?; command(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs index a0a188471d..ff2c4f98a2 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs @@ -61,7 +61,7 @@ impl Command for LastDF { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuDataFrame::try_from_value(value)?; command(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs index 5167a0c968..0cb75f34f2 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs @@ -109,8 +109,7 @@ impl Command for RenameDF { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; command_lazy(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs index 73dadacb2b..a6ab42052c 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs @@ -76,7 +76,7 @@ impl Command for ToNu { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { dataframe_command(engine_state, stack, call, value) } else { diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs index 52ceefceb4..79d3427e8a 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs @@ -102,8 +102,7 @@ impl Command for WithColumn { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; command_lazy(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs index b2d79be010..4cc56e030b 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs @@ -172,7 +172,7 @@ macro_rules! lazy_expr_command { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let lazy = NuLazyFrame::try_from_value(value)?; let lazy = NuLazyFrame::new( @@ -271,7 +271,7 @@ macro_rules! lazy_expr_command { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let lazy = NuLazyFrame::try_from_value(value)?; let lazy = NuLazyFrame::new( diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs index 0ba507f97f..eb97c575b7 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs @@ -91,7 +91,7 @@ impl Command for ExprOtherwise { let otherwise_predicate: Value = call.req(engine_state, stack, 0)?; let otherwise_predicate = NuExpression::try_from_value(otherwise_predicate)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let complete: NuExpression = match NuWhen::try_from_value(value)? { NuWhen::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(), NuWhen::ChainedThen(chained_when) => chained_when diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs index d82a0faf0a..aaa1029ee9 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs @@ -67,7 +67,7 @@ impl Command for ExprQuantile { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let quantile: f64 = call.req(engine_state, stack, 0)?; let expr = NuExpression::try_from_value(value)?; diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs index d70fd00825..5a6aad2de7 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs @@ -103,7 +103,7 @@ impl Command for ExprWhen { let then_predicate: Value = call.req(engine_state, stack, 1)?; let then_predicate = NuExpression::try_from_value(then_predicate)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let when_then: NuWhen = match value { Value::Nothing { .. } => when(when_predicate.into_polars()) .then(then_predicate.into_polars()) diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs index 8e32ae8040..a027e84d36 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs @@ -100,7 +100,7 @@ impl Command for LazyExplode { } pub(crate) fn explode(call: &Call, input: PipelineData) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; let columns: Vec = call diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs index a9a1eb1590..4c75f1d9a3 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs @@ -82,7 +82,7 @@ impl Command for LazyFillNA { input: PipelineData, ) -> Result { let fill: Value = call.req(engine_state, stack, 0)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuExpression::can_downcast(&value) { let expr = NuExpression::try_from_value(value)?; diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs index b3d35d2b8d..88be2a9e88 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs @@ -59,7 +59,7 @@ impl Command for LazyFillNull { input: PipelineData, ) -> Result { let fill: Value = call.req(engine_state, stack, 0)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuExpression::can_downcast(&value) { let expr = NuExpression::try_from_value(value)?; diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs index 7f7d1ab66b..4ae297acfd 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs @@ -219,7 +219,7 @@ impl Command for LazyJoin { let suffix: Option = call.get_flag(engine_state, stack, "suffix")?; let suffix = suffix.unwrap_or_else(|| "_x".into()); - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value(value)?; let from_eager = lazy.from_eager; let lazy = lazy.into_polars(); diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs index d17a444f49..ac8ec590c6 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs @@ -54,7 +54,7 @@ impl Command for LazyQuantile { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let quantile: f64 = call.req(engine_state, stack, 0)?; let lazy = NuLazyFrame::try_from_value(value)?; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs index ce66f69877..4ed33ce951 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs @@ -68,7 +68,7 @@ impl Command for IsNotNull { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuDataFrame::try_from_value(value)?; command(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs index d7921da347..b99d48af66 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs @@ -68,7 +68,7 @@ impl Command for IsNull { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuDataFrame::try_from_value(value)?; command(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs index b23ab4e20d..c6d6e829f8 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs @@ -60,7 +60,7 @@ impl Command for NUnique { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuDataFrame::try_from_value(value)?; command(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs b/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs index bf842840b4..2f40cf0a45 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs @@ -56,8 +56,7 @@ impl Command for Shift { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; command_lazy(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs index 13012b4fb3..1bc2e0dc1b 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs @@ -72,8 +72,7 @@ impl Command for Unique { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; command_lazy(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs b/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs index d6febf7e43..39c30be9dd 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs @@ -80,7 +80,8 @@ pub fn test_dataframe_example(engine_state: &mut Box, example: &Exa let result = eval_block::(engine_state, &mut stack, &block, PipelineData::empty()) .unwrap_or_else(|err| panic!("test eval error in `{}`: {:?}", example.example, err)) - .into_value(Span::test_data()); + .into_value(Span::test_data()) + .expect("ok value"); println!("input: {}", example.example); println!("result: {result:?}"); diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs index 8b828aee50..967e03580f 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs @@ -297,7 +297,7 @@ impl NuDataFrame { } pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(value) } diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs index 8646cdefd0..cee31d7b53 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs @@ -84,7 +84,7 @@ impl NuExpression { } pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(value) } diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs index f03d9f0cc8..355516d340 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs @@ -134,7 +134,7 @@ impl NuLazyFrame { } pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(value) } diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs index e942e3be97..e1bcb30069 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs @@ -107,7 +107,7 @@ impl NuLazyGroupBy { } pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(value) } } diff --git a/crates/nu-cmd-extra/src/extra/bits/into.rs b/crates/nu-cmd-extra/src/extra/bits/into.rs index c7fd09b728..cf85f92ac5 100644 --- a/crates/nu-cmd-extra/src/extra/bits/into.rs +++ b/crates/nu-cmd-extra/src/extra/bits/into.rs @@ -118,22 +118,12 @@ fn into_bits( let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - match input { - PipelineData::ExternalStream { stdout: None, .. } => { - Ok(Value::binary(vec![], head).into_pipeline_data()) - } - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - // TODO: in the future, we may want this to stream out, converting each to bytes - let output = stream.into_bytes()?; - Ok(Value::binary(output.item, head).into_pipeline_data()) - } - _ => { - let args = Arguments { cell_paths }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) - } + if let PipelineData::ByteStream(stream, ..) = input { + // TODO: in the future, we may want this to stream out, converting each to bytes + Ok(Value::binary(stream.into_bytes()?, head).into_pipeline_data()) + } else { + let args = Arguments { cell_paths }; + operate(action, args, input, call.head, engine_state.ctrlc.clone()) } } diff --git a/crates/nu-cmd-extra/src/extra/filters/each_while.rs b/crates/nu-cmd-extra/src/extra/filters/each_while.rs index 939f194f43..58679c8eea 100644 --- a/crates/nu-cmd-extra/src/extra/filters/each_while.rs +++ b/crates/nu-cmd-extra/src/extra/filters/each_while.rs @@ -78,38 +78,40 @@ impl Command for EachWhile { | PipelineData::ListStream(..) => { let mut closure = ClosureEval::new(engine_state, stack, closure); Ok(input - .into_iter() - .map_while(move |value| match closure.run_with_value(value) { - Ok(data) => { - let value = data.into_value(head); - (!value.is_nothing()).then_some(value) - } - Err(_) => None, - }) - .fuse() - .into_pipeline_data(head, engine_state.ctrlc.clone())) - } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - let mut closure = ClosureEval::new(engine_state, stack, closure); - Ok(stream .into_iter() .map_while(move |value| { - let value = value.ok()?; - match closure.run_with_value(value) { - Ok(data) => { - let value = data.into_value(head); - (!value.is_nothing()).then_some(value) - } + match closure + .run_with_value(value) + .and_then(|data| data.into_value(head)) + { + Ok(value) => (!value.is_nothing()).then_some(value), Err(_) => None, } }) .fuse() .into_pipeline_data(head, engine_state.ctrlc.clone())) } + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + if let Some(chunks) = stream.chunks() { + let mut closure = ClosureEval::new(engine_state, stack, closure); + Ok(chunks + .map_while(move |value| { + let value = value.ok()?; + match closure + .run_with_value(value) + .and_then(|data| data.into_value(span)) + { + Ok(value) => (!value.is_nothing()).then_some(value), + Err(_) => None, + } + }) + .fuse() + .into_pipeline_data(head, engine_state.ctrlc.clone())) + } else { + Ok(PipelineData::Empty) + } + } // This match allows non-iterables to be accepted, // which is currently considered undesirable (Nov 2022). PipelineData::Value(value, ..) => { diff --git a/crates/nu-cmd-extra/src/extra/filters/roll/roll_down.rs b/crates/nu-cmd-extra/src/extra/filters/roll/roll_down.rs index 465b9f1f4c..24ea1bc309 100644 --- a/crates/nu-cmd-extra/src/extra/filters/roll/roll_down.rs +++ b/crates/nu-cmd-extra/src/extra/filters/roll/roll_down.rs @@ -56,7 +56,7 @@ impl Command for RollDown { let by: Option = call.get_flag(engine_state, stack, "by")?; let metadata = input.metadata(); - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let rotated_value = vertical_rotate_value(value, by, VerticalDirection::Down)?; Ok(rotated_value.into_pipeline_data().set_metadata(metadata)) diff --git a/crates/nu-cmd-extra/src/extra/filters/roll/roll_left.rs b/crates/nu-cmd-extra/src/extra/filters/roll/roll_left.rs index ff69f23268..789b70830d 100644 --- a/crates/nu-cmd-extra/src/extra/filters/roll/roll_left.rs +++ b/crates/nu-cmd-extra/src/extra/filters/roll/roll_left.rs @@ -94,7 +94,7 @@ impl Command for RollLeft { let metadata = input.metadata(); let cells_only = call.has_flag(engine_state, stack, "cells-only")?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let rotated_value = horizontal_rotate_value(value, by, cells_only, &HorizontalDirection::Left)?; diff --git a/crates/nu-cmd-extra/src/extra/filters/roll/roll_right.rs b/crates/nu-cmd-extra/src/extra/filters/roll/roll_right.rs index d190960581..55a1e42158 100644 --- a/crates/nu-cmd-extra/src/extra/filters/roll/roll_right.rs +++ b/crates/nu-cmd-extra/src/extra/filters/roll/roll_right.rs @@ -94,7 +94,7 @@ impl Command for RollRight { let metadata = input.metadata(); let cells_only = call.has_flag(engine_state, stack, "cells-only")?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let rotated_value = horizontal_rotate_value(value, by, cells_only, &HorizontalDirection::Right)?; diff --git a/crates/nu-cmd-extra/src/extra/filters/roll/roll_up.rs b/crates/nu-cmd-extra/src/extra/filters/roll/roll_up.rs index 1cd74fe247..7b9480599d 100644 --- a/crates/nu-cmd-extra/src/extra/filters/roll/roll_up.rs +++ b/crates/nu-cmd-extra/src/extra/filters/roll/roll_up.rs @@ -56,7 +56,7 @@ impl Command for RollUp { let by: Option = call.get_flag(engine_state, stack, "by")?; let metadata = input.metadata(); - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let rotated_value = vertical_rotate_value(value, by, VerticalDirection::Up)?; Ok(rotated_value.into_pipeline_data().set_metadata(metadata)) diff --git a/crates/nu-cmd-extra/src/extra/filters/update_cells.rs b/crates/nu-cmd-extra/src/extra/filters/update_cells.rs index 9fe9bfe389..c90e933410 100644 --- a/crates/nu-cmd-extra/src/extra/filters/update_cells.rs +++ b/crates/nu-cmd-extra/src/extra/filters/update_cells.rs @@ -152,7 +152,7 @@ impl Iterator for UpdateCellIterator { fn eval_value(closure: &mut ClosureEval, span: Span, value: Value) -> Value { closure .run_with_value(value) - .map(|data| data.into_value(span)) + .and_then(|data| data.into_value(span)) .unwrap_or_else(|err| Value::error(err, span)) } diff --git a/crates/nu-cmd-extra/src/extra/strings/format/command.rs b/crates/nu-cmd-extra/src/extra/strings/format/command.rs index 932b5ccb7f..1c72627779 100644 --- a/crates/nu-cmd-extra/src/extra/strings/format/command.rs +++ b/crates/nu-cmd-extra/src/extra/strings/format/command.rs @@ -39,7 +39,7 @@ impl Command for FormatPattern { let mut working_set = StateWorkingSet::new(engine_state); let specified_pattern: Result = call.req(engine_state, stack, 0); - let input_val = input.into_value(call.head); + let input_val = input.into_value(call.head)?; // add '$it' variable to support format like this: $it.column1.column2. let it_id = working_set.add_variable(b"$it".to_vec(), call.head, Type::Any, false); stack.add_var(it_id, input_val.clone()); diff --git a/crates/nu-cmd-extra/tests/commands/bytes/starts_with.rs b/crates/nu-cmd-extra/tests/commands/bytes/starts_with.rs index e7d57698b5..c3ad1ec448 100644 --- a/crates/nu-cmd-extra/tests/commands/bytes/starts_with.rs +++ b/crates/nu-cmd-extra/tests/commands/bytes/starts_with.rs @@ -19,102 +19,102 @@ fn basic_string_fails() { assert_eq!(actual.out, ""); } -#[test] -fn short_stream_binary() { - let actual = nu!(r#" - nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101] - "#); +// #[test] +// fn short_stream_binary() { +// let actual = nu!(r#" +// nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101] +// "#); - assert_eq!(actual.out, "true"); -} +// assert_eq!(actual.out, "true"); +// } -#[test] -fn short_stream_mismatch() { - let actual = nu!(r#" - nu --testbin repeater (0x[010203]) 5 | bytes starts-with 0x[010204] - "#); +// #[test] +// fn short_stream_mismatch() { +// let actual = nu!(r#" +// nu --testbin repeater (0x[010203]) 5 | bytes starts-with 0x[010204] +// "#); - assert_eq!(actual.out, "false"); -} +// assert_eq!(actual.out, "false"); +// } -#[test] -fn short_stream_binary_overflow() { - let actual = nu!(r#" - nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101010101] - "#); +// #[test] +// fn short_stream_binary_overflow() { +// let actual = nu!(r#" +// nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101010101] +// "#); - assert_eq!(actual.out, "false"); -} +// assert_eq!(actual.out, "false"); +// } -#[test] -fn long_stream_binary() { - let actual = nu!(r#" - nu --testbin repeater (0x[01]) 32768 | bytes starts-with 0x[010101] - "#); +// #[test] +// fn long_stream_binary() { +// let actual = nu!(r#" +// nu --testbin repeater (0x[01]) 32768 | bytes starts-with 0x[010101] +// "#); - assert_eq!(actual.out, "true"); -} +// assert_eq!(actual.out, "true"); +// } -#[test] -fn long_stream_binary_overflow() { - // .. ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow - let actual = nu!(r#" - nu --testbin repeater (0x[01]) 32768 | bytes starts-with (0..32768 | each {|| 0x[01] } | bytes collect) - "#); +// #[test] +// fn long_stream_binary_overflow() { +// // .. ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow +// let actual = nu!(r#" +// nu --testbin repeater (0x[01]) 32768 | bytes starts-with (0..32768 | each {|| 0x[01] } | bytes collect) +// "#); - assert_eq!(actual.out, "false"); -} +// assert_eq!(actual.out, "false"); +// } -#[test] -fn long_stream_binary_exact() { - // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow - let actual = nu!(r#" - nu --testbin repeater (0x[01020304]) 8192 | bytes starts-with (0..<8192 | each {|| 0x[01020304] } | bytes collect) - "#); +// #[test] +// fn long_stream_binary_exact() { +// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow +// let actual = nu!(r#" +// nu --testbin repeater (0x[01020304]) 8192 | bytes starts-with (0..<8192 | each {|| 0x[01020304] } | bytes collect) +// "#); - assert_eq!(actual.out, "true"); -} +// assert_eq!(actual.out, "true"); +// } -#[test] -fn long_stream_string_exact() { - // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow - let actual = nu!(r#" - nu --testbin repeater hell 8192 | bytes starts-with (0..<8192 | each {|| "hell" | into binary } | bytes collect) - "#); +// #[test] +// fn long_stream_string_exact() { +// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow +// let actual = nu!(r#" +// nu --testbin repeater hell 8192 | bytes starts-with (0..<8192 | each {|| "hell" | into binary } | bytes collect) +// "#); - assert_eq!(actual.out, "true"); -} +// assert_eq!(actual.out, "true"); +// } -#[test] -fn long_stream_mixed_exact() { - // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow - let actual = nu!(r#" - let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect) - let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect) +// #[test] +// fn long_stream_mixed_exact() { +// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow +// let actual = nu!(r#" +// let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect) +// let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect) - nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg) - "#); +// nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg) +// "#); - assert_eq!( - actual.err, "", - "invocation failed. command line limit likely reached" - ); - assert_eq!(actual.out, "true"); -} +// assert_eq!( +// actual.err, "", +// "invocation failed. command line limit likely reached" +// ); +// assert_eq!(actual.out, "true"); +// } -#[test] -fn long_stream_mixed_overflow() { - // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow - let actual = nu!(r#" - let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect) - let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect) +// #[test] +// fn long_stream_mixed_overflow() { +// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow +// let actual = nu!(r#" +// let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect) +// let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect) - nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg 0x[01]) - "#); +// nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg 0x[01]) +// "#); - assert_eq!( - actual.err, "", - "invocation failed. command line limit likely reached" - ); - assert_eq!(actual.out, "false"); -} +// assert_eq!( +// actual.err, "", +// "invocation failed. command line limit likely reached" +// ); +// assert_eq!(actual.out, "false"); +// } diff --git a/crates/nu-cmd-lang/src/core_commands/collect.rs b/crates/nu-cmd-lang/src/core_commands/collect.rs index eae41e8690..404aa568da 100644 --- a/crates/nu-cmd-lang/src/core_commands/collect.rs +++ b/crates/nu-cmd-lang/src/core_commands/collect.rs @@ -43,7 +43,7 @@ impl Command for Collect { stack.captures_to_stack_preserve_out_dest(closure.captures.clone()); let metadata = input.metadata(); - let input = input.into_value(call.head); + let input = input.into_value(call.head)?; let mut saved_positional = None; if let Some(var) = block.signature.get_positional(0) { diff --git a/crates/nu-cmd-lang/src/core_commands/describe.rs b/crates/nu-cmd-lang/src/core_commands/describe.rs index e1934f8bad..7d6d7f6f83 100644 --- a/crates/nu-cmd-lang/src/core_commands/describe.rs +++ b/crates/nu-cmd-lang/src/core_commands/describe.rs @@ -1,5 +1,5 @@ use nu_engine::command_prelude::*; -use nu_protocol::{engine::StateWorkingSet, PipelineMetadata}; +use nu_protocol::{engine::StateWorkingSet, ByteStreamSource, PipelineMetadata}; #[derive(Clone)] pub struct Describe; @@ -162,73 +162,38 @@ fn run( let metadata = input.metadata(); let description = match input { - PipelineData::ExternalStream { - ref stdout, - ref stderr, - ref exit_code, - .. - } => { - if options.detailed { - let stdout = if stdout.is_some() { - Value::record( - record! { - "type" => Value::string("stream", head), - "origin" => Value::string("external", head), - "subtype" => Value::string("any", head), - }, - head, - ) - } else { - Value::nothing(head) - }; - - let stderr = if stderr.is_some() { - Value::record( - record! { - "type" => Value::string("stream", head), - "origin" => Value::string("external", head), - "subtype" => Value::string("any", head), - }, - head, - ) - } else { - Value::nothing(head) - }; - - let exit_code = if exit_code.is_some() { - Value::record( - record! { - "type" => Value::string("stream", head), - "origin" => Value::string("external", head), - "subtype" => Value::string("int", head), - }, - head, - ) - } else { - Value::nothing(head) + PipelineData::ByteStream(stream, ..) => { + let description = if options.detailed { + let origin = match stream.source() { + ByteStreamSource::Read(_) => "unknown", + ByteStreamSource::File(_) => "file", + ByteStreamSource::Child(_) => "external", }; Value::record( record! { - "type" => Value::string("stream", head), - "origin" => Value::string("external", head), - "stdout" => stdout, - "stderr" => stderr, - "exit_code" => exit_code, + "type" => Value::string("byte stream", head), + "origin" => Value::string(origin, head), "metadata" => metadata_to_value(metadata, head), }, head, ) } else { - Value::string("raw input", head) + Value::string("byte stream", head) + }; + + if !options.no_collect { + stream.drain()?; } + + description } - PipelineData::ListStream(_, _) => { + PipelineData::ListStream(stream, ..) => { if options.detailed { let subtype = if options.no_collect { Value::string("any", head) } else { - describe_value(input.into_value(head), head, engine_state) + describe_value(stream.into_value(), head, engine_state) }; Value::record( record! { @@ -242,19 +207,19 @@ fn run( } else if options.no_collect { Value::string("stream", head) } else { - let value = input.into_value(head); + let value = stream.into_value(); let base_description = value.get_type().to_string(); Value::string(format!("{} (stream)", base_description), head) } } - _ => { - let value = input.into_value(head); + PipelineData::Value(value, ..) => { if !options.detailed { Value::string(value.get_type().to_string(), head) } else { describe_value(value, head, engine_state) } } + PipelineData::Empty => Value::string(Type::Nothing.to_string(), head), }; Ok(description.into_pipeline_data()) diff --git a/crates/nu-cmd-lang/src/core_commands/do_.rs b/crates/nu-cmd-lang/src/core_commands/do_.rs index b057880cf3..5f14e88c07 100644 --- a/crates/nu-cmd-lang/src/core_commands/do_.rs +++ b/crates/nu-cmd-lang/src/core_commands/do_.rs @@ -1,6 +1,13 @@ use nu_engine::{command_prelude::*, get_eval_block_with_early_return, redirect_env}; -use nu_protocol::{engine::Closure, ListStream, OutDest, RawStream}; -use std::thread; +use nu_protocol::{ + engine::Closure, + process::{ChildPipe, ChildProcess, ExitStatus}, + ByteStream, ByteStreamSource, OutDest, +}; +use std::{ + io::{Cursor, Read}, + thread, +}; #[derive(Clone)] pub struct Do; @@ -86,115 +93,91 @@ impl Command for Do { } match result { - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - }) if capture_errors => { - // Use a thread to receive stdout message. - // Or we may get a deadlock if child process sends out too much bytes to stderr. - // - // For example: in normal linux system, stderr pipe's limit is 65535 bytes. - // if child process sends out 65536 bytes, the process will be hanged because no consumer - // consumes the first 65535 bytes - // So we need a thread to receive stdout message, then the current thread can continue to consume - // stderr messages. - let stdout_handler = stdout - .map(|stdout_stream| { - thread::Builder::new() - .name("stderr redirector".to_string()) - .spawn(move || { - let ctrlc = stdout_stream.ctrlc.clone(); - let span = stdout_stream.span; - RawStream::new( - Box::new(std::iter::once( - stdout_stream.into_bytes().map(|s| s.item), - )), - ctrlc, - span, - None, - ) + Ok(PipelineData::ByteStream(stream, metadata)) if capture_errors => { + let span = stream.span(); + match stream.into_child() { + Ok(mut child) => { + // Use a thread to receive stdout message. + // Or we may get a deadlock if child process sends out too much bytes to stderr. + // + // For example: in normal linux system, stderr pipe's limit is 65535 bytes. + // if child process sends out 65536 bytes, the process will be hanged because no consumer + // consumes the first 65535 bytes + // So we need a thread to receive stdout message, then the current thread can continue to consume + // stderr messages. + let stdout_handler = child + .stdout + .take() + .map(|mut stdout| { + thread::Builder::new() + .name("stdout consumer".to_string()) + .spawn(move || { + let mut buf = Vec::new(); + stdout.read_to_end(&mut buf)?; + Ok::<_, ShellError>(buf) + }) + .err_span(head) }) - .err_span(head) - }) - .transpose()?; + .transpose()?; - // Intercept stderr so we can return it in the error if the exit code is non-zero. - // The threading issues mentioned above dictate why we also need to intercept stdout. - let mut stderr_ctrlc = None; - let stderr_msg = match stderr { - None => "".to_string(), - Some(stderr_stream) => { - stderr_ctrlc.clone_from(&stderr_stream.ctrlc); - stderr_stream.into_string().map(|s| s.item)? - } - }; + // Intercept stderr so we can return it in the error if the exit code is non-zero. + // The threading issues mentioned above dictate why we also need to intercept stdout. + let stderr_msg = match child.stderr.take() { + None => String::new(), + Some(mut stderr) => { + let mut buf = String::new(); + stderr.read_to_string(&mut buf).err_span(span)?; + buf + } + }; - let stdout = if let Some(handle) = stdout_handler { - match handle.join() { - Err(err) => { + let stdout = if let Some(handle) = stdout_handler { + match handle.join() { + Err(err) => { + return Err(ShellError::ExternalCommand { + label: "Fail to receive external commands stdout message" + .to_string(), + help: format!("{err:?}"), + span, + }); + } + Ok(res) => Some(res?), + } + } else { + None + }; + + if child.wait()? != ExitStatus::Exited(0) { return Err(ShellError::ExternalCommand { - label: "Fail to receive external commands stdout message" - .to_string(), - help: format!("{err:?}"), + label: "External command failed".to_string(), + help: stderr_msg, span, }); } - Ok(res) => Some(res), - } - } else { - None - }; - let exit_code: Vec = match exit_code { - None => vec![], - Some(exit_code_stream) => exit_code_stream.into_iter().collect(), - }; - if let Some(Value::Int { val: code, .. }) = exit_code.last() { - if *code != 0 { - return Err(ShellError::ExternalCommand { - label: "External command failed".to_string(), - help: stderr_msg, - span, - }); + let mut child = ChildProcess::from_raw(None, None, None, span); + if let Some(stdout) = stdout { + child.stdout = Some(ChildPipe::Tee(Box::new(Cursor::new(stdout)))); + } + if !stderr_msg.is_empty() { + child.stderr = Some(ChildPipe::Tee(Box::new(Cursor::new(stderr_msg)))); + } + Ok(PipelineData::ByteStream( + ByteStream::child(child, span), + metadata, + )) } + Err(stream) => Ok(PipelineData::ByteStream(stream, metadata)), } - - Ok(PipelineData::ExternalStream { - stdout, - stderr: Some(RawStream::new( - Box::new(std::iter::once(Ok(stderr_msg.into_bytes()))), - stderr_ctrlc, - span, - None, - )), - exit_code: Some(ListStream::new(exit_code.into_iter(), span, None)), - span, - metadata, - trim_end_newline, - }) } - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code: _, - span, - metadata, - trim_end_newline, - }) if ignore_program_errors - && !matches!(caller_stack.stdout(), OutDest::Pipe | OutDest::Capture) => + Ok(PipelineData::ByteStream(mut stream, metadata)) + if ignore_program_errors + && !matches!(caller_stack.stdout(), OutDest::Pipe | OutDest::Capture) => { - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code: None, - span, - metadata, - trim_end_newline, - }) + if let ByteStreamSource::Child(child) = stream.source_mut() { + child.set_exit_code(0) + } + Ok(PipelineData::ByteStream(stream, metadata)) } Ok(PipelineData::Value(Value::Error { .. }, ..)) | Err(_) if ignore_shell_errors => { Ok(PipelineData::empty()) diff --git a/crates/nu-cmd-lang/src/core_commands/for_.rs b/crates/nu-cmd-lang/src/core_commands/for_.rs index 64e6c0a6ba..6f9391614e 100644 --- a/crates/nu-cmd-lang/src/core_commands/for_.rs +++ b/crates/nu-cmd-lang/src/core_commands/for_.rs @@ -121,12 +121,14 @@ impl Command for For { Err(err) => { return Err(err); } - Ok(pipeline) => { - let exit_code = pipeline.drain_with_exit_code()?; - if exit_code != 0 { - return Ok(PipelineData::new_external_stream_with_only_exit_code( - exit_code, - )); + Ok(data) => { + if let Some(status) = data.drain()? { + let code = status.code(); + if code != 0 { + return Ok( + PipelineData::new_external_stream_with_only_exit_code(code), + ); + } } } } @@ -159,12 +161,14 @@ impl Command for For { Err(err) => { return Err(err); } - Ok(pipeline) => { - let exit_code = pipeline.drain_with_exit_code()?; - if exit_code != 0 { - return Ok(PipelineData::new_external_stream_with_only_exit_code( - exit_code, - )); + Ok(data) => { + if let Some(status) = data.drain()? { + let code = status.code(); + if code != 0 { + return Ok( + PipelineData::new_external_stream_with_only_exit_code(code), + ); + } } } } @@ -173,7 +177,7 @@ impl Command for For { x => { stack.add_var(var_id, x); - eval_block(&engine_state, stack, block, PipelineData::empty())?.into_value(head); + eval_block(&engine_state, stack, block, PipelineData::empty())?.into_value(head)?; } } Ok(PipelineData::empty()) diff --git a/crates/nu-cmd-lang/src/core_commands/let_.rs b/crates/nu-cmd-lang/src/core_commands/let_.rs index c780954bc6..cc5504d8d6 100644 --- a/crates/nu-cmd-lang/src/core_commands/let_.rs +++ b/crates/nu-cmd-lang/src/core_commands/let_.rs @@ -61,7 +61,7 @@ impl Command for Let { let eval_block = get_eval_block(engine_state); let stack = &mut stack.start_capture(); let pipeline_data = eval_block(engine_state, stack, block, input)?; - let value = pipeline_data.into_value(call.head); + let value = pipeline_data.into_value(call.head)?; // if given variable type is Glob, and our result is string // then nushell need to convert from Value::String to Value::Glob diff --git a/crates/nu-cmd-lang/src/core_commands/loop_.rs b/crates/nu-cmd-lang/src/core_commands/loop_.rs index 29f22649eb..9b1e36a057 100644 --- a/crates/nu-cmd-lang/src/core_commands/loop_.rs +++ b/crates/nu-cmd-lang/src/core_commands/loop_.rs @@ -53,12 +53,12 @@ impl Command for Loop { Err(err) => { return Err(err); } - Ok(pipeline) => { - let exit_code = pipeline.drain_with_exit_code()?; - if exit_code != 0 { - return Ok(PipelineData::new_external_stream_with_only_exit_code( - exit_code, - )); + Ok(data) => { + if let Some(status) = data.drain()? { + let code = status.code(); + if code != 0 { + return Ok(PipelineData::new_external_stream_with_only_exit_code(code)); + } } } } diff --git a/crates/nu-cmd-lang/src/core_commands/mut_.rs b/crates/nu-cmd-lang/src/core_commands/mut_.rs index be2d66aff4..60c4c146db 100644 --- a/crates/nu-cmd-lang/src/core_commands/mut_.rs +++ b/crates/nu-cmd-lang/src/core_commands/mut_.rs @@ -61,7 +61,7 @@ impl Command for Mut { let eval_block = get_eval_block(engine_state); let stack = &mut stack.start_capture(); let pipeline_data = eval_block(engine_state, stack, block, input)?; - let value = pipeline_data.into_value(call.head); + let value = pipeline_data.into_value(call.head)?; // if given variable type is Glob, and our result is string // then nushell need to convert from Value::String to Value::Glob diff --git a/crates/nu-cmd-lang/src/core_commands/try_.rs b/crates/nu-cmd-lang/src/core_commands/try_.rs index bc96f3c28a..0b399e368a 100644 --- a/crates/nu-cmd-lang/src/core_commands/try_.rs +++ b/crates/nu-cmd-lang/src/core_commands/try_.rs @@ -62,10 +62,11 @@ impl Command for Try { } // external command may fail to run Ok(pipeline) => { - let (pipeline, external_failed) = pipeline.check_external_failed(); + let (pipeline, external_failed) = pipeline.check_external_failed()?; if external_failed { - let exit_code = pipeline.drain_with_exit_code()?; - stack.add_env_var("LAST_EXIT_CODE".into(), Value::int(exit_code, call.head)); + let status = pipeline.drain()?; + let code = status.map(|status| status.code()).unwrap_or(0); + stack.add_env_var("LAST_EXIT_CODE".into(), Value::int(code.into(), call.head)); let err_value = Value::nothing(call.head); handle_catch(err_value, catch_block, engine_state, stack, eval_block) } else { diff --git a/crates/nu-cmd-lang/src/core_commands/while_.rs b/crates/nu-cmd-lang/src/core_commands/while_.rs index e42e4ab6d1..bf9076aa0c 100644 --- a/crates/nu-cmd-lang/src/core_commands/while_.rs +++ b/crates/nu-cmd-lang/src/core_commands/while_.rs @@ -70,14 +70,16 @@ impl Command for While { Err(err) => { return Err(err); } - Ok(pipeline) => { - let exit_code = pipeline.drain_with_exit_code()?; - if exit_code != 0 { - return Ok( - PipelineData::new_external_stream_with_only_exit_code( - exit_code, - ), - ); + Ok(data) => { + if let Some(status) = data.drain()? { + let code = status.code(); + if code != 0 { + return Ok( + PipelineData::new_external_stream_with_only_exit_code( + code, + ), + ); + } } } } diff --git a/crates/nu-cmd-lang/src/example_support.rs b/crates/nu-cmd-lang/src/example_support.rs index 860572f349..bb03bbaf8c 100644 --- a/crates/nu-cmd-lang/src/example_support.rs +++ b/crates/nu-cmd-lang/src/example_support.rs @@ -122,10 +122,9 @@ pub fn eval_block( stack.add_env_var("PWD".to_string(), Value::test_string(cwd.to_string_lossy())); - match nu_engine::eval_block::(engine_state, &mut stack, &block, input) { - Err(err) => panic!("test eval error in `{}`: {:?}", "TODO", err), - Ok(result) => result.into_value(Span::test_data()), - } + nu_engine::eval_block::(engine_state, &mut stack, &block, input) + .and_then(|data| data.into_value(Span::test_data())) + .unwrap_or_else(|err| panic!("test eval error in `{}`: {:?}", "TODO", err)) } pub fn check_example_evaluates_to_expected_output( diff --git a/crates/nu-color-config/src/style_computer.rs b/crates/nu-color-config/src/style_computer.rs index 2293439183..91907c1428 100644 --- a/crates/nu-color-config/src/style_computer.rs +++ b/crates/nu-color-config/src/style_computer.rs @@ -58,11 +58,11 @@ impl<'a> StyleComputer<'a> { Some(ComputableStyle::Closure(closure, span)) => { let result = ClosureEvalOnce::new(self.engine_state, self.stack, closure.clone()) .debug(false) - .run_with_value(value.clone()); + .run_with_value(value.clone()) + .and_then(|data| data.into_value(*span)); match result { - Ok(v) => { - let value = v.into_value(*span); + Ok(value) => { // These should be the same color data forms supported by color_config. match value { Value::Record { .. } => color_record_to_nustyle(&value), diff --git a/crates/nu-command/src/bytes/starts_with.rs b/crates/nu-command/src/bytes/starts_with.rs index 69187894b4..2d7ca3e26a 100644 --- a/crates/nu-command/src/bytes/starts_with.rs +++ b/crates/nu-command/src/bytes/starts_with.rs @@ -60,63 +60,13 @@ impl Command for BytesStartsWith { pattern, cell_paths, }; - - match input { - PipelineData::ExternalStream { - stdout: Some(stream), - span, - .. - } => { - let mut i = 0; - - for item in stream { - let byte_slice = match &item { - // String and binary data are valid byte patterns - Ok(Value::String { val, .. }) => val.as_bytes(), - Ok(Value::Binary { val, .. }) => val, - // If any Error value is output, echo it back - Ok(v @ Value::Error { .. }) => return Ok(v.clone().into_pipeline_data()), - // Unsupported data - Ok(other) => { - return Ok(Value::error( - ShellError::OnlySupportsThisInputType { - exp_input_type: "string and binary".into(), - wrong_type: other.get_type().to_string(), - dst_span: span, - src_span: other.span(), - }, - span, - ) - .into_pipeline_data()); - } - Err(err) => return Err(err.to_owned()), - }; - - let max = byte_slice.len().min(arg.pattern.len() - i); - - if byte_slice[..max] == arg.pattern[i..i + max] { - i += max; - - if i >= arg.pattern.len() { - return Ok(Value::bool(true, span).into_pipeline_data()); - } - } else { - return Ok(Value::bool(false, span).into_pipeline_data()); - } - } - - // We reached the end of the stream and never returned, - // the pattern wasn't exhausted so it probably doesn't match - Ok(Value::bool(false, span).into_pipeline_data()) - } - _ => operate( - starts_with, - arg, - input, - call.head, - engine_state.ctrlc.clone(), - ), - } + operate( + starts_with, + arg, + input, + call.head, + engine_state.ctrlc.clone(), + ) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/charting/histogram.rs b/crates/nu-command/src/charting/histogram.rs index 35a9d82a3d..52964b087d 100755 --- a/crates/nu-command/src/charting/histogram.rs +++ b/crates/nu-command/src/charting/histogram.rs @@ -121,7 +121,7 @@ impl Command for Histogram { }; let span = call.head; - let data_as_value = input.into_value(span); + let data_as_value = input.into_value(span)?; let value_span = data_as_value.span(); // `input` is not a list, here we can return an error. run_histogram( diff --git a/crates/nu-command/src/conversions/into/binary.rs b/crates/nu-command/src/conversions/into/binary.rs index 6fb997a590..479b0fc7d7 100644 --- a/crates/nu-command/src/conversions/into/binary.rs +++ b/crates/nu-command/src/conversions/into/binary.rs @@ -127,25 +127,15 @@ fn into_binary( let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - match input { - PipelineData::ExternalStream { stdout: None, .. } => { - Ok(Value::binary(vec![], head).into_pipeline_data()) - } - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - // TODO: in the future, we may want this to stream out, converting each to bytes - let output = stream.into_bytes()?; - Ok(Value::binary(output.item, head).into_pipeline_data()) - } - _ => { - let args = Arguments { - cell_paths, - compact: call.has_flag(engine_state, stack, "compact")?, - }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) - } + if let PipelineData::ByteStream(stream, ..) = input { + // TODO: in the future, we may want this to stream out, converting each to bytes + Ok(Value::binary(stream.into_bytes()?, head).into_pipeline_data()) + } else { + let args = Arguments { + cell_paths, + compact: call.has_flag(engine_state, stack, "compact")?, + }; + operate(action, args, input, call.head, engine_state.ctrlc.clone()) } } diff --git a/crates/nu-command/src/conversions/into/cell_path.rs b/crates/nu-command/src/conversions/into/cell_path.rs index 4faa6e83d6..6da317abd3 100644 --- a/crates/nu-command/src/conversions/into/cell_path.rs +++ b/crates/nu-command/src/conversions/into/cell_path.rs @@ -101,11 +101,11 @@ fn into_cell_path(call: &Call, input: PipelineData) -> Result = stream.into_iter().collect(); Ok(list_to_cell_path(&list, head)?.into_pipeline_data()) } - PipelineData::ExternalStream { span, .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, int".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: head, - src_span: span, + src_span: stream.span(), }), PipelineData::Empty => Err(ShellError::PipelineEmpty { dst_span: head }), } diff --git a/crates/nu-command/src/conversions/into/glob.rs b/crates/nu-command/src/conversions/into/glob.rs index 8c167b0dc0..e5d03093f4 100644 --- a/crates/nu-command/src/conversions/into/glob.rs +++ b/crates/nu-command/src/conversions/into/glob.rs @@ -82,20 +82,12 @@ fn glob_helper( let head = call.head; let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - let args = Arguments { cell_paths }; - match input { - PipelineData::ExternalStream { stdout: None, .. } => { - Ok(Value::glob(String::new(), false, head).into_pipeline_data()) - } - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - // TODO: in the future, we may want this to stream out, converting each to bytes - let output = stream.into_string()?; - Ok(Value::glob(output.item, false, head).into_pipeline_data()) - } - _ => operate(action, args, input, head, engine_state.ctrlc.clone()), + if let PipelineData::ByteStream(stream, ..) = input { + // TODO: in the future, we may want this to stream out, converting each to bytes + Ok(Value::glob(stream.into_string()?, false, head).into_pipeline_data()) + } else { + let args = Arguments { cell_paths }; + operate(action, args, input, head, engine_state.ctrlc.clone()) } } diff --git a/crates/nu-command/src/conversions/into/record.rs b/crates/nu-command/src/conversions/into/record.rs index c9342e8e39..e867f06e15 100644 --- a/crates/nu-command/src/conversions/into/record.rs +++ b/crates/nu-command/src/conversions/into/record.rs @@ -108,7 +108,7 @@ fn into_record( call: &Call, input: PipelineData, ) -> Result { - let input = input.into_value(call.head); + let input = input.into_value(call.head)?; let input_type = input.get_type(); let span = input.span(); let res = match input { diff --git a/crates/nu-command/src/conversions/into/string.rs b/crates/nu-command/src/conversions/into/string.rs index bc791a37b2..eda4f7e5a5 100644 --- a/crates/nu-command/src/conversions/into/string.rs +++ b/crates/nu-command/src/conversions/into/string.rs @@ -155,26 +155,18 @@ fn string_helper( } let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - let config = engine_state.get_config().clone(); - let args = Arguments { - decimals_value, - cell_paths, - config, - }; - match input { - PipelineData::ExternalStream { stdout: None, .. } => { - Ok(Value::string(String::new(), head).into_pipeline_data()) - } - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - // TODO: in the future, we may want this to stream out, converting each to bytes - let output = stream.into_string()?; - Ok(Value::string(output.item, head).into_pipeline_data()) - } - _ => operate(action, args, input, head, engine_state.ctrlc.clone()), + if let PipelineData::ByteStream(stream, ..) = input { + // TODO: in the future, we may want this to stream out, converting each to bytes + Ok(Value::string(stream.into_string()?, head).into_pipeline_data()) + } else { + let config = engine_state.get_config().clone(); + let args = Arguments { + decimals_value, + cell_paths, + config, + }; + operate(action, args, input, head, engine_state.ctrlc.clone()) } } diff --git a/crates/nu-command/src/database/values/sqlite.rs b/crates/nu-command/src/database/values/sqlite.rs index 9778f44993..483da7672e 100644 --- a/crates/nu-command/src/database/values/sqlite.rs +++ b/crates/nu-command/src/database/values/sqlite.rs @@ -91,7 +91,7 @@ impl SQLiteDatabase { } pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(value) } diff --git a/crates/nu-command/src/debug/inspect.rs b/crates/nu-command/src/debug/inspect.rs index 681d2ef6c6..ad6163c7b6 100644 --- a/crates/nu-command/src/debug/inspect.rs +++ b/crates/nu-command/src/debug/inspect.rs @@ -29,7 +29,7 @@ impl Command for Inspect { input: PipelineData, ) -> Result { let input_metadata = input.metadata(); - let input_val = input.into_value(call.head); + let input_val = input.into_value(call.head)?; if input_val.is_nothing() { return Err(ShellError::PipelineEmpty { dst_span: call.head, diff --git a/crates/nu-command/src/debug/timeit.rs b/crates/nu-command/src/debug/timeit.rs index 92f8fe18cd..a445679b81 100644 --- a/crates/nu-command/src/debug/timeit.rs +++ b/crates/nu-command/src/debug/timeit.rs @@ -53,13 +53,12 @@ impl Command for TimeIt { eval_block(engine_state, stack, block, input)? } else { let eval_expression_with_input = get_eval_expression_with_input(engine_state); - eval_expression_with_input(engine_state, stack, command_to_run, input) - .map(|res| res.0)? + eval_expression_with_input(engine_state, stack, command_to_run, input)?.0 } } else { PipelineData::empty() } - .into_value(call.head); + .into_value(call.head)?; let end_time = Instant::now(); diff --git a/crates/nu-command/src/filesystem/open.rs b/crates/nu-command/src/filesystem/open.rs index 23664bb576..5fb8527511 100644 --- a/crates/nu-command/src/filesystem/open.rs +++ b/crates/nu-command/src/filesystem/open.rs @@ -1,8 +1,8 @@ use super::util::get_rest_for_glob_pattern; #[allow(deprecated)] use nu_engine::{command_prelude::*, current_dir, get_eval_block}; -use nu_protocol::{BufferedReader, DataSource, NuGlob, PipelineMetadata, RawStream}; -use std::{io::BufReader, path::Path}; +use nu_protocol::{ByteStream, DataSource, NuGlob, PipelineMetadata}; +use std::path::Path; #[cfg(feature = "sqlite")] use crate::database::SQLiteDatabase; @@ -143,23 +143,13 @@ impl Command for Open { } }; - let buf_reader = BufReader::new(file); - - let file_contents = PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(BufferedReader::new(buf_reader)), - ctrlc.clone(), - call_span, - None, - )), - stderr: None, - exit_code: None, - span: call_span, - metadata: Some(PipelineMetadata { + let stream = PipelineData::ByteStream( + ByteStream::file(file, call_span, ctrlc.clone()), + Some(PipelineMetadata { data_source: DataSource::FilePath(path.to_path_buf()), }), - trim_end_newline: false, - }; + ); + let exts_opt: Option> = if raw { None } else { @@ -184,9 +174,9 @@ impl Command for Open { let decl = engine_state.get_decl(converter_id); let command_output = if let Some(block_id) = decl.get_block_id() { let block = engine_state.get_block(block_id); - eval_block(engine_state, stack, block, file_contents) + eval_block(engine_state, stack, block, stream) } else { - decl.run(engine_state, stack, &Call::new(call_span), file_contents) + decl.run(engine_state, stack, &Call::new(call_span), stream) }; output.push(command_output.map_err(|inner| { ShellError::GenericError{ @@ -198,7 +188,7 @@ impl Command for Open { } })?); } - None => output.push(file_contents), + None => output.push(stream), } } } diff --git a/crates/nu-command/src/filesystem/save.rs b/crates/nu-command/src/filesystem/save.rs index 0826284798..ca9943eafb 100644 --- a/crates/nu-command/src/filesystem/save.rs +++ b/crates/nu-command/src/filesystem/save.rs @@ -5,12 +5,15 @@ use nu_engine::{command_prelude::*, current_dir}; use nu_path::expand_path_with; use nu_protocol::{ ast::{Expr, Expression}, - DataSource, OutDest, PipelineMetadata, RawStream, + byte_stream::copy_with_interrupt, + process::ChildPipe, + ByteStreamSource, DataSource, OutDest, PipelineMetadata, }; use std::{ fs::File, - io::Write, + io::{self, BufRead, BufReader, Read, Write}, path::{Path, PathBuf}, + sync::{atomic::AtomicBool, Arc}, thread, }; @@ -104,12 +107,7 @@ impl Command for Save { }); match input { - PipelineData::ExternalStream { - stdout, - stderr, - metadata, - .. - } => { + PipelineData::ByteStream(stream, metadata) => { check_saving_to_source_file(metadata.as_ref(), &path, stderr_path.as_ref())?; let (file, stderr_file) = get_files( @@ -121,40 +119,97 @@ impl Command for Save { force, )?; - match (stdout, stderr) { - (Some(stdout), stderr) => { - // delegate a thread to redirect stderr to result. - let handler = stderr - .map(|stderr| match stderr_file { - Some(stderr_file) => thread::Builder::new() - .name("stderr redirector".to_string()) - .spawn(move || { - stream_to_file(stderr, stderr_file, span, progress) - }), - None => thread::Builder::new() - .name("stderr redirector".to_string()) - .spawn(move || stderr.drain()), - }) - .transpose() - .err_span(span)?; + let size = stream.known_size(); + let ctrlc = engine_state.ctrlc.clone(); - let res = stream_to_file(stdout, file, span, progress); - if let Some(h) = handler { - h.join().map_err(|err| ShellError::ExternalCommand { - label: "Fail to receive external commands stderr message" - .to_string(), - help: format!("{err:?}"), - span, - })??; - } - res?; + match stream.into_source() { + ByteStreamSource::Read(read) => { + stream_to_file(read, size, ctrlc, file, span, progress)?; } - (None, Some(stderr)) => match stderr_file { - Some(stderr_file) => stream_to_file(stderr, stderr_file, span, progress)?, - None => stderr.drain()?, - }, - (None, None) => {} - }; + ByteStreamSource::File(source) => { + stream_to_file(source, size, ctrlc, file, span, progress)?; + } + ByteStreamSource::Child(mut child) => { + fn write_or_consume_stderr( + stderr: ChildPipe, + file: Option, + span: Span, + ctrlc: Option>, + progress: bool, + ) -> Result<(), ShellError> { + if let Some(file) = file { + match stderr { + ChildPipe::Pipe(pipe) => { + stream_to_file(pipe, None, ctrlc, file, span, progress) + } + ChildPipe::Tee(tee) => { + stream_to_file(tee, None, ctrlc, file, span, progress) + } + }? + } else { + match stderr { + ChildPipe::Pipe(mut pipe) => { + io::copy(&mut pipe, &mut io::sink()) + } + ChildPipe::Tee(mut tee) => io::copy(&mut tee, &mut io::sink()), + } + .err_span(span)?; + } + Ok(()) + } + + match (child.stdout.take(), child.stderr.take()) { + (Some(stdout), stderr) => { + // delegate a thread to redirect stderr to result. + let handler = stderr + .map(|stderr| { + let ctrlc = ctrlc.clone(); + thread::Builder::new().name("stderr saver".into()).spawn( + move || { + write_or_consume_stderr( + stderr, + stderr_file, + span, + ctrlc, + progress, + ) + }, + ) + }) + .transpose() + .err_span(span)?; + + let res = match stdout { + ChildPipe::Pipe(pipe) => { + stream_to_file(pipe, None, ctrlc, file, span, progress) + } + ChildPipe::Tee(tee) => { + stream_to_file(tee, None, ctrlc, file, span, progress) + } + }; + if let Some(h) = handler { + h.join().map_err(|err| ShellError::ExternalCommand { + label: "Fail to receive external commands stderr message" + .to_string(), + help: format!("{err:?}"), + span, + })??; + } + res?; + } + (None, Some(stderr)) => { + write_or_consume_stderr( + stderr, + stderr_file, + span, + ctrlc, + progress, + )?; + } + (None, None) => {} + }; + } + } Ok(PipelineData::Empty) } @@ -302,8 +357,7 @@ fn input_to_bytes( ) -> Result, ShellError> { let ext = if raw { None - // if is extern stream , in other words , not value - } else if let PipelineData::ExternalStream { .. } = input { + } else if let PipelineData::ByteStream(..) = input { None } else if let PipelineData::Value(Value::String { .. }, ..) = input { None @@ -318,7 +372,7 @@ fn input_to_bytes( input }; - value_to_bytes(input.into_value(span)) + value_to_bytes(input.into_value(span)?) } /// Convert given data into content of file of specified extension if @@ -448,84 +502,54 @@ fn get_files( } fn stream_to_file( - mut stream: RawStream, + mut source: impl Read, + known_size: Option, + ctrlc: Option>, mut file: File, span: Span, progress: bool, ) -> Result<(), ShellError> { - // https://github.com/nushell/nushell/pull/9377 contains the reason - // for not using BufWriter - let writer = &mut file; + // https://github.com/nushell/nushell/pull/9377 contains the reason for not using `BufWriter` + if progress { + let mut bytes_processed = 0; - let mut bytes_processed: u64 = 0; - let bytes_processed_p = &mut bytes_processed; - let file_total_size = stream.known_size; - let mut process_failed = false; - let process_failed_p = &mut process_failed; + let mut bar = progress_bar::NuProgressBar::new(known_size); - // Create the progress bar - // It looks a bit messy but I am doing it this way to avoid - // creating the bar when is not needed - let (mut bar_opt, bar_opt_clone) = if progress { - let tmp_bar = progress_bar::NuProgressBar::new(file_total_size); - let tmp_bar_clone = tmp_bar.clone(); + // TODO: reduce the number of progress bar updates? - (Some(tmp_bar), Some(tmp_bar_clone)) - } else { - (None, None) - }; + let mut reader = BufReader::new(source); - stream.try_for_each(move |result| { - let buf = match result { - Ok(v) => match v { - Value::String { val, .. } => val.into_bytes(), - Value::Binary { val, .. } => val, - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => return Err(*error), - other => { - return Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "string or binary".into(), - wrong_type: other.get_type().to_string(), - dst_span: span, - src_span: other.span(), - }); + let res = loop { + if nu_utils::ctrl_c::was_pressed(&ctrlc) { + bar.abandoned_msg("# Cancelled #".to_owned()); + return Ok(()); + } + + match reader.fill_buf() { + Ok(&[]) => break Ok(()), + Ok(buf) => { + file.write_all(buf).err_span(span)?; + let len = buf.len(); + reader.consume(len); + bytes_processed += len as u64; + bar.update_bar(bytes_processed); } - }, - Err(err) => { - *process_failed_p = true; - return Err(err); + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => break Err(e), } }; - // If the `progress` flag is set then - if progress { - // Update the total amount of bytes that has been saved and then print the progress bar - *bytes_processed_p += buf.len() as u64; - if let Some(bar) = &mut bar_opt { - bar.update_bar(*bytes_processed_p); - } - } - - if let Err(err) = writer.write_all(&buf) { - *process_failed_p = true; - return Err(ShellError::IOError { - msg: err.to_string(), - }); - } - Ok(()) - })?; - - // If the `progress` flag is set then - if progress { // If the process failed, stop the progress bar with an error message. - if process_failed { - if let Some(bar) = bar_opt_clone { - bar.abandoned_msg("# Error while saving #".to_owned()); - } + if let Err(err) = res { + let _ = file.flush(); + bar.abandoned_msg("# Error while saving #".to_owned()); + Err(err.into_spanned(span).into()) + } else { + file.flush().err_span(span)?; + Ok(()) } + } else { + copy_with_interrupt(&mut source, &mut file, span, ctrlc.as_deref())?; + Ok(()) } - - file.flush()?; - - Ok(()) } diff --git a/crates/nu-command/src/filters/columns.rs b/crates/nu-command/src/filters/columns.rs index b6e15af8df..da8bc5ae57 100644 --- a/crates/nu-command/src/filters/columns.rs +++ b/crates/nu-command/src/filters/columns.rs @@ -125,13 +125,11 @@ fn getcol(head: Span, input: PipelineData) -> Result { .into_pipeline_data() .set_metadata(metadata)) } - PipelineData::ExternalStream { .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "record or table".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: head, - src_span: input - .span() - .expect("PipelineData::ExternalStream had no span"), + src_span: stream.span(), }), } } diff --git a/crates/nu-command/src/filters/drop/column.rs b/crates/nu-command/src/filters/drop/column.rs index 3354492570..01c13deee4 100644 --- a/crates/nu-command/src/filters/drop/column.rs +++ b/crates/nu-command/src/filters/drop/column.rs @@ -133,11 +133,11 @@ fn drop_cols( } } PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::ExternalStream { span, .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "table or record".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: head, - src_span: span, + src_span: stream.span(), }), } } diff --git a/crates/nu-command/src/filters/each.rs b/crates/nu-command/src/filters/each.rs index 65d61fd3a8..a074f63abb 100644 --- a/crates/nu-command/src/filters/each.rs +++ b/crates/nu-command/src/filters/each.rs @@ -129,7 +129,9 @@ with 'transpose' first."# } Some(Value::list(vals, span)) } - Ok(data) => Some(data.into_value(head)), + Ok(data) => Some(data.into_value(head).unwrap_or_else(|err| { + Value::error(chain_error_with_input(err, is_error, span), span) + })), Err(ShellError::Continue { span }) => Some(Value::nothing(span)), Err(ShellError::Break { .. }) => None, Err(error) => { @@ -140,37 +142,39 @@ with 'transpose' first."# }) .into_pipeline_data(head, engine_state.ctrlc.clone())) } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - let mut closure = ClosureEval::new(engine_state, stack, closure); - Ok(stream - .into_iter() - .map_while(move |value| { - let value = match value { - Ok(value) => value, - Err(ShellError::Continue { span }) => { - return Some(Value::nothing(span)) - } - Err(ShellError::Break { .. }) => return None, - Err(err) => return Some(Value::error(err, head)), - }; + PipelineData::ByteStream(stream, ..) => { + if let Some(chunks) = stream.chunks() { + let mut closure = ClosureEval::new(engine_state, stack, closure); + Ok(chunks + .map_while(move |value| { + let value = match value { + Ok(value) => value, + Err(ShellError::Continue { span }) => { + return Some(Value::nothing(span)) + } + Err(ShellError::Break { .. }) => return None, + Err(err) => return Some(Value::error(err, head)), + }; - let span = value.span(); - let is_error = value.is_error(); - match closure.run_with_value(value) { - Ok(data) => Some(data.into_value(head)), - Err(ShellError::Continue { span }) => Some(Value::nothing(span)), - Err(ShellError::Break { .. }) => None, - Err(error) => { - let error = chain_error_with_input(error, is_error, span); - Some(Value::error(error, span)) + let span = value.span(); + let is_error = value.is_error(); + match closure + .run_with_value(value) + .and_then(|data| data.into_value(head)) + { + Ok(value) => Some(value), + Err(ShellError::Continue { span }) => Some(Value::nothing(span)), + Err(ShellError::Break { .. }) => None, + Err(error) => { + let error = chain_error_with_input(error, is_error, span); + Some(Value::error(error, span)) + } } - } - }) - .into_pipeline_data(head, engine_state.ctrlc.clone())) + }) + .into_pipeline_data(head, engine_state.ctrlc.clone())) + } else { + Ok(PipelineData::Empty) + } } // This match allows non-iterables to be accepted, // which is currently considered undesirable (Nov 2022). diff --git a/crates/nu-command/src/filters/empty.rs b/crates/nu-command/src/filters/empty.rs index fd55921414..f4dd428b77 100644 --- a/crates/nu-command/src/filters/empty.rs +++ b/crates/nu-command/src/filters/empty.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use std::io::Read; pub fn empty( engine_state: &EngineState, @@ -36,29 +37,26 @@ pub fn empty( } else { match input { PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::ExternalStream { stdout, .. } => match stdout { - Some(s) => { - let bytes = s.into_bytes(); - - match bytes { - Ok(s) => { - if negate { - Ok(Value::bool(!s.item.is_empty(), head).into_pipeline_data()) - } else { - Ok(Value::bool(s.item.is_empty(), head).into_pipeline_data()) - } + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + match stream.reader() { + Some(reader) => { + let is_empty = reader.bytes().next().transpose().err_span(span)?.is_none(); + if negate { + Ok(Value::bool(!is_empty, head).into_pipeline_data()) + } else { + Ok(Value::bool(is_empty, head).into_pipeline_data()) + } + } + None => { + if negate { + Ok(Value::bool(false, head).into_pipeline_data()) + } else { + Ok(Value::bool(true, head).into_pipeline_data()) } - Err(err) => Err(err), } } - None => { - if negate { - Ok(Value::bool(false, head).into_pipeline_data()) - } else { - Ok(Value::bool(true, head).into_pipeline_data()) - } - } - }, + } PipelineData::ListStream(s, ..) => { let empty = s.into_iter().next().is_none(); if negate { diff --git a/crates/nu-command/src/filters/filter.rs b/crates/nu-command/src/filters/filter.rs index b158dd3be3..1ba1508839 100644 --- a/crates/nu-command/src/filters/filter.rs +++ b/crates/nu-command/src/filters/filter.rs @@ -58,33 +58,13 @@ a variable. On the other hand, the "row condition" syntax is not supported."# | PipelineData::ListStream(..) => { let mut closure = ClosureEval::new(engine_state, stack, closure); Ok(input - .into_iter() - .filter_map(move |value| match closure.run_with_value(value.clone()) { - Ok(pred) => pred.into_value(head).is_true().then_some(value), - Err(err) => { - let span = value.span(); - let err = chain_error_with_input(err, value.is_error(), span); - Some(Value::error(err, span)) - } - }) - .into_pipeline_data(head, engine_state.ctrlc.clone())) - } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - let mut closure = ClosureEval::new(engine_state, stack, closure); - Ok(stream .into_iter() .filter_map(move |value| { - let value = match value { - Ok(value) => value, - Err(err) => return Some(Value::error(err, head)), - }; - - match closure.run_with_value(value.clone()) { - Ok(pred) => pred.into_value(head).is_true().then_some(value), + match closure + .run_with_value(value.clone()) + .and_then(|data| data.into_value(head)) + { + Ok(cond) => cond.is_true().then_some(value), Err(err) => { let span = value.span(); let err = chain_error_with_input(err, value.is_error(), span); @@ -94,14 +74,43 @@ a variable. On the other hand, the "row condition" syntax is not supported."# }) .into_pipeline_data(head, engine_state.ctrlc.clone())) } + PipelineData::ByteStream(stream, ..) => { + if let Some(chunks) = stream.chunks() { + let mut closure = ClosureEval::new(engine_state, stack, closure); + Ok(chunks + .into_iter() + .filter_map(move |value| { + let value = match value { + Ok(value) => value, + Err(err) => return Some(Value::error(err, head)), + }; + + match closure + .run_with_value(value.clone()) + .and_then(|data| data.into_value(head)) + { + Ok(cond) => cond.is_true().then_some(value), + Err(err) => { + let span = value.span(); + let err = chain_error_with_input(err, value.is_error(), span); + Some(Value::error(err, span)) + } + } + }) + .into_pipeline_data(head, engine_state.ctrlc.clone())) + } else { + Ok(PipelineData::Empty) + } + } // This match allows non-iterables to be accepted, // which is currently considered undesirable (Nov 2022). PipelineData::Value(value, ..) => { let result = ClosureEvalOnce::new(engine_state, stack, closure) - .run_with_value(value.clone()); + .run_with_value(value.clone()) + .and_then(|data| data.into_value(head)); Ok(match result { - Ok(pred) => pred.into_value(head).is_true().then_some(value), + Ok(cond) => cond.is_true().then_some(value), Err(err) => { let span = value.span(); let err = chain_error_with_input(err, value.is_error(), span); diff --git a/crates/nu-command/src/filters/find.rs b/crates/nu-command/src/filters/find.rs index b45fe8d810..dfdef66969 100644 --- a/crates/nu-command/src/filters/find.rs +++ b/crates/nu-command/src/filters/find.rs @@ -447,57 +447,35 @@ fn find_with_rest_and_highlight( Ok(PipelineData::ListStream(stream, metadata)) } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - let mut output: Vec = vec![]; - for filter_val in stream { - match filter_val { - Ok(value) => { - let span = value.span(); - match value { - Value::String { val, .. } => { - let split_char = if val.contains("\r\n") { "\r\n" } else { "\n" }; + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + if let Some(lines) = stream.lines() { + let terms = lower_terms + .into_iter() + .map(|term| term.to_expanded_string("", &filter_config).to_lowercase()) + .collect::>(); - for line in val.split(split_char) { - for term in lower_terms.iter() { - let term_str = term.to_expanded_string("", &filter_config); - let lower_val = line.to_lowercase(); - if lower_val.contains( - &term.to_expanded_string("", &config).to_lowercase(), - ) { - output.push(Value::string( - highlight_search_string( - line, - &term_str, - &string_style, - &highlight_style, - )?, - span, - )) - } - } - } - } - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => return Err(*error), - other => { - return Err(ShellError::UnsupportedInput { - msg: "unsupported type from raw stream".into(), - input: format!("input: {:?}", other.get_type()), - msg_span: span, - input_span: other.span(), - }); - } + let mut output: Vec = vec![]; + for line in lines { + let line = line?.to_lowercase(); + for term in &terms { + if line.contains(term) { + output.push(Value::string( + highlight_search_string( + &line, + term, + &string_style, + &highlight_style, + )?, + span, + )) } } - // Propagate any errors that were in the stream - Err(e) => return Err(e), - }; + } + Ok(Value::list(output, span).into_pipeline_data()) + } else { + Ok(PipelineData::Empty) } - Ok(output.into_pipeline_data(span, ctrlc)) } } } diff --git a/crates/nu-command/src/filters/first.rs b/crates/nu-command/src/filters/first.rs index 1bc51f2562..e581c3e84d 100644 --- a/crates/nu-command/src/filters/first.rs +++ b/crates/nu-command/src/filters/first.rs @@ -170,11 +170,11 @@ fn first_helper( )) } } - PipelineData::ExternalStream { span, .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: head, - src_span: span, + src_span: stream.span(), }), PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), diff --git a/crates/nu-command/src/filters/get.rs b/crates/nu-command/src/filters/get.rs index a481372db1..07f0ea9440 100644 --- a/crates/nu-command/src/filters/get.rs +++ b/crates/nu-command/src/filters/get.rs @@ -81,7 +81,7 @@ If multiple cell paths are given, this will produce a list of values."# let paths = std::iter::once(cell_path).chain(rest); - let input = input.into_value(span); + let input = input.into_value(span)?; for path in paths { let val = input.clone().follow_cell_path(&path.members, !sensitive); diff --git a/crates/nu-command/src/filters/group_by.rs b/crates/nu-command/src/filters/group_by.rs index 24559c1eca..c1d76ebe08 100644 --- a/crates/nu-command/src/filters/group_by.rs +++ b/crates/nu-command/src/filters/group_by.rs @@ -207,7 +207,7 @@ fn group_closure( for value in values { let key = closure .run_with_value(value.clone())? - .into_value(span) + .into_value(span)? .coerce_into_string()?; groups.entry(key).or_default().push(value); diff --git a/crates/nu-command/src/filters/headers.rs b/crates/nu-command/src/filters/headers.rs index d7492d0b76..6e63c33ff9 100644 --- a/crates/nu-command/src/filters/headers.rs +++ b/crates/nu-command/src/filters/headers.rs @@ -66,7 +66,7 @@ impl Command for Headers { let config = engine_state.get_config(); let metadata = input.metadata(); let span = input.span().unwrap_or(call.head); - let value = input.into_value(span); + let value = input.into_value(span)?; let Value::List { vals: table, .. } = value else { return Err(ShellError::TypeMismatch { err_message: "not a table".to_string(), diff --git a/crates/nu-command/src/filters/insert.rs b/crates/nu-command/src/filters/insert.rs index d9fb165a16..e8794304c8 100644 --- a/crates/nu-command/src/filters/insert.rs +++ b/crates/nu-command/src/filters/insert.rs @@ -190,7 +190,7 @@ fn insert( let value = value.unwrap_or(Value::nothing(head)); let new_value = ClosureEvalOnce::new(engine_state, stack, *val) .run_with_value(value.clone())? - .into_value(head); + .into_value(head)?; pre_elems.push(new_value); if !end_of_stream { @@ -261,8 +261,8 @@ fn insert( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ExternalStream { .. } => Err(ShellError::IncompatiblePathAccess { - type_name: "external stream".to_string(), + PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { + type_name: "byte stream".to_string(), span: head, }), } @@ -284,7 +284,7 @@ fn insert_value_by_closure( value.clone() }; - let new_value = closure.run_with_value(value_at_path)?.into_value(span); + let new_value = closure.run_with_value(value_at_path)?.into_value(span)?; value.insert_data_at_cell_path(cell_path, new_value, span) } @@ -304,7 +304,7 @@ fn insert_single_value_by_closure( value.clone() }; - let new_value = closure.run_with_value(value_at_path)?.into_value(span); + let new_value = closure.run_with_value(value_at_path)?.into_value(span)?; value.insert_data_at_cell_path(cell_path, new_value, span) } diff --git a/crates/nu-command/src/filters/items.rs b/crates/nu-command/src/filters/items.rs index f0cba01888..6afc0bc536 100644 --- a/crates/nu-command/src/filters/items.rs +++ b/crates/nu-command/src/filters/items.rs @@ -55,10 +55,11 @@ impl Command for Items { let result = closure .add_arg(Value::string(col, span)) .add_arg(val) - .run_with_input(PipelineData::Empty); + .run_with_input(PipelineData::Empty) + .and_then(|data| data.into_value(head)); match result { - Ok(data) => Some(data.into_value(head)), + Ok(value) => Some(value), Err(ShellError::Break { .. }) => None, Err(err) => { let err = chain_error_with_input(err, false, span); @@ -77,20 +78,18 @@ impl Command for Items { }), } } - PipelineData::ListStream(..) => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ListStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "record".into(), wrong_type: "stream".into(), - dst_span: head, - src_span: head, + dst_span: call.head, + src_span: stream.span(), + }), + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "record".into(), + wrong_type: "byte stream".into(), + dst_span: call.head, + src_span: stream.span(), }), - PipelineData::ExternalStream { span, .. } => { - Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "record".into(), - wrong_type: "raw data".into(), - dst_span: head, - src_span: span, - }) - } } .map(|data| data.set_metadata(metadata)) } diff --git a/crates/nu-command/src/filters/join.rs b/crates/nu-command/src/filters/join.rs index 343cc0eb19..f5e6d63deb 100644 --- a/crates/nu-command/src/filters/join.rs +++ b/crates/nu-command/src/filters/join.rs @@ -75,7 +75,7 @@ impl Command for Join { let join_type = join_type(engine_state, stack, call)?; // FIXME: we should handle ListStreams properly instead of collecting - let collected_input = input.into_value(span); + let collected_input = input.into_value(span)?; match (&collected_input, &table_2, &l_on, &r_on) { ( diff --git a/crates/nu-command/src/filters/last.rs b/crates/nu-command/src/filters/last.rs index f41b7c7e4d..7530126c26 100644 --- a/crates/nu-command/src/filters/last.rs +++ b/crates/nu-command/src/filters/last.rs @@ -160,14 +160,12 @@ impl Command for Last { }), } } - PipelineData::ExternalStream { span, .. } => { - Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "raw data".into(), - dst_span: head, - src_span: span, - }) - } + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: "byte stream".into(), + dst_span: head, + src_span: stream.span(), + }), PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), wrong_type: "null".into(), diff --git a/crates/nu-command/src/filters/lines.rs b/crates/nu-command/src/filters/lines.rs index e3e0f5d9fd..0b037dcaac 100644 --- a/crates/nu-command/src/filters/lines.rs +++ b/crates/nu-command/src/filters/lines.rs @@ -1,6 +1,4 @@ use nu_engine::command_prelude::*; -use nu_protocol::RawStream; -use std::collections::VecDeque; #[derive(Clone)] pub struct Lines; @@ -33,23 +31,33 @@ impl Command for Lines { let span = input.span().unwrap_or(call.head); match input { - PipelineData::Value(Value::String { val, .. }, ..) => { - let lines = if skip_empty { - val.lines() - .filter_map(|s| { - if s.trim().is_empty() { - None - } else { - Some(Value::string(s, span)) - } - }) - .collect() - } else { - val.lines().map(|s| Value::string(s, span)).collect() - }; + PipelineData::Value(value, ..) => match value { + Value::String { val, .. } => { + let lines = if skip_empty { + val.lines() + .filter_map(|s| { + if s.trim().is_empty() { + None + } else { + Some(Value::string(s, span)) + } + }) + .collect() + } else { + val.lines().map(|s| Value::string(s, span)).collect() + }; - Ok(Value::list(lines, span).into_pipeline_data()) - } + Ok(Value::list(lines, span).into_pipeline_data()) + } + // Propagate existing errors + Value::Error { error, .. } => Err(*error), + value => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "string or byte stream".into(), + wrong_type: value.get_type().to_string(), + dst_span: head, + src_span: value.span(), + }), + }, PipelineData::Empty => Ok(PipelineData::Empty), PipelineData::ListStream(stream, metadata) => { let stream = stream.modify(|iter| { @@ -76,27 +84,18 @@ impl Command for Lines { Ok(PipelineData::ListStream(stream, metadata)) } - PipelineData::Value(val, ..) => { - match val { - // Propagate existing errors - Value::Error { error, .. } => Err(*error), - _ => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "string or raw data".into(), - wrong_type: val.get_type().to_string(), - dst_span: head, - src_span: val.span(), - }), + PipelineData::ByteStream(stream, ..) => { + if let Some(lines) = stream.lines() { + Ok(lines + .map(move |line| match line { + Ok(line) => Value::string(line, head), + Err(err) => Value::error(err, head), + }) + .into_pipeline_data(head, ctrlc)) + } else { + Ok(PipelineData::empty()) } } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - metadata, - .. - } => Ok(RawStreamLinesAdapter::new(stream, head, skip_empty) - .map(move |x| x.unwrap_or_else(|err| Value::error(err, head))) - .into_pipeline_data(head, ctrlc) - .set_metadata(metadata)), } } @@ -112,108 +111,6 @@ impl Command for Lines { } } -#[derive(Debug)] -struct RawStreamLinesAdapter { - inner: RawStream, - inner_complete: bool, - skip_empty: bool, - span: Span, - incomplete_line: String, - queue: VecDeque, -} - -impl Iterator for RawStreamLinesAdapter { - type Item = Result; - - fn next(&mut self) -> Option { - loop { - if let Some(s) = self.queue.pop_front() { - if self.skip_empty && s.trim().is_empty() { - continue; - } - return Some(Ok(Value::string(s, self.span))); - } else { - // inner is complete, feed out remaining state - if self.inner_complete { - return if self.incomplete_line.is_empty() { - None - } else { - Some(Ok(Value::string( - std::mem::take(&mut self.incomplete_line), - self.span, - ))) - }; - } - - // pull more data from inner - if let Some(result) = self.inner.next() { - match result { - Ok(v) => { - let span = v.span(); - match v { - // TODO: Value::Binary support required? - Value::String { val, .. } => { - self.span = span; - - let mut lines = val.lines(); - - // handle incomplete line from previous - if !self.incomplete_line.is_empty() { - if let Some(first) = lines.next() { - self.incomplete_line.push_str(first); - self.queue.push_back(std::mem::take( - &mut self.incomplete_line, - )); - } - } - - // save completed lines - self.queue.extend(lines.map(String::from)); - - if !val.ends_with('\n') { - // incomplete line, save for next time - // if `val` and `incomplete_line` were empty, - // then pop will return none - if let Some(s) = self.queue.pop_back() { - self.incomplete_line = s; - } - } - } - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => return Some(Err(*error)), - other => { - return Some(Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "string".into(), - wrong_type: other.get_type().to_string(), - dst_span: self.span, - src_span: other.span(), - })); - } - } - } - Err(err) => return Some(Err(err)), - } - } else { - self.inner_complete = true; - } - } - } - } -} - -impl RawStreamLinesAdapter { - pub fn new(inner: RawStream, span: Span, skip_empty: bool) -> Self { - Self { - inner, - span, - skip_empty, - incomplete_line: String::new(), - queue: VecDeque::new(), - inner_complete: false, - } - } -} - #[cfg(test)] mod test { use super::*; diff --git a/crates/nu-command/src/filters/par_each.rs b/crates/nu-command/src/filters/par_each.rs index 52c3024270..af72895df6 100644 --- a/crates/nu-command/src/filters/par_each.rs +++ b/crates/nu-command/src/filters/par_each.rs @@ -143,17 +143,16 @@ impl Command for ParEach { .map(move |(index, value)| { let span = value.span(); let is_error = value.is_error(); - let result = + let value = ClosureEvalOnce::new(engine_state, stack, closure.clone()) - .run_with_value(value); - - let value = match result { - Ok(data) => data.into_value(span), - Err(err) => Value::error( - chain_error_with_input(err, is_error, span), - span, - ), - }; + .run_with_value(value) + .and_then(|data| data.into_value(span)) + .unwrap_or_else(|err| { + Value::error( + chain_error_with_input(err, is_error, span), + span, + ) + }); (index, value) }) @@ -170,17 +169,16 @@ impl Command for ParEach { .map(move |(index, value)| { let span = value.span(); let is_error = value.is_error(); - let result = + let value = ClosureEvalOnce::new(engine_state, stack, closure.clone()) - .run_with_value(value); - - let value = match result { - Ok(data) => data.into_value(span), - Err(err) => Value::error( - chain_error_with_input(err, is_error, span), - span, - ), - }; + .run_with_value(value) + .and_then(|data| data.into_value(span)) + .unwrap_or_else(|err| { + Value::error( + chain_error_with_input(err, is_error, span), + span, + ) + }); (index, value) }) @@ -203,40 +201,12 @@ impl Command for ParEach { .map(move |(index, value)| { let span = value.span(); let is_error = value.is_error(); - let result = ClosureEvalOnce::new(engine_state, stack, closure.clone()) - .run_with_value(value); - - let value = match result { - Ok(data) => data.into_value(head), - Err(err) => { - Value::error(chain_error_with_input(err, is_error, span), span) - } - }; - - (index, value) - }) - .collect::>(); - - apply_order(vec).into_pipeline_data(head, engine_state.ctrlc.clone()) - })), - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => Ok(create_pool(max_threads)?.install(|| { - let vec = stream - .enumerate() - .par_bridge() - .map(move |(index, value)| { - let value = match value { - Ok(value) => value, - Err(err) => return (index, Value::error(err, head)), - }; - let value = ClosureEvalOnce::new(engine_state, stack, closure.clone()) .run_with_value(value) - .map(|data| data.into_value(head)) - .unwrap_or_else(|err| Value::error(err, head)); + .and_then(|data| data.into_value(head)) + .unwrap_or_else(|err| { + Value::error(chain_error_with_input(err, is_error, span), span) + }); (index, value) }) @@ -244,6 +214,34 @@ impl Command for ParEach { apply_order(vec).into_pipeline_data(head, engine_state.ctrlc.clone()) })), + PipelineData::ByteStream(stream, ..) => { + if let Some(chunks) = stream.chunks() { + Ok(create_pool(max_threads)?.install(|| { + let vec = chunks + .enumerate() + .par_bridge() + .map(move |(index, value)| { + let value = match value { + Ok(value) => value, + Err(err) => return (index, Value::error(err, head)), + }; + + let value = + ClosureEvalOnce::new(engine_state, stack, closure.clone()) + .run_with_value(value) + .and_then(|data| data.into_value(head)) + .unwrap_or_else(|err| Value::error(err, head)); + + (index, value) + }) + .collect::>(); + + apply_order(vec).into_pipeline_data(head, engine_state.ctrlc.clone()) + })) + } else { + Ok(PipelineData::empty()) + } + } } .and_then(|x| x.filter(|v| !v.is_nothing(), engine_state.ctrlc.clone())) .map(|data| data.set_metadata(metadata)) diff --git a/crates/nu-command/src/filters/reduce.rs b/crates/nu-command/src/filters/reduce.rs index 756fe051a9..fc808ca9af 100644 --- a/crates/nu-command/src/filters/reduce.rs +++ b/crates/nu-command/src/filters/reduce.rs @@ -115,7 +115,7 @@ impl Command for Reduce { .add_arg(value) .add_arg(acc) .run_with_input(PipelineData::Empty)? - .into_value(head); + .into_value(head)?; } Ok(acc.with_span(head).into_pipeline_data()) diff --git a/crates/nu-command/src/filters/reject.rs b/crates/nu-command/src/filters/reject.rs index 251e92c905..f8583d3f47 100644 --- a/crates/nu-command/src/filters/reject.rs +++ b/crates/nu-command/src/filters/reject.rs @@ -173,7 +173,7 @@ fn reject( ) -> Result { let mut unique_rows: HashSet = HashSet::new(); let metadata = input.metadata(); - let val = input.into_value(span); + let val = input.into_value(span)?; let mut val = val; let mut new_columns = vec![]; let mut new_rows = vec![]; diff --git a/crates/nu-command/src/filters/skip/skip_.rs b/crates/nu-command/src/filters/skip/skip_.rs index 1919263aa3..9048b34a58 100644 --- a/crates/nu-command/src/filters/skip/skip_.rs +++ b/crates/nu-command/src/filters/skip/skip_.rs @@ -87,15 +87,14 @@ impl Command for Skip { let ctrlc = engine_state.ctrlc.clone(); let input_span = input.span().unwrap_or(call.head); match input { - PipelineData::ExternalStream { .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: call.head, - src_span: input_span, + src_span: stream.span(), }), PipelineData::Value(Value::Binary { val, .. }, metadata) => { let bytes = val.into_iter().skip(n).collect::>(); - Ok(Value::binary(bytes, input_span).into_pipeline_data_with_metadata(metadata)) } _ => Ok(input diff --git a/crates/nu-command/src/filters/skip/skip_until.rs b/crates/nu-command/src/filters/skip/skip_until.rs index 74deeda84d..bb36785e00 100644 --- a/crates/nu-command/src/filters/skip/skip_until.rs +++ b/crates/nu-command/src/filters/skip/skip_until.rs @@ -85,7 +85,8 @@ impl Command for SkipUntil { .skip_while(move |value| { closure .run_with_value(value.clone()) - .map(|data| data.into_value(head).is_false()) + .and_then(|data| data.into_value(head)) + .map(|cond| cond.is_false()) .unwrap_or(false) }) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) diff --git a/crates/nu-command/src/filters/skip/skip_while.rs b/crates/nu-command/src/filters/skip/skip_while.rs index a832d8f7b1..2747ea6f97 100644 --- a/crates/nu-command/src/filters/skip/skip_while.rs +++ b/crates/nu-command/src/filters/skip/skip_while.rs @@ -90,7 +90,8 @@ impl Command for SkipWhile { .skip_while(move |value| { closure .run_with_value(value.clone()) - .map(|data| data.into_value(head).is_true()) + .and_then(|data| data.into_value(head)) + .map(|cond| cond.is_true()) .unwrap_or(false) }) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) diff --git a/crates/nu-command/src/filters/take/take_.rs b/crates/nu-command/src/filters/take/take_.rs index 01700420b8..12840aa8d6 100644 --- a/crates/nu-command/src/filters/take/take_.rs +++ b/crates/nu-command/src/filters/take/take_.rs @@ -78,14 +78,12 @@ impl Command for Take { stream.modify(|iter| iter.take(rows_desired)), metadata, )), - PipelineData::ExternalStream { span, .. } => { - Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "raw data".into(), - dst_span: head, - src_span: span, - }) - } + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: "byte stream".into(), + dst_span: head, + src_span: stream.span(), + }), PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), wrong_type: "null".into(), diff --git a/crates/nu-command/src/filters/take/take_until.rs b/crates/nu-command/src/filters/take/take_until.rs index e3a2a37162..0df2407cb1 100644 --- a/crates/nu-command/src/filters/take/take_until.rs +++ b/crates/nu-command/src/filters/take/take_until.rs @@ -81,7 +81,8 @@ impl Command for TakeUntil { .take_while(move |value| { closure .run_with_value(value.clone()) - .map(|data| data.into_value(head).is_false()) + .and_then(|data| data.into_value(head)) + .map(|cond| cond.is_false()) .unwrap_or(false) }) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) diff --git a/crates/nu-command/src/filters/take/take_while.rs b/crates/nu-command/src/filters/take/take_while.rs index 632c165847..7c282ac38a 100644 --- a/crates/nu-command/src/filters/take/take_while.rs +++ b/crates/nu-command/src/filters/take/take_while.rs @@ -81,7 +81,8 @@ impl Command for TakeWhile { .take_while(move |value| { closure .run_with_value(value.clone()) - .map(|data| data.into_value(head).is_true()) + .and_then(|data| data.into_value(head)) + .map(|cond| cond.is_true()) .unwrap_or(false) }) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) diff --git a/crates/nu-command/src/filters/tee.rs b/crates/nu-command/src/filters/tee.rs index 319f70905c..936dee5c79 100644 --- a/crates/nu-command/src/filters/tee.rs +++ b/crates/nu-command/src/filters/tee.rs @@ -1,6 +1,17 @@ use nu_engine::{command_prelude::*, get_eval_block_with_early_return}; -use nu_protocol::{engine::Closure, OutDest, RawStream}; -use std::{sync::mpsc, thread}; +use nu_protocol::{ + byte_stream::copy_with_interrupt, engine::Closure, process::ChildPipe, ByteStream, + ByteStreamSource, OutDest, +}; +use std::{ + io::{self, Read, Write}, + sync::{ + atomic::AtomicBool, + mpsc::{self, Sender}, + Arc, + }, + thread::{self, JoinHandle}, +}; #[derive(Clone)] pub struct Tee; @@ -67,138 +78,205 @@ use it in your pipeline."# let head = call.head; let use_stderr = call.has_flag(engine_state, stack, "stderr")?; - let Spanned { - item: Closure { block_id, captures }, - span: closure_span, - } = call.req(engine_state, stack, 0)?; + let closure: Spanned = call.req(engine_state, stack, 0)?; + let closure_span = closure.span; + let closure = closure.item; - let closure_engine_state = engine_state.clone(); - let mut closure_stack = stack - .captures_to_stack_preserve_out_dest(captures) - .reset_pipes(); + let mut eval_block = { + let closure_engine_state = engine_state.clone(); + let mut closure_stack = stack + .captures_to_stack_preserve_out_dest(closure.captures) + .reset_pipes(); + let eval_block_with_early_return = get_eval_block_with_early_return(engine_state); - let metadata = input.metadata(); - let metadata_clone = metadata.clone(); + move |input| { + let result = eval_block_with_early_return( + &closure_engine_state, + &mut closure_stack, + closure_engine_state.get_block(closure.block_id), + input, + ); + // Make sure to drain any iterator produced to avoid unexpected behavior + result.and_then(|data| data.drain().map(|_| ())) + } + }; - let eval_block_with_early_return = get_eval_block_with_early_return(engine_state); + if let PipelineData::ByteStream(stream, metadata) = input { + let span = stream.span(); + let ctrlc = engine_state.ctrlc.clone(); + let eval_block = { + let metadata = metadata.clone(); + move |stream| eval_block(PipelineData::ByteStream(stream, metadata)) + }; - match input { - // Handle external streams specially, to make sure they pass through - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - } => { - let known_size = if use_stderr { - stderr.as_ref().and_then(|s| s.known_size) - } else { - stdout.as_ref().and_then(|s| s.known_size) - }; + match stream.into_source() { + ByteStreamSource::Read(read) => { + if use_stderr { + return stderr_misuse(span, head); + } - let with_stream = move |rx: mpsc::Receiver, ShellError>>| { - let iter = rx.into_iter(); - let input_from_channel = PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(iter), - closure_engine_state.ctrlc.clone(), - span, - known_size, - )), - stderr: None, - exit_code: None, - span, - metadata: metadata_clone, - trim_end_newline, + let tee = IoTee::new(read, span, eval_block)?; + + Ok(PipelineData::ByteStream( + ByteStream::read(tee, span, ctrlc), + metadata, + )) + } + ByteStreamSource::File(file) => { + if use_stderr { + return stderr_misuse(span, head); + } + + let tee = IoTee::new(file, span, eval_block)?; + + Ok(PipelineData::ByteStream( + ByteStream::read(tee, span, ctrlc), + metadata, + )) + } + ByteStreamSource::Child(mut child) => { + let stderr_thread = if use_stderr { + let stderr_thread = if let Some(stderr) = child.stderr.take() { + match stack.stderr() { + OutDest::Pipe | OutDest::Capture => { + let tee = IoTee::new(stderr, span, eval_block)?; + child.stderr = Some(ChildPipe::Tee(Box::new(tee))); + None + } + OutDest::Null => Some(tee_pipe_on_thread( + stderr, + io::sink(), + span, + ctrlc.as_ref(), + eval_block, + )?), + OutDest::Inherit => Some(tee_pipe_on_thread( + stderr, + io::stderr(), + span, + ctrlc.as_ref(), + eval_block, + )?), + OutDest::File(file) => Some(tee_pipe_on_thread( + stderr, + file.clone(), + span, + ctrlc.as_ref(), + eval_block, + )?), + } + } else { + None + }; + + if let Some(stdout) = child.stdout.take() { + match stack.stdout() { + OutDest::Pipe | OutDest::Capture => { + child.stdout = Some(stdout); + Ok(()) + } + OutDest::Null => { + copy_pipe(stdout, io::sink(), span, ctrlc.as_deref()) + } + OutDest::Inherit => { + copy_pipe(stdout, io::stdout(), span, ctrlc.as_deref()) + } + OutDest::File(file) => { + copy_pipe(stdout, file.as_ref(), span, ctrlc.as_deref()) + } + }?; + } + + stderr_thread + } else { + let stderr_thread = if let Some(stderr) = child.stderr.take() { + match stack.stderr() { + OutDest::Pipe | OutDest::Capture => { + child.stderr = Some(stderr); + Ok(None) + } + OutDest::Null => { + copy_pipe_on_thread(stderr, io::sink(), span, ctrlc.as_ref()) + .map(Some) + } + OutDest::Inherit => { + copy_pipe_on_thread(stderr, io::stderr(), span, ctrlc.as_ref()) + .map(Some) + } + OutDest::File(file) => { + copy_pipe_on_thread(stderr, file.clone(), span, ctrlc.as_ref()) + .map(Some) + } + }? + } else { + None + }; + + if let Some(stdout) = child.stdout.take() { + match stack.stdout() { + OutDest::Pipe | OutDest::Capture => { + let tee = IoTee::new(stdout, span, eval_block)?; + child.stdout = Some(ChildPipe::Tee(Box::new(tee))); + Ok(()) + } + OutDest::Null => { + tee_pipe(stdout, io::sink(), span, ctrlc.as_deref(), eval_block) + } + OutDest::Inherit => tee_pipe( + stdout, + io::stdout(), + span, + ctrlc.as_deref(), + eval_block, + ), + OutDest::File(file) => tee_pipe( + stdout, + file.as_ref(), + span, + ctrlc.as_deref(), + eval_block, + ), + }?; + } + + stderr_thread }; - let result = eval_block_with_early_return( - &closure_engine_state, - &mut closure_stack, - closure_engine_state.get_block(block_id), - input_from_channel, - ); - // Make sure to drain any iterator produced to avoid unexpected behavior - result.and_then(|data| data.drain()) - }; - if use_stderr { - let stderr = stderr - .map(|stderr| { - let iter = tee(stderr.stream, with_stream).err_span(head)?; - Ok::<_, ShellError>(RawStream::new( - Box::new(iter.map(flatten_result)), - stderr.ctrlc, - stderr.span, - stderr.known_size, - )) - }) - .transpose()?; - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - }) - } else { - let stdout = stdout - .map(|stdout| { - let iter = tee(stdout.stream, with_stream).err_span(head)?; - Ok::<_, ShellError>(RawStream::new( - Box::new(iter.map(flatten_result)), - stdout.ctrlc, - stdout.span, - stdout.known_size, - )) - }) - .transpose()?; - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - }) + if child.stdout.is_some() || child.stderr.is_some() { + Ok(PipelineData::ByteStream( + ByteStream::child(*child, span), + metadata, + )) + } else { + if let Some(thread) = stderr_thread { + thread.join().unwrap_or_else(|_| Err(panic_error()))?; + } + child.wait()?; + Ok(PipelineData::Empty) + } } } - // --stderr is not allowed if the input is not an external stream - _ if use_stderr => Err(ShellError::UnsupportedInput { - msg: "--stderr can only be used on external streams".into(), - input: "the input to `tee` is not an external stream".into(), - msg_span: head, - input_span: input.span().unwrap_or(head), - }), - // Handle others with the plain iterator - _ => { - let teed = tee(input.into_iter(), move |rx| { - let input_from_channel = rx.into_pipeline_data_with_metadata( - head, - closure_engine_state.ctrlc.clone(), - metadata_clone, - ); - let result = eval_block_with_early_return( - &closure_engine_state, - &mut closure_stack, - closure_engine_state.get_block(block_id), - input_from_channel, - ); - // Make sure to drain any iterator produced to avoid unexpected behavior - result.and_then(|data| data.drain()) - }) - .err_span(head)? - .map(move |result| result.unwrap_or_else(|err| Value::error(err, closure_span))) - .into_pipeline_data_with_metadata( - head, - engine_state.ctrlc.clone(), - metadata, - ); - - Ok(teed) + } else { + if use_stderr { + return stderr_misuse(input.span().unwrap_or(head), head); } + + let span = input.span().unwrap_or(head); + let ctrlc = engine_state.ctrlc.clone(); + let metadata = input.metadata(); + let metadata_clone = metadata.clone(); + + Ok(tee(input.into_iter(), move |rx| { + let input = rx.into_pipeline_data_with_metadata(span, ctrlc, metadata_clone); + eval_block(input) + }) + .err_span(call.head)? + .map(move |result| result.unwrap_or_else(|err| Value::error(err, closure_span))) + .into_pipeline_data_with_metadata( + span, + engine_state.ctrlc.clone(), + metadata, + )) } } @@ -213,10 +291,6 @@ fn panic_error() -> ShellError { } } -fn flatten_result(result: Result, E>) -> Result { - result.unwrap_or_else(Err) -} - /// Copies the iterator to a channel on another thread. If an error is produced on that thread, /// it is embedded in the resulting iterator as an `Err` as soon as possible. When the iterator /// finishes, it waits for the other thread to finish, also handling any error produced at that @@ -233,7 +307,7 @@ where let mut thread = Some( thread::Builder::new() - .name("stderr consumer".into()) + .name("tee".into()) .spawn(move || with_cloned_stream(rx))?, ); @@ -273,6 +347,134 @@ where })) } +fn stderr_misuse(span: Span, head: Span) -> Result { + Err(ShellError::UnsupportedInput { + msg: "--stderr can only be used on external commands".into(), + input: "the input to `tee` is not an external commands".into(), + msg_span: head, + input_span: span, + }) +} + +struct IoTee { + reader: R, + sender: Option>>, + thread: Option>>, +} + +impl IoTee { + fn new( + reader: R, + span: Span, + eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, + ) -> Result { + let (sender, receiver) = mpsc::channel(); + + let thread = thread::Builder::new() + .name("tee".into()) + .spawn(move || eval_block(ByteStream::from_iter(receiver, span, None))) + .err_span(span)?; + + Ok(Self { + reader, + sender: Some(sender), + thread: Some(thread), + }) + } +} + +impl Read for IoTee { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if let Some(thread) = self.thread.take() { + if thread.is_finished() { + if let Err(err) = thread.join().unwrap_or_else(|_| Err(panic_error())) { + return Err(io::Error::new(io::ErrorKind::Other, err)); + } + } else { + self.thread = Some(thread) + } + } + let len = self.reader.read(buf)?; + if len == 0 { + self.sender = None; + if let Some(thread) = self.thread.take() { + if let Err(err) = thread.join().unwrap_or_else(|_| Err(panic_error())) { + return Err(io::Error::new(io::ErrorKind::Other, err)); + } + } + } else if let Some(sender) = self.sender.as_mut() { + if sender.send(buf[..len].to_vec()).is_err() { + self.sender = None; + } + } + Ok(len) + } +} + +fn tee_pipe( + pipe: ChildPipe, + mut dest: impl Write, + span: Span, + ctrlc: Option<&AtomicBool>, + eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, +) -> Result<(), ShellError> { + match pipe { + ChildPipe::Pipe(pipe) => { + let mut tee = IoTee::new(pipe, span, eval_block)?; + copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; + } + ChildPipe::Tee(tee) => { + let mut tee = IoTee::new(tee, span, eval_block)?; + copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; + } + } + Ok(()) +} + +fn tee_pipe_on_thread( + pipe: ChildPipe, + dest: impl Write + Send + 'static, + span: Span, + ctrlc: Option<&Arc>, + eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, +) -> Result>, ShellError> { + let ctrlc = ctrlc.cloned(); + thread::Builder::new() + .name("stderr tee".into()) + .spawn(move || tee_pipe(pipe, dest, span, ctrlc.as_deref(), eval_block)) + .map_err(|e| e.into_spanned(span).into()) +} + +fn copy_pipe( + pipe: ChildPipe, + mut dest: impl Write, + span: Span, + ctrlc: Option<&AtomicBool>, +) -> Result<(), ShellError> { + match pipe { + ChildPipe::Pipe(mut pipe) => { + copy_with_interrupt(&mut pipe, &mut dest, span, ctrlc)?; + } + ChildPipe::Tee(mut tee) => { + copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; + } + } + Ok(()) +} + +fn copy_pipe_on_thread( + pipe: ChildPipe, + dest: impl Write + Send + 'static, + span: Span, + ctrlc: Option<&Arc>, +) -> Result>, ShellError> { + let ctrlc = ctrlc.cloned(); + thread::Builder::new() + .name("stderr copier".into()) + .spawn(move || copy_pipe(pipe, dest, span, ctrlc.as_deref())) + .map_err(|e| e.into_spanned(span).into()) +} + #[test] fn tee_copies_values_to_other_thread_and_passes_them_through() { let (tx, rx) = mpsc::channel(); diff --git a/crates/nu-command/src/filters/update.rs b/crates/nu-command/src/filters/update.rs index d963e39995..0d914d2d8e 100644 --- a/crates/nu-command/src/filters/update.rs +++ b/crates/nu-command/src/filters/update.rs @@ -225,8 +225,8 @@ fn update( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ExternalStream { .. } => Err(ShellError::IncompatiblePathAccess { - type_name: "external stream".to_string(), + PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { + type_name: "byte stream".to_string(), span: head, }), } @@ -250,7 +250,7 @@ fn update_value_by_closure( let new_value = closure .add_arg(arg.clone()) .run_with_input(value_at_path.into_pipeline_data())? - .into_value(span); + .into_value(span)?; value.update_data_at_cell_path(cell_path, new_value) } @@ -273,7 +273,7 @@ fn update_single_value_by_closure( let new_value = closure .add_arg(arg.clone()) .run_with_input(value_at_path.into_pipeline_data())? - .into_value(span); + .into_value(span)?; value.update_data_at_cell_path(cell_path, new_value) } diff --git a/crates/nu-command/src/filters/upsert.rs b/crates/nu-command/src/filters/upsert.rs index 6b62b1d7bc..4313addd89 100644 --- a/crates/nu-command/src/filters/upsert.rs +++ b/crates/nu-command/src/filters/upsert.rs @@ -218,7 +218,7 @@ fn upsert( if let Value::Closure { val, .. } = replacement { ClosureEvalOnce::new(engine_state, stack, *val) .run_with_value(value)? - .into_value(head) + .into_value(head)? } else { replacement } @@ -285,8 +285,8 @@ fn upsert( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ExternalStream { .. } => Err(ShellError::IncompatiblePathAccess { - type_name: "external stream".to_string(), + PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { + type_name: "byte stream".to_string(), span: head, }), } @@ -311,7 +311,11 @@ fn upsert_value_by_closure( .map(IntoPipelineData::into_pipeline_data) .unwrap_or(PipelineData::Empty); - let new_value = closure.add_arg(arg).run_with_input(input)?.into_value(span); + let new_value = closure + .add_arg(arg) + .run_with_input(input)? + .into_value(span)?; + value.upsert_data_at_cell_path(cell_path, new_value) } @@ -334,7 +338,11 @@ fn upsert_single_value_by_closure( .map(IntoPipelineData::into_pipeline_data) .unwrap_or(PipelineData::Empty); - let new_value = closure.add_arg(arg).run_with_input(input)?.into_value(span); + let new_value = closure + .add_arg(arg) + .run_with_input(input)? + .into_value(span)?; + value.upsert_data_at_cell_path(cell_path, new_value) } diff --git a/crates/nu-command/src/filters/utils.rs b/crates/nu-command/src/filters/utils.rs index 0ef7d916b7..8d9b1300f6 100644 --- a/crates/nu-command/src/filters/utils.rs +++ b/crates/nu-command/src/filters/utils.rs @@ -36,7 +36,7 @@ pub fn boolean_fold( break; } - let pred = closure.run_with_value(value)?.into_value(head).is_true(); + let pred = closure.run_with_value(value)?.into_value(head)?.is_true(); if pred == accumulator { return Ok(Value::bool(accumulator, head).into_pipeline_data()); diff --git a/crates/nu-command/src/filters/values.rs b/crates/nu-command/src/filters/values.rs index aa576de874..ed33ebf643 100644 --- a/crates/nu-command/src/filters/values.rs +++ b/crates/nu-command/src/filters/values.rs @@ -180,13 +180,11 @@ fn values( Err(err) => Err(err), } } - PipelineData::ExternalStream { .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "record or table".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: head, - src_span: input - .span() - .expect("PipelineData::ExternalStream had no span"), + src_span: stream.span(), }), } } diff --git a/crates/nu-command/src/filters/where_.rs b/crates/nu-command/src/filters/where_.rs index 7507a7ede1..fe73de354f 100644 --- a/crates/nu-command/src/filters/where_.rs +++ b/crates/nu-command/src/filters/where_.rs @@ -57,9 +57,14 @@ not supported."# let metadata = input.metadata(); Ok(input .into_iter_strict(head)? - .filter_map(move |value| match closure.run_with_value(value.clone()) { - Ok(data) => data.into_value(head).is_true().then_some(value), - Err(err) => Some(Value::error(err, head)), + .filter_map(move |value| { + match closure + .run_with_value(value.clone()) + .and_then(|data| data.into_value(head)) + { + Ok(cond) => cond.is_true().then_some(value), + Err(err) => Some(Value::error(err, head)), + } }) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) } diff --git a/crates/nu-command/src/filters/wrap.rs b/crates/nu-command/src/filters/wrap.rs index 24ce8e6821..52a0fb22c3 100644 --- a/crates/nu-command/src/filters/wrap.rs +++ b/crates/nu-command/src/filters/wrap.rs @@ -43,8 +43,8 @@ impl Command for Wrap { .into_iter() .map(move |x| Value::record(record! { name.clone() => x }, span)) .into_pipeline_data_with_metadata(span, engine_state.ctrlc.clone(), metadata)), - PipelineData::ExternalStream { .. } => Ok(Value::record( - record! { name => input.into_value(span) }, + PipelineData::ByteStream(stream, ..) => Ok(Value::record( + record! { name => stream.into_value()? }, span, ) .into_pipeline_data_with_metadata(metadata)), diff --git a/crates/nu-command/src/formats/from/json.rs b/crates/nu-command/src/formats/from/json.rs index 44f127152f..ea449711c1 100644 --- a/crates/nu-command/src/formats/from/json.rs +++ b/crates/nu-command/src/formats/from/json.rs @@ -59,7 +59,7 @@ impl Command for FromJson { let (string_input, span, metadata) = input.collect_string_strict(span)?; if string_input.is_empty() { - return Ok(PipelineData::new_with_metadata(metadata, span)); + return Ok(Value::nothing(span).into_pipeline_data()); } let strict = call.has_flag(engine_state, stack, "strict")?; diff --git a/crates/nu-command/src/formats/from/msgpack.rs b/crates/nu-command/src/formats/from/msgpack.rs index 75f2be2056..4d8ea5e320 100644 --- a/crates/nu-command/src/formats/from/msgpack.rs +++ b/crates/nu-command/src/formats/from/msgpack.rs @@ -2,9 +2,8 @@ // implementation here is unique. use std::{ - collections::VecDeque, error::Error, - io::{self, Cursor, ErrorKind, Write}, + io::{self, Cursor, ErrorKind}, string::FromUtf8Error, sync::{atomic::AtomicBool, Arc}, }; @@ -12,7 +11,6 @@ use std::{ use byteorder::{BigEndian, ReadBytesExt}; use chrono::{TimeZone, Utc}; use nu_engine::command_prelude::*; -use nu_protocol::RawStream; use rmp::decode::{self as mp, ValueReadError}; /// Max recursion depth @@ -121,12 +119,20 @@ MessagePack: https://msgpack.org/ read_msgpack(Cursor::new(bytes), opts) } // Deserialize from a raw stream directly without having to collect it - PipelineData::ExternalStream { - stdout: Some(raw_stream), - .. - } => read_msgpack(ReadRawStream::new(raw_stream), opts), + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + if let Some(reader) = stream.reader() { + read_msgpack(reader, opts) + } else { + Err(ShellError::PipelineMismatch { + exp_input_type: "binary or byte stream".into(), + dst_span: call.head, + src_span: span, + }) + } + } input => Err(ShellError::PipelineMismatch { - exp_input_type: "binary".into(), + exp_input_type: "binary or byte stream".into(), dst_span: call.head, src_span: input.span().unwrap_or(call.head), }), @@ -483,57 +489,6 @@ where .map_err(|err| ReadError::Io(err, span)) } -/// Adapter to read MessagePack from a `RawStream` -/// -/// TODO: contribute this back to `RawStream` in general, with more polish, if it works -pub(crate) struct ReadRawStream { - pub stream: RawStream, - // Use a `VecDeque` for read efficiency - pub leftover: VecDeque, -} - -impl ReadRawStream { - pub(crate) fn new(mut stream: RawStream) -> ReadRawStream { - ReadRawStream { - leftover: std::mem::take(&mut stream.leftover).into(), - stream, - } - } -} - -impl io::Read for ReadRawStream { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if buf.is_empty() { - Ok(0) - } else if !self.leftover.is_empty() { - // Take as many leftover bytes as possible - self.leftover.read(buf) - } else { - // Try to get data from the RawStream. We have to be careful not to break on a zero-len - // buffer though, since that would mean EOF - loop { - if let Some(result) = self.stream.stream.next() { - let bytes = result.map_err(|err| io::Error::new(ErrorKind::Other, err))?; - if !bytes.is_empty() { - let min_len = bytes.len().min(buf.len()); - let (source, leftover_bytes) = bytes.split_at(min_len); - buf[0..min_len].copy_from_slice(source); - // Keep whatever bytes we couldn't use in the leftover vec - self.leftover.write_all(leftover_bytes)?; - return Ok(min_len); - } else { - // Zero-length buf, continue - continue; - } - } else { - // End of input - return Ok(0); - } - } - } - } -} - /// Return an error if this is not the end of file. /// /// This can help detect if parsing succeeded incorrectly, perhaps due to corruption. diff --git a/crates/nu-command/src/formats/from/msgpackz.rs b/crates/nu-command/src/formats/from/msgpackz.rs index 3200d5d876..7960f3f97a 100644 --- a/crates/nu-command/src/formats/from/msgpackz.rs +++ b/crates/nu-command/src/formats/from/msgpackz.rs @@ -2,7 +2,7 @@ use std::io::Cursor; use nu_engine::command_prelude::*; -use super::msgpack::{read_msgpack, Opts, ReadRawStream}; +use super::msgpack::{read_msgpack, Opts}; const BUFFER_SIZE: usize = 65536; @@ -50,15 +50,21 @@ impl Command for FromMsgpackz { read_msgpack(reader, opts) } // Deserialize from a raw stream directly without having to collect it - PipelineData::ExternalStream { - stdout: Some(raw_stream), - .. - } => { - let reader = brotli::Decompressor::new(ReadRawStream::new(raw_stream), BUFFER_SIZE); - read_msgpack(reader, opts) + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + if let Some(reader) = stream.reader() { + let reader = brotli::Decompressor::new(reader, BUFFER_SIZE); + read_msgpack(reader, opts) + } else { + Err(ShellError::PipelineMismatch { + exp_input_type: "binary or byte stream".into(), + dst_span: call.head, + src_span: span, + }) + } } _ => Err(ShellError::PipelineMismatch { - exp_input_type: "binary".into(), + exp_input_type: "binary or byte stream".into(), dst_span: call.head, src_span: span, }), diff --git a/crates/nu-command/src/formats/from/ods.rs b/crates/nu-command/src/formats/from/ods.rs index fff9e98be6..ff7a76c0ca 100644 --- a/crates/nu-command/src/formats/from/ods.rs +++ b/crates/nu-command/src/formats/from/ods.rs @@ -81,28 +81,32 @@ fn convert_columns(columns: &[Value]) -> Result, ShellError> { } fn collect_binary(input: PipelineData, span: Span) -> Result, ShellError> { - let mut bytes = vec![]; - let mut values = input.into_iter(); + if let PipelineData::ByteStream(stream, ..) = input { + stream.into_bytes() + } else { + let mut bytes = vec![]; + let mut values = input.into_iter(); - loop { - match values.next() { - Some(Value::Binary { val: b, .. }) => { - bytes.extend_from_slice(&b); + loop { + match values.next() { + Some(Value::Binary { val: b, .. }) => { + bytes.extend_from_slice(&b); + } + Some(Value::Error { error, .. }) => return Err(*error), + Some(x) => { + return Err(ShellError::UnsupportedInput { + msg: "Expected binary from pipeline".to_string(), + input: "value originates from here".into(), + msg_span: span, + input_span: x.span(), + }) + } + None => break, } - Some(Value::Error { error, .. }) => return Err(*error), - Some(x) => { - return Err(ShellError::UnsupportedInput { - msg: "Expected binary from pipeline".to_string(), - input: "value originates from here".into(), - msg_span: span, - input_span: x.span(), - }) - } - None => break, } - } - Ok(bytes) + Ok(bytes) + } } fn from_ods( diff --git a/crates/nu-command/src/formats/from/xlsx.rs b/crates/nu-command/src/formats/from/xlsx.rs index b54cffe3aa..21e2567b45 100644 --- a/crates/nu-command/src/formats/from/xlsx.rs +++ b/crates/nu-command/src/formats/from/xlsx.rs @@ -82,27 +82,31 @@ fn convert_columns(columns: &[Value]) -> Result, ShellError> { } fn collect_binary(input: PipelineData, span: Span) -> Result, ShellError> { - let mut bytes = vec![]; - let mut values = input.into_iter(); + if let PipelineData::ByteStream(stream, ..) = input { + stream.into_bytes() + } else { + let mut bytes = vec![]; + let mut values = input.into_iter(); - loop { - match values.next() { - Some(Value::Binary { val: b, .. }) => { - bytes.extend_from_slice(&b); + loop { + match values.next() { + Some(Value::Binary { val: b, .. }) => { + bytes.extend_from_slice(&b); + } + Some(x) => { + return Err(ShellError::UnsupportedInput { + msg: "Expected binary from pipeline".to_string(), + input: "value originates from here".into(), + msg_span: span, + input_span: x.span(), + }) + } + None => break, } - Some(x) => { - return Err(ShellError::UnsupportedInput { - msg: "Expected binary from pipeline".to_string(), - input: "value originates from here".into(), - msg_span: span, - input_span: x.span(), - }) - } - None => break, } - } - Ok(bytes) + Ok(bytes) + } } fn from_xlsx( diff --git a/crates/nu-command/src/formats/to/delimited.rs b/crates/nu-command/src/formats/to/delimited.rs index a10f611f60..490983d67b 100644 --- a/crates/nu-command/src/formats/to/delimited.rs +++ b/crates/nu-command/src/formats/to/delimited.rs @@ -150,7 +150,7 @@ pub fn to_delimited_data( span: Span, config: &Config, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; let output = match from_value_to_delimited_string(&value, sep, config, span) { Ok(mut x) => { if noheaders { diff --git a/crates/nu-command/src/formats/to/json.rs b/crates/nu-command/src/formats/to/json.rs index 0796abc8dc..c4c87f804f 100644 --- a/crates/nu-command/src/formats/to/json.rs +++ b/crates/nu-command/src/formats/to/json.rs @@ -46,7 +46,7 @@ impl Command for ToJson { let span = call.head; // allow ranges to expand and turn into array let input = input.try_expand_range()?; - let value = input.into_value(span); + let value = input.into_value(span)?; let json_value = value_to_json_value(&value)?; let json_result = if raw { diff --git a/crates/nu-command/src/formats/to/msgpack.rs b/crates/nu-command/src/formats/to/msgpack.rs index a4575f37e4..bfeb428e3e 100644 --- a/crates/nu-command/src/formats/to/msgpack.rs +++ b/crates/nu-command/src/formats/to/msgpack.rs @@ -75,7 +75,7 @@ MessagePack: https://msgpack.org/ input: PipelineData, ) -> Result { let value_span = input.span().unwrap_or(call.head); - let value = input.into_value(value_span); + let value = input.into_value(value_span)?; let mut out = vec![]; write_value(&mut out, &value, 0)?; diff --git a/crates/nu-command/src/formats/to/msgpackz.rs b/crates/nu-command/src/formats/to/msgpackz.rs index a07e1206c1..9168d05018 100644 --- a/crates/nu-command/src/formats/to/msgpackz.rs +++ b/crates/nu-command/src/formats/to/msgpackz.rs @@ -70,7 +70,7 @@ impl Command for ToMsgpackz { .transpose()?; let value_span = input.span().unwrap_or(call.head); - let value = input.into_value(value_span); + let value = input.into_value(value_span)?; let mut out_buf = vec![]; let mut out = brotli::CompressorWriter::new( &mut out_buf, diff --git a/crates/nu-command/src/formats/to/nuon.rs b/crates/nu-command/src/formats/to/nuon.rs index e747ac58f6..f40b7b5c1d 100644 --- a/crates/nu-command/src/formats/to/nuon.rs +++ b/crates/nu-command/src/formats/to/nuon.rs @@ -53,7 +53,7 @@ impl Command for ToNuon { }; let span = call.head; - let value = input.into_value(span); + let value = input.into_value(span)?; match nuon::to_nuon(&value, style, Some(span)) { Ok(serde_nuon_string) => { diff --git a/crates/nu-command/src/formats/to/text.rs b/crates/nu-command/src/formats/to/text.rs index 7c12dc2821..7f1d632c13 100644 --- a/crates/nu-command/src/formats/to/text.rs +++ b/crates/nu-command/src/formats/to/text.rs @@ -1,6 +1,12 @@ use chrono_humanize::HumanTime; use nu_engine::command_prelude::*; -use nu_protocol::{format_duration, format_filesize_from_conf, Config, RawStream, ValueIterator}; +use nu_protocol::{format_duration, format_filesize_from_conf, ByteStream, Config}; + +const LINE_ENDING: &str = if cfg!(target_os = "windows") { + "\r\n" +} else { + "\n" +}; #[derive(Clone)] pub struct ToText; @@ -28,39 +34,28 @@ impl Command for ToText { input: PipelineData, ) -> Result { let span = call.head; - let config = engine_state.get_config(); - - let line_ending = if cfg!(target_os = "windows") { - "\r\n" - } else { - "\n" - }; let input = input.try_expand_range()?; - if let PipelineData::ListStream(stream, _) = input { - Ok(PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(ListStreamIterator { - stream: stream.into_inner(), - separator: line_ending.into(), - config: config.clone(), - }), - engine_state.ctrlc.clone(), - span, - None, - )), - stderr: None, - exit_code: None, - span, - metadata: None, - trim_end_newline: false, - }) - } else { - // FIXME: don't collect! stream the output wherever possible! - // Even if the data is collected when it arrives at `to text`, we should be able to stream it out - let collected_input = local_into_string(input.into_value(span), line_ending, config); - - Ok(Value::string(collected_input, span).into_pipeline_data()) + match input { + PipelineData::Empty => Ok(Value::string(String::new(), span).into_pipeline_data()), + PipelineData::Value(value, ..) => { + let str = local_into_string(value, LINE_ENDING, engine_state.get_config()); + Ok(Value::string(str, span).into_pipeline_data()) + } + PipelineData::ListStream(stream, meta) => { + let span = stream.span(); + let config = engine_state.get_config().clone(); + let iter = stream.into_inner().map(move |value| { + let mut str = local_into_string(value, LINE_ENDING, &config); + str.push_str(LINE_ENDING); + str + }); + Ok(PipelineData::ByteStream( + ByteStream::from_iter(iter, span, engine_state.ctrlc.clone()), + meta, + )) + } + PipelineData::ByteStream(stream, meta) => Ok(PipelineData::ByteStream(stream, meta)), } } @@ -85,26 +80,6 @@ impl Command for ToText { } } -struct ListStreamIterator { - stream: ValueIterator, - separator: String, - config: Config, -} - -impl Iterator for ListStreamIterator { - type Item = Result, ShellError>; - - fn next(&mut self) -> Option { - if let Some(item) = self.stream.next() { - let mut string = local_into_string(item, &self.separator, &self.config); - string.push_str(&self.separator); - Some(Ok(string.as_bytes().to_vec())) - } else { - None - } - } -} - fn local_into_string(value: Value, separator: &str, config: &Config) -> String { let span = value.span(); match value { diff --git a/crates/nu-command/src/formats/to/toml.rs b/crates/nu-command/src/formats/to/toml.rs index 385e9576f3..7423f147dd 100644 --- a/crates/nu-command/src/formats/to/toml.rs +++ b/crates/nu-command/src/formats/to/toml.rs @@ -141,7 +141,7 @@ fn to_toml( input: PipelineData, span: Span, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; let toml_value = value_to_toml_value(engine_state, &value, span)?; match toml_value { diff --git a/crates/nu-command/src/formats/to/xml.rs b/crates/nu-command/src/formats/to/xml.rs index 2cfec24470..648094318b 100644 --- a/crates/nu-command/src/formats/to/xml.rs +++ b/crates/nu-command/src/formats/to/xml.rs @@ -132,7 +132,7 @@ impl Job { } fn run(mut self, input: PipelineData, head: Span) -> Result { - let value = input.into_value(head); + let value = input.into_value(head)?; self.write_xml_entry(value, true).and_then(|_| { let b = self.writer.into_inner().into_inner(); diff --git a/crates/nu-command/src/formats/to/yaml.rs b/crates/nu-command/src/formats/to/yaml.rs index d03c886328..bea2dd3381 100644 --- a/crates/nu-command/src/formats/to/yaml.rs +++ b/crates/nu-command/src/formats/to/yaml.rs @@ -95,7 +95,7 @@ pub fn value_to_yaml_value(v: &Value) -> Result { } fn to_yaml(input: PipelineData, head: Span) -> Result { - let value = input.into_value(head); + let value = input.into_value(head)?; let yaml_value = value_to_yaml_value(&value)?; match serde_yaml::to_string(&yaml_value) { diff --git a/crates/nu-command/src/generators/generate.rs b/crates/nu-command/src/generators/generate.rs index 0a01b79c08..3549667ff0 100644 --- a/crates/nu-command/src/generators/generate.rs +++ b/crates/nu-command/src/generators/generate.rs @@ -158,14 +158,16 @@ used as the next argument to the closure, otherwise generation stops. } Ok(other) => { - let val = other.into_value(head); - let error = ShellError::GenericError { - error: "Invalid block return".into(), - msg: format!("Expected record, found {}", val.get_type()), - span: Some(val.span()), - help: None, - inner: vec![], - }; + let error = other + .into_value(head) + .map(|val| ShellError::GenericError { + error: "Invalid block return".into(), + msg: format!("Expected record, found {}", val.get_type()), + span: Some(val.span()), + help: None, + inner: vec![], + }) + .unwrap_or_else(|err| err); (Some(Value::error(error, head)), None) } diff --git a/crates/nu-command/src/hash/generic_digest.rs b/crates/nu-command/src/hash/generic_digest.rs index 476915f07d..ab15ccae7a 100644 --- a/crates/nu-command/src/hash/generic_digest.rs +++ b/crates/nu-command/src/hash/generic_digest.rs @@ -1,7 +1,6 @@ use nu_cmd_base::input_handler::{operate, CmdArgument}; use nu_engine::command_prelude::*; - -use std::marker::PhantomData; +use std::{io::Write, marker::PhantomData}; pub trait HashDigest: digest::Digest + Clone { fn name() -> &'static str; @@ -38,7 +37,7 @@ impl CmdArgument for Arguments { impl Command for GenericDigest where - D: HashDigest + Send + Sync + 'static, + D: HashDigest + Write + Send + Sync + 'static, digest::Output: core::fmt::LowerHex, { fn name(&self) -> &str { @@ -81,54 +80,23 @@ where call: &Call, input: PipelineData, ) -> Result { + let head = call.head; let binary = call.has_flag(engine_state, stack, "binary")?; let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - let args = Arguments { binary, cell_paths }; let mut hasher = D::new(); - match input { - PipelineData::ExternalStream { - stdout: Some(stream), - span, - .. - } => { - for item in stream { - match item { - // String and binary data are valid byte patterns - Ok(Value::String { val, .. }) => hasher.update(val.as_bytes()), - Ok(Value::Binary { val, .. }) => hasher.update(val), - // If any Error value is output, echo it back - Ok(v @ Value::Error { .. }) => return Ok(v.into_pipeline_data()), - // Unsupported data - Ok(other) => { - return Ok(Value::error( - ShellError::OnlySupportsThisInputType { - exp_input_type: "string and binary".into(), - wrong_type: other.get_type().to_string(), - dst_span: span, - src_span: other.span(), - }, - span, - ) - .into_pipeline_data()); - } - Err(err) => return Err(err), - }; - } - let digest = hasher.finalize(); - if args.binary { - Ok(Value::binary(digest.to_vec(), span).into_pipeline_data()) - } else { - Ok(Value::string(format!("{digest:x}"), span).into_pipeline_data()) - } + + if let PipelineData::ByteStream(stream, ..) = input { + stream.write_to(&mut hasher)?; + let digest = hasher.finalize(); + if binary { + Ok(Value::binary(digest.to_vec(), head).into_pipeline_data()) + } else { + Ok(Value::string(format!("{digest:x}"), head).into_pipeline_data()) } - _ => operate( - action::, - args, - input, - call.head, - engine_state.ctrlc.clone(), - ), + } else { + let args = Arguments { binary, cell_paths }; + operate(action::, args, input, head, engine_state.ctrlc.clone()) } } } diff --git a/crates/nu-command/src/misc/tutor.rs b/crates/nu-command/src/misc/tutor.rs index 8eeec6f393..6b1c43534b 100644 --- a/crates/nu-command/src/misc/tutor.rs +++ b/crates/nu-command/src/misc/tutor.rs @@ -409,15 +409,15 @@ fn display(help: &str, engine_state: &EngineState, stack: &mut Stack, span: Span //TODO: support no-color mode if let Some(highlighter) = engine_state.find_decl(b"nu-highlight", &[]) { let decl = engine_state.get_decl(highlighter); - - if let Ok(output) = decl.run( + let result = decl.run( engine_state, stack, &Call::new(span), Value::string(item, Span::unknown()).into_pipeline_data(), - ) { - let result = output.into_value(Span::unknown()); - match result.coerce_into_string() { + ); + + if let Ok(value) = result.and_then(|data| data.into_value(Span::unknown())) { + match value.coerce_into_string() { Ok(s) => { build.push_str(&s); } diff --git a/crates/nu-command/src/network/http/client.rs b/crates/nu-command/src/network/http/client.rs index 2833f76a97..54f7749627 100644 --- a/crates/nu-command/src/network/http/client.rs +++ b/crates/nu-command/src/network/http/client.rs @@ -5,10 +5,9 @@ use base64::{ Engine, }; use nu_engine::command_prelude::*; -use nu_protocol::{BufferedReader, RawStream}; +use nu_protocol::ByteStream; use std::{ collections::HashMap, - io::BufReader, path::PathBuf, str::FromStr, sync::{ @@ -119,21 +118,11 @@ pub fn response_to_buffer( }; let reader = response.into_reader(); - let buffered_input = BufReader::new(reader); - PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(BufferedReader::new(buffered_input)), - engine_state.ctrlc.clone(), - span, - buffer_size, - )), - stderr: None, - exit_code: None, - span, - metadata: None, - trim_end_newline: false, - } + PipelineData::ByteStream( + ByteStream::read(reader, span, engine_state.ctrlc.clone()).with_known_size(buffer_size), + None, + ) } pub fn request_add_authorization_header( @@ -529,25 +518,25 @@ fn request_handle_response_content( if flags.full { let response_status = resp.status(); - let request_headers_value = match headers_to_nu(&extract_request_headers(&request), span) { - Ok(headers) => headers.into_value(span), - Err(_) => Value::nothing(span), - }; + let request_headers_value = headers_to_nu(&extract_request_headers(&request), span) + .and_then(|data| data.into_value(span)) + .unwrap_or(Value::nothing(span)); - let response_headers_value = match headers_to_nu(&extract_response_headers(&resp), span) { - Ok(headers) => headers.into_value(span), - Err(_) => Value::nothing(span), - }; + let response_headers_value = headers_to_nu(&extract_response_headers(&resp), span) + .and_then(|data| data.into_value(span)) + .unwrap_or(Value::nothing(span)); let headers = record! { "request" => request_headers_value, "response" => response_headers_value, }; + let body = consume_response_body(resp)?.into_value(span)?; + let full_response = Value::record( record! { "headers" => Value::record(headers, span), - "body" => consume_response_body(resp)?.into_value(span), + "body" => body, "status" => Value::int(response_status as i64, span), }, span, diff --git a/crates/nu-command/src/network/url/parse.rs b/crates/nu-command/src/network/url/parse.rs index 8a80553eca..e71c8d472a 100644 --- a/crates/nu-command/src/network/url/parse.rs +++ b/crates/nu-command/src/network/url/parse.rs @@ -42,7 +42,7 @@ impl Command for SubCommand { call: &Call, input: PipelineData, ) -> Result { - parse(input.into_value(call.head), call.head, engine_state) + parse(input.into_value(call.head)?, call.head, engine_state) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/path/join.rs b/crates/nu-command/src/path/join.rs index eb820606af..19d65b4c46 100644 --- a/crates/nu-command/src/path/join.rs +++ b/crates/nu-command/src/path/join.rs @@ -171,8 +171,8 @@ fn run(call: &Call, args: &Arguments, input: PipelineData) -> Result Ok(PipelineData::Value(handle_value(val, args, head), md)), - PipelineData::ListStream(..) => Ok(PipelineData::Value( - handle_value(input.into_value(head), args, head), + PipelineData::ListStream(stream, ..) => Ok(PipelineData::Value( + handle_value(stream.into_value(), args, head), metadata, )), PipelineData::Empty { .. } => Err(ShellError::PipelineEmpty { dst_span: head }), diff --git a/crates/nu-command/src/progress_bar.rs b/crates/nu-command/src/progress_bar.rs index 17ddeeb64e..db4d4e23b1 100644 --- a/crates/nu-command/src/progress_bar.rs +++ b/crates/nu-command/src/progress_bar.rs @@ -6,8 +6,6 @@ use std::fmt; pub struct NuProgressBar { pub pb: ProgressBar, - bytes_processed: u64, - total_bytes: Option, } impl NuProgressBar { @@ -40,8 +38,6 @@ impl NuProgressBar { NuProgressBar { pb: new_progress_bar, - total_bytes: None, - bytes_processed: 0, } } @@ -57,12 +53,4 @@ impl NuProgressBar { pub fn abandoned_msg(&self, msg: String) { self.pb.abandon_with_message(msg); } - - pub fn clone(&self) -> NuProgressBar { - NuProgressBar { - pb: self.pb.clone(), - bytes_processed: self.bytes_processed, - total_bytes: self.total_bytes, - } - } } diff --git a/crates/nu-command/src/strings/encode_decode/decode.rs b/crates/nu-command/src/strings/encode_decode/decode.rs index 25b8f59ec2..9b13fad202 100644 --- a/crates/nu-command/src/strings/encode_decode/decode.rs +++ b/crates/nu-command/src/strings/encode_decode/decode.rs @@ -57,16 +57,12 @@ documentation link at https://docs.rs/encoding_rs/latest/encoding_rs/#statics"# let encoding: Option> = call.opt(engine_state, stack, 0)?; match input { - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - span: input_span, - .. - } => { - let bytes: Vec = stream.into_bytes()?.item; + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + let bytes = stream.into_bytes()?; match encoding { Some(encoding_name) => super::encoding::decode(head, encoding_name, &bytes), - None => super::encoding::detect_encoding_name(head, input_span, &bytes) + None => super::encoding::detect_encoding_name(head, span, &bytes) .map(|encoding| encoding.decode(&bytes).0.into_owned()) .map(|s| Value::string(s, head)), } diff --git a/crates/nu-command/src/strings/encode_decode/encode.rs b/crates/nu-command/src/strings/encode_decode/encode.rs index 98fcc34179..113c0fe548 100644 --- a/crates/nu-command/src/strings/encode_decode/encode.rs +++ b/crates/nu-command/src/strings/encode_decode/encode.rs @@ -81,13 +81,10 @@ documentation link at https://docs.rs/encoding_rs/latest/encoding_rs/#statics"# let ignore_errors = call.has_flag(engine_state, stack, "ignore-errors")?; match input { - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); let s = stream.into_string()?; - super::encoding::encode(head, encoding, &s.item, s.span, ignore_errors) + super::encoding::encode(head, encoding, &s, span, ignore_errors) .map(|val| val.into_pipeline_data()) } PipelineData::Value(v, ..) => { diff --git a/crates/nu-command/src/strings/parse.rs b/crates/nu-command/src/strings/parse.rs index 51067a16a2..bc70d4679c 100644 --- a/crates/nu-command/src/strings/parse.rs +++ b/crates/nu-command/src/strings/parse.rs @@ -208,30 +208,21 @@ fn operate( } }) .into()), - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::Empty), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - // Collect all `stream` chunks into a single `chunk` to be able to deal with matches that - // extend across chunk boundaries. - // This is a stop-gap solution until the `regex` crate supports streaming or an alternative - // solution is found. - // See https://github.com/nushell/nushell/issues/9795 - let str = stream.into_string()?.item; + PipelineData::ByteStream(stream, ..) => { + if let Some(lines) = stream.lines() { + let iter = ParseIter { + captures: VecDeque::new(), + regex, + columns, + iter: lines, + span: head, + ctrlc, + }; - // let iter = stream.lines(); - - let iter = ParseIter { - captures: VecDeque::new(), - regex, - columns, - iter: std::iter::once(Ok(str)), - span: head, - ctrlc, - }; - - Ok(ListStream::new(iter, head, None).into()) + Ok(ListStream::new(iter, head, None).into()) + } else { + Ok(PipelineData::Empty) + } } } } diff --git a/crates/nu-command/src/system/complete.rs b/crates/nu-command/src/system/complete.rs index c622c86f3c..409cef1f27 100644 --- a/crates/nu-command/src/system/complete.rs +++ b/crates/nu-command/src/system/complete.rs @@ -1,6 +1,5 @@ use nu_engine::command_prelude::*; use nu_protocol::OutDest; -use std::thread; #[derive(Clone)] pub struct Complete; @@ -31,78 +30,53 @@ impl Command for Complete { call: &Call, input: PipelineData, ) -> Result { + let head = call.head; match input { - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - .. - } => { - let mut record = Record::new(); - - // use a thread to receive stderr message. - // Or we may get a deadlock if child process sends out too much bytes to stdout. - // - // For example: in normal linux system, stdout pipe's limit is 65535 bytes. - // if child process sends out 65536 bytes, the process will be hanged because no consumer - // consumes the first 65535 bytes - // So we need a thread to receive stderr message, then the current thread can continue to consume - // stdout messages. - let stderr_handler = stderr - .map(|stderr| { - let stderr_span = stderr.span; - thread::Builder::new() - .name("stderr consumer".to_string()) - .spawn(move || { - let stderr = stderr.into_bytes()?; - if let Ok(st) = String::from_utf8(stderr.item.clone()) { - Ok::<_, ShellError>(Value::string(st, stderr.span)) - } else { - Ok::<_, ShellError>(Value::binary(stderr.item, stderr.span)) - } - }) - .map(|handle| (handle, stderr_span)) - .err_span(call.head) - }) - .transpose()?; - - if let Some(stdout) = stdout { - let stdout = stdout.into_bytes()?; - record.push( - "stdout", - if let Ok(st) = String::from_utf8(stdout.item.clone()) { - Value::string(st, stdout.span) - } else { - Value::binary(stdout.item, stdout.span) - }, - ) - } - - if let Some((handler, stderr_span)) = stderr_handler { - let res = handler.join().map_err(|err| ShellError::ExternalCommand { - label: "Fail to receive external commands stderr message".to_string(), - help: format!("{err:?}"), - span: stderr_span, - })??; - record.push("stderr", res) + PipelineData::ByteStream(stream, ..) => { + let Ok(child) = stream.into_child() else { + return Err(ShellError::GenericError { + error: "Complete only works with external commands".into(), + msg: "complete only works on external commands".into(), + span: Some(call.head), + help: None, + inner: vec![], + }); }; - if let Some(exit_code) = exit_code { - let mut v: Vec<_> = exit_code.into_iter().collect(); + let output = child.wait_with_output()?; + let exit_code = output.exit_status.code(); + let mut record = Record::new(); - if let Some(v) = v.pop() { - record.push("exit_code", v); - } + if let Some(stdout) = output.stdout { + record.push( + "stdout", + match String::from_utf8(stdout) { + Ok(str) => Value::string(str, head), + Err(err) => Value::binary(err.into_bytes(), head), + }, + ); } + if let Some(stderr) = output.stderr { + record.push( + "stderr", + match String::from_utf8(stderr) { + Ok(str) => Value::string(str, head), + Err(err) => Value::binary(err.into_bytes(), head), + }, + ); + } + + record.push("exit_code", Value::int(exit_code.into(), head)); + Ok(Value::record(record, call.head).into_pipeline_data()) } // bubble up errors from the previous command PipelineData::Value(Value::Error { error, .. }, _) => Err(*error), _ => Err(ShellError::GenericError { - error: "Complete only works with external streams".into(), - msg: "complete only works on external streams".into(), - span: Some(call.head), + error: "Complete only works with external commands".into(), + msg: "complete only works on external commands".into(), + span: Some(head), help: None, inner: vec![], }), diff --git a/crates/nu-command/src/system/nu_check.rs b/crates/nu-command/src/system/nu_check.rs index 260a1c7a59..f9e0879c00 100644 --- a/crates/nu-command/src/system/nu_check.rs +++ b/crates/nu-command/src/system/nu_check.rs @@ -69,18 +69,8 @@ impl Command for NuCheck { parse_script(&mut working_set, None, &contents, is_debug, call.head) } } - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - let mut contents = vec![]; - let raw_stream: Vec<_> = stream.stream.collect(); - for r in raw_stream { - match r { - Ok(v) => contents.extend(v), - Err(error) => return Err(error), - }; - } + PipelineData::ByteStream(stream, ..) => { + let contents = stream.into_bytes()?; if as_module { parse_module(&mut working_set, None, &contents, is_debug, call.head) @@ -160,7 +150,7 @@ impl Command for NuCheck { result: None, }, Example { - description: "Parse an external stream as script by showing error message", + description: "Parse a byte stream as script by showing error message", example: "open foo.nu | nu-check --debug script.nu", result: None, }, diff --git a/crates/nu-command/src/system/run_external.rs b/crates/nu-command/src/system/run_external.rs index e73bd4ab50..2941d80de3 100644 --- a/crates/nu-command/src/system/run_external.rs +++ b/crates/nu-command/src/system/run_external.rs @@ -1,16 +1,16 @@ use nu_cmd_base::hook::eval_hook; use nu_engine::{command_prelude::*, env_to_strings, get_eval_expression}; -use nu_protocol::{ast::Expr, did_you_mean, ListStream, NuGlob, OutDest, RawStream}; +use nu_protocol::{ast::Expr, did_you_mean, process::ChildProcess, ByteStream, NuGlob, OutDest}; use nu_system::ForegroundChild; use nu_utils::IgnoreCaseExt; use os_pipe::PipeReader; use pathdiff::diff_paths; use std::{ collections::HashMap, - io::{BufRead, BufReader, Read, Write}, + io::Write, path::{Path, PathBuf}, process::{Command as CommandSys, Stdio}, - sync::{mpsc, Arc}, + sync::Arc, thread, }; @@ -163,89 +163,124 @@ impl ExternalCommand { ) -> Result { let head = self.name.span; - #[allow(unused_mut)] - let (cmd, mut reader) = self.create_process(&input, false, head)?; - - #[cfg(all(not(unix), not(windows)))] // are there any systems like this? - let child = ForegroundChild::spawn(cmd); - #[cfg(windows)] - let child = match ForegroundChild::spawn(cmd) { - Ok(child) => Ok(child), - Err(err) => { - // Running external commands on Windows has 2 points of complication: - // 1. Some common Windows commands are actually built in to cmd.exe, not executables in their own right. - // 2. We need to let users run batch scripts etc. (.bat, .cmd) without typing their extension + let (child, reader, input) = { + // We may need to run `create_process` again, so we have to clone the underlying + // file or pipe in `input` here first. + let (input_consumed, stdin) = match &input { + PipelineData::ByteStream(stream, ..) => match stream.source() { + nu_protocol::ByteStreamSource::Read(_) => (false, Stdio::piped()), + nu_protocol::ByteStreamSource::File(file) => { + (true, file.try_clone().err_span(head)?.into()) + } + nu_protocol::ByteStreamSource::Child(child) => { + if let Some(nu_protocol::process::ChildPipe::Pipe(pipe)) = &child.stdout { + (true, pipe.try_clone().err_span(head)?.into()) + } else { + (false, Stdio::piped()) + } + } + }, + PipelineData::Empty => (false, Stdio::inherit()), + _ => (false, Stdio::piped()), + }; - // To support these situations, we have a fallback path that gets run if a command - // fails to be run as a normal executable: - // 1. "shell out" to cmd.exe if the command is a known cmd.exe internal command - // 2. Otherwise, use `which-rs` to look for batch files etc. then run those in cmd.exe + let mut input = input; + let (cmd, mut reader) = self.create_process(stdin, false, head)?; + let child = match ForegroundChild::spawn(cmd) { + Ok(child) => { + if input_consumed { + input = PipelineData::Empty; + } + Ok(child) + } + Err(err) => { + // Running external commands on Windows has 2 points of complication: + // 1. Some common Windows commands are actually built in to cmd.exe, not executables in their own right. + // 2. We need to let users run batch scripts etc. (.bat, .cmd) without typing their extension - // set the default value, maybe we'll override it later - let mut child = Err(err); + // To support these situations, we have a fallback path that gets run if a command + // fails to be run as a normal executable: + // 1. "shell out" to cmd.exe if the command is a known cmd.exe internal command + // 2. Otherwise, use `which-rs` to look for batch files etc. then run those in cmd.exe - // This has the full list of cmd.exe "internal" commands: https://ss64.com/nt/syntax-internal.html - // I (Reilly) went through the full list and whittled it down to ones that are potentially useful: - const CMD_INTERNAL_COMMANDS: [&str; 9] = [ - "ASSOC", "CLS", "ECHO", "FTYPE", "MKLINK", "PAUSE", "START", "VER", "VOL", - ]; - let command_name = &self.name.item; - let looks_like_cmd_internal = CMD_INTERNAL_COMMANDS - .iter() - .any(|&cmd| command_name.eq_ignore_ascii_case(cmd)); + // set the default value, maybe we'll override it later + let mut child = Err(err); - if looks_like_cmd_internal { - let (cmd, new_reader) = self.create_process(&input, true, head)?; - reader = new_reader; - child = ForegroundChild::spawn(cmd); - } else { - #[cfg(feature = "which-support")] - { - // maybe it's a batch file (foo.cmd) and the user typed `foo`. Try to find it with `which-rs` - // TODO: clean this up with an if-let chain once those are stable - if let Ok(path) = - nu_engine::env::path_str(engine_state, stack, self.name.span) + // This has the full list of cmd.exe "internal" commands: https://ss64.com/nt/syntax-internal.html + // I (Reilly) went through the full list and whittled it down to ones that are potentially useful: + const CMD_INTERNAL_COMMANDS: [&str; 9] = [ + "ASSOC", "CLS", "ECHO", "FTYPE", "MKLINK", "PAUSE", "START", "VER", "VOL", + ]; + let command_name = &self.name.item; + let looks_like_cmd_internal = CMD_INTERNAL_COMMANDS + .iter() + .any(|&cmd| command_name.eq_ignore_ascii_case(cmd)); + + let (data, stdin) = extract_stdio(input); + input = data; + + if looks_like_cmd_internal { + let (cmd, new_reader) = self.create_process(stdin, true, head)?; + reader = new_reader; + child = ForegroundChild::spawn(cmd); + } else { + #[cfg(feature = "which-support")] { - if let Some(cwd) = self.env_vars.get("PWD") { - // append cwd to PATH so `which-rs` looks in the cwd too. - // this approximates what cmd.exe does. - let path_with_cwd = format!("{};{}", cwd, path); - if let Ok(which_path) = - which::which_in(&self.name.item, Some(path_with_cwd), cwd) - { - if let Some(file_name) = which_path.file_name() { - if !file_name.to_string_lossy().eq_ignore_case(command_name) - { - // which-rs found an executable file with a slightly different name - // than the one the user tried. Let's try running it - let mut new_command = self.clone(); - new_command.name = Spanned { - item: file_name.to_string_lossy().to_string(), - span: self.name.span, - }; - let (cmd, new_reader) = - new_command.create_process(&input, true, head)?; - reader = new_reader; - child = ForegroundChild::spawn(cmd); + // maybe it's a batch file (foo.cmd) and the user typed `foo`. Try to find it with `which-rs` + // TODO: clean this up with an if-let chain once those are stable + if let Ok(path) = + nu_engine::env::path_str(engine_state, stack, self.name.span) + { + if let Some(cwd) = self.env_vars.get("PWD") { + // append cwd to PATH so `which-rs` looks in the cwd too. + // this approximates what cmd.exe does. + let path_with_cwd = format!("{};{}", cwd, path); + if let Ok(which_path) = + which::which_in(&self.name.item, Some(path_with_cwd), cwd) + { + if let Some(file_name) = which_path.file_name() { + if !file_name + .to_string_lossy() + .eq_ignore_case(command_name) + { + // which-rs found an executable file with a slightly different name + // than the one the user tried. Let's try running it + let mut new_command = self.clone(); + new_command.name = Spanned { + item: file_name.to_string_lossy().to_string(), + span: self.name.span, + }; + let (cmd, new_reader) = new_command + .create_process(stdin, true, head)?; + reader = new_reader; + child = ForegroundChild::spawn(cmd); + } } } } } } } - } - child - } + child + } + }; + + (child, reader, input) }; #[cfg(unix)] - let child = ForegroundChild::spawn( - cmd, - engine_state.is_interactive, - &engine_state.pipeline_externals_state, - ); + let (child, reader, input) = { + let (input, stdin) = extract_stdio(input); + let (cmd, reader) = self.create_process(stdin, false, head)?; + let child = ForegroundChild::spawn( + cmd, + engine_state.is_interactive, + &engine_state.pipeline_externals_state, + ); + (child, reader, input) + }; match child { Err(err) => { @@ -381,9 +416,8 @@ impl ExternalCommand { .name("external stdin worker".to_string()) .spawn(move || { let input = match input { - input @ PipelineData::Value(Value::Binary { .. }, ..) => { - Ok(input) - } + input @ PipelineData::ByteStream(..) => input, + input @ PipelineData::Value(Value::Binary { .. }, ..) => input, input => { let stack = &mut stack.start_capture(); // Attempt to render the input as a table before piping it to the external. @@ -397,143 +431,39 @@ impl ExternalCommand { stack, &Call::new(head), input, - ) + )? } }; - if let Ok(input) = input { + if let PipelineData::ByteStream(stream, ..) = input { + stream.write_to(&mut stdin_write)?; + } else { for value in input.into_iter() { - let buf = match value { - Value::String { val, .. } => val.into_bytes(), - Value::Binary { val, .. } => val, - _ => return Err(()), - }; - if stdin_write.write(&buf).is_err() { - return Ok(()); - } + let buf = value.coerce_into_binary()?; + stdin_write.write_all(&buf)?; } } - Ok(()) + Ok::<_, ShellError>(()) }) .err_span(head)?; } } - #[cfg(unix)] - let commandname = self.name.item.clone(); - let span = self.name.span; - let (exit_code_tx, exit_code_rx) = mpsc::channel(); + let child = + ChildProcess::new(child, reader, matches!(self.err, OutDest::Pipe), head)?; - let (stdout, stderr) = if let Some(combined) = reader { - ( - Some(RawStream::new( - Box::new(ByteLines::new(combined)), - engine_state.ctrlc.clone(), - head, - None, - )), - None, - ) - } else { - let stdout = child.as_mut().stdout.take().map(|out| { - RawStream::new( - Box::new(ByteLines::new(out)), - engine_state.ctrlc.clone(), - head, - None, - ) - }); - - let stderr = child.as_mut().stderr.take().map(|err| { - RawStream::new( - Box::new(ByteLines::new(err)), - engine_state.ctrlc.clone(), - head, - None, - ) - }); - - if matches!(self.err, OutDest::Pipe) { - (stderr, stdout) - } else { - (stdout, stderr) - } - }; - - // Create a thread to wait for an exit code. - thread::Builder::new() - .name("exit code waiter".into()) - .spawn(move || match child.as_mut().wait() { - Err(err) => Err(ShellError::ExternalCommand { - label: "External command exited with error".into(), - help: err.to_string(), - span, - }), - Ok(x) => { - #[cfg(unix)] - { - use nix::sys::signal::Signal; - use nu_ansi_term::{Color, Style}; - use std::os::unix::process::ExitStatusExt; - - if x.core_dumped() { - let cause = x - .signal() - .and_then(|sig| { - Signal::try_from(sig).ok().map(Signal::as_str) - }) - .unwrap_or("Something went wrong"); - - let style = Style::new().bold().on(Color::Red); - let message = format!( - "{cause}: child process '{commandname}' core dumped" - ); - eprintln!("{}", style.paint(&message)); - let _ = exit_code_tx.send(Value::error( - ShellError::ExternalCommand { - label: "core dumped".into(), - help: message, - span: head, - }, - head, - )); - return Ok(()); - } - } - if let Some(code) = x.code() { - let _ = exit_code_tx.send(Value::int(code as i64, head)); - } else if x.success() { - let _ = exit_code_tx.send(Value::int(0, head)); - } else { - let _ = exit_code_tx.send(Value::int(-1, head)); - } - Ok(()) - } - }) - .err_span(head)?; - - let exit_code = Some(ListStream::new( - ValueReceiver::new(exit_code_rx), - head, + Ok(PipelineData::ByteStream( + ByteStream::child(child, head), None, - )); - - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span: head, - metadata: None, - trim_end_newline: true, - }) + )) } } } pub fn create_process( &self, - input: &PipelineData, + stdin: Stdio, use_cmd: bool, span: Span, ) -> Result<(CommandSys, Option), ShellError> { @@ -578,11 +508,7 @@ impl ExternalCommand { None }; - // If there is an input from the pipeline. The stdin from the process - // is piped so it can be used to send the input information - if !input.is_nothing() { - process.stdin(Stdio::piped()); - } + process.stdin(stdin); Ok((process, reader)) } @@ -764,51 +690,14 @@ fn remove_quotes(input: String) -> String { } } -struct ByteLines(BufReader); - -impl ByteLines { - fn new(read: R) -> Self { - Self(BufReader::new(read)) - } -} - -impl Iterator for ByteLines { - type Item = Result, ShellError>; - - fn next(&mut self) -> Option { - let mut buf = Vec::new(); - // `read_until` will never stop reading unless `\n` or EOF is encountered, - // so let's limit the number of bytes using `take` as the Rust docs suggest. - let capacity = self.0.capacity() as u64; - let mut reader = (&mut self.0).take(capacity); - match reader.read_until(b'\n', &mut buf) { - Ok(0) => None, - Ok(_) => Some(Ok(buf)), - Err(e) => Some(Err(e.into())), - } - } -} - -// Receiver used for the ListStream -// It implements iterator so it can be used as a ListStream -struct ValueReceiver { - rx: mpsc::Receiver, -} - -impl ValueReceiver { - pub fn new(rx: mpsc::Receiver) -> Self { - Self { rx } - } -} - -impl Iterator for ValueReceiver { - type Item = Value; - - fn next(&mut self) -> Option { - match self.rx.recv() { - Ok(v) => Some(v), - Err(_) => None, - } +fn extract_stdio(pipeline: PipelineData) -> (PipelineData, Stdio) { + match pipeline { + PipelineData::ByteStream(stream, metadata) => match stream.into_stdio() { + Ok(pipe) => (PipelineData::Empty, pipe), + Err(stream) => (PipelineData::ByteStream(stream, metadata), Stdio::piped()), + }, + PipelineData::Empty => (PipelineData::Empty, Stdio::inherit()), + data => (data, Stdio::piped()), } } diff --git a/crates/nu-command/src/viewers/table.rs b/crates/nu-command/src/viewers/table.rs index f4df2e03bc..26b8c921c5 100644 --- a/crates/nu-command/src/viewers/table.rs +++ b/crates/nu-command/src/viewers/table.rs @@ -6,7 +6,7 @@ use lscolors::{LsColors, Style}; use nu_color_config::{color_from_hex, StyleComputer, TextStyle}; use nu_engine::{command_prelude::*, env::get_config, env_to_string}; use nu_protocol::{ - Config, DataSource, ListStream, PipelineMetadata, RawStream, TableMode, ValueIterator, + ByteStream, Config, DataSource, ListStream, PipelineMetadata, TableMode, ValueIterator, }; use nu_table::{ common::create_nu_table_config, CollapsedTable, ExpandedTable, JustTable, NuTable, NuTableCell, @@ -14,8 +14,12 @@ use nu_table::{ }; use nu_utils::get_ls_colors; use std::{ - collections::VecDeque, io::IsTerminal, path::PathBuf, str::FromStr, sync::atomic::AtomicBool, - sync::Arc, time::Instant, + collections::VecDeque, + io::{Cursor, IsTerminal}, + path::PathBuf, + str::FromStr, + sync::{atomic::AtomicBool, Arc}, + time::Instant, }; use terminal_size::{Height, Width}; use url::Url; @@ -360,25 +364,16 @@ fn handle_table_command( ) -> Result { let span = input.data.span().unwrap_or(input.call.head); match input.data { - PipelineData::ExternalStream { .. } => Ok(input.data), + PipelineData::ByteStream(..) => Ok(input.data), PipelineData::Value(Value::Binary { val, .. }, ..) => { - let bytes = format!("{}\n", nu_pretty_hex::pretty_hex(&val)).into_bytes(); + let bytes = { + let mut str = nu_pretty_hex::pretty_hex(&val); + str.push('\n'); + str.into_bytes() + }; let ctrlc = input.engine_state.ctrlc.clone(); - let stream = RawStream::new( - Box::new([Ok(bytes)].into_iter()), - ctrlc, - input.call.head, - None, - ); - - Ok(PipelineData::ExternalStream { - stdout: Some(stream), - stderr: None, - exit_code: None, - span: input.call.head, - metadata: None, - trim_end_newline: false, - }) + let stream = ByteStream::read(Cursor::new(bytes), input.call.head, ctrlc); + Ok(PipelineData::ByteStream(stream, None)) } // None of these two receive a StyleComputer because handle_row_stream() can produce it by itself using engine_state and stack. PipelineData::Value(Value::List { vals, .. }, metadata) => { @@ -613,16 +608,8 @@ fn handle_row_stream( ctrlc.clone(), cfg, ); - let stream = RawStream::new(Box::new(paginator), ctrlc, input.call.head, None); - - Ok(PipelineData::ExternalStream { - stdout: Some(stream), - stderr: None, - exit_code: None, - span: input.call.head, - metadata: None, - trim_end_newline: false, - }) + let stream = ByteStream::from_result_iter(paginator, input.call.head, None); + Ok(PipelineData::ByteStream(stream, None)) } fn make_clickable_link( diff --git a/crates/nu-command/tests/format_conversions/csv.rs b/crates/nu-command/tests/format_conversions/csv.rs index 5915b3c4d4..a9be76d5c3 100644 --- a/crates/nu-command/tests/format_conversions/csv.rs +++ b/crates/nu-command/tests/format_conversions/csv.rs @@ -183,6 +183,7 @@ fn from_csv_text_with_tab_separator_to_table() { } #[test] +#[ignore = "csv crate has a bug when the last line is a comment: https://github.com/BurntSushi/rust-csv/issues/363"] fn from_csv_text_with_comments_to_table() { Playground::setup("filter_from_csv_test_5", |dirs, sandbox| { sandbox.with_files(&[FileWithContentToBeTrimmed( diff --git a/crates/nu-command/tests/format_conversions/tsv.rs b/crates/nu-command/tests/format_conversions/tsv.rs index 9627d0d0be..be57c60242 100644 --- a/crates/nu-command/tests/format_conversions/tsv.rs +++ b/crates/nu-command/tests/format_conversions/tsv.rs @@ -106,6 +106,7 @@ fn from_tsv_text_to_table() { } #[test] +#[ignore = "csv crate has a bug when the last line is a comment: https://github.com/BurntSushi/rust-csv/issues/363"] fn from_tsv_text_with_comments_to_table() { Playground::setup("filter_from_tsv_test_2", |dirs, sandbox| { sandbox.with_files(&[FileWithContentToBeTrimmed( diff --git a/crates/nu-engine/src/documentation.rs b/crates/nu-engine/src/documentation.rs index 2e966a312f..62e68eaa6c 100644 --- a/crates/nu-engine/src/documentation.rs +++ b/crates/nu-engine/src/documentation.rs @@ -53,7 +53,7 @@ fn nu_highlight_string(code_string: &str, engine_state: &EngineState, stack: &mu Value::string(code_string, Span::unknown()).into_pipeline_data(), ) { let result = output.into_value(Span::unknown()); - if let Ok(s) = result.coerce_into_string() { + if let Ok(s) = result.and_then(Value::coerce_into_string) { return s; // successfully highlighted string } } @@ -280,7 +280,7 @@ fn get_documentation( ) { Ok(output) => { let result = output.into_value(Span::unknown()); - match result.coerce_into_string() { + match result.and_then(Value::coerce_into_string) { Ok(s) => { let _ = write!(long_desc, "\n > {s}\n"); } diff --git a/crates/nu-engine/src/env.rs b/crates/nu-engine/src/env.rs index 44692dd131..048d9bfb99 100644 --- a/crates/nu-engine/src/env.rs +++ b/crates/nu-engine/src/env.rs @@ -350,14 +350,15 @@ fn get_converted_value( .and_then(|record| record.get(direction)); if let Some(conversion) = conversion { - match conversion.as_closure() { - Ok(closure) => ClosureEvalOnce::new(engine_state, stack, closure.clone()) - .debug(false) - .run_with_value(orig_val.clone()) - .map(|data| ConversionResult::Ok(data.into_value(orig_val.span()))) - .unwrap_or_else(ConversionResult::ConversionError), - Err(e) => ConversionResult::ConversionError(e), - } + conversion + .as_closure() + .and_then(|closure| { + ClosureEvalOnce::new(engine_state, stack, closure.clone()) + .debug(false) + .run_with_value(orig_val.clone()) + }) + .and_then(|data| data.into_value(orig_val.span())) + .map_or_else(ConversionResult::ConversionError, ConversionResult::Ok) } else { ConversionResult::CellPathError } diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index b8d806b708..0bc0c3727c 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -9,8 +9,8 @@ use nu_protocol::{ debugger::DebugContext, engine::{Closure, EngineState, Redirection, Stack}, eval_base::Eval, - Config, FromValue, IntoPipelineData, OutDest, PipelineData, ShellError, Span, Spanned, Type, - Value, VarId, ENV_VARIABLE_ID, + ByteStreamSource, Config, FromValue, IntoPipelineData, OutDest, PipelineData, ShellError, Span, + Spanned, Type, Value, VarId, ENV_VARIABLE_ID, }; use nu_utils::IgnoreCaseExt; use std::{borrow::Cow, fs::OpenOptions, path::PathBuf}; @@ -209,7 +209,6 @@ pub fn redirect_env(engine_state: &EngineState, caller_stack: &mut Stack, callee } } -#[allow(clippy::too_many_arguments)] fn eval_external( engine_state: &EngineState, stack: &mut Stack, @@ -284,7 +283,7 @@ pub fn eval_expression_with_input( let stack = &mut stack.start_capture(); // FIXME: protect this collect with ctrl-c input = eval_subexpression::(engine_state, stack, block, input)? - .into_value(*span) + .into_value(*span)? .follow_cell_path(&full_cell_path.tail, false)? .into_pipeline_data() } else { @@ -301,7 +300,7 @@ pub fn eval_expression_with_input( } }; - // If input is PipelineData::ExternalStream, + // If input an external command, // then `might_consume_external_result` will consume `stderr` if `stdout` is `None`. // This should not happen if the user wants to capture stderr. if !matches!(stack.stdout(), OutDest::Pipe | OutDest::Capture) @@ -309,15 +308,10 @@ pub fn eval_expression_with_input( { Ok((input, false)) } else { - Ok(might_consume_external_result(input)) + input.check_external_failed() } } -// Try to catch and detect if external command runs to failed. -fn might_consume_external_result(input: PipelineData) -> (PipelineData, bool) { - input.check_external_failed() -} - fn eval_redirection( engine_state: &EngineState, stack: &mut Stack, @@ -410,10 +404,17 @@ fn eval_element_with_input_inner( element: &PipelineElement, input: PipelineData, ) -> Result<(PipelineData, bool), ShellError> { - let (data, ok) = eval_expression_with_input::(engine_state, stack, &element.expr, input)?; + let (data, failed) = + eval_expression_with_input::(engine_state, stack, &element.expr, input)?; - if !matches!(data, PipelineData::ExternalStream { .. }) { - if let Some(redirection) = element.redirection.as_ref() { + if let Some(redirection) = element.redirection.as_ref() { + let is_external = if let PipelineData::ByteStream(stream, ..) = &data { + matches!(stream.source(), ByteStreamSource::Child(..)) + } else { + false + }; + + if !is_external { match redirection { &PipelineRedirection::Single { source: RedirectionSource::Stderr, @@ -424,8 +425,8 @@ fn eval_element_with_input_inner( .. } => { return Err(ShellError::GenericError { - error: "`e>|` only works with external streams".into(), - msg: "`e>|` only works on external streams".into(), + error: "`e>|` only works on external commands".into(), + msg: "`e>|` only works on external commands".into(), span: Some(span), help: None, inner: vec![], @@ -436,8 +437,8 @@ fn eval_element_with_input_inner( target: RedirectionTarget::Pipe { span }, } => { return Err(ShellError::GenericError { - error: "`o+e>|` only works with external streams".into(), - msg: "`o+e>|` only works on external streams".into(), + error: "`o+e>|` only works on external commands".into(), + msg: "`o+e>|` only works on external commands".into(), span: Some(span), help: None, inner: vec![], @@ -448,15 +449,33 @@ fn eval_element_with_input_inner( } } - let data = if matches!(stack.pipe_stdout(), Some(OutDest::File(_))) - && !matches!(stack.pipe_stderr(), Some(OutDest::Pipe)) - { - data.write_to_out_dests(engine_state, stack)? - } else { - data + let has_stdout_file = matches!(stack.pipe_stdout(), Some(OutDest::File(_))); + + let data = match &data { + PipelineData::Value(..) | PipelineData::ListStream(..) => { + if has_stdout_file { + data.write_to_out_dests(engine_state, stack)?; + PipelineData::Empty + } else { + data + } + } + PipelineData::ByteStream(stream, ..) => { + let write = match stream.source() { + ByteStreamSource::Read(_) | ByteStreamSource::File(_) => has_stdout_file, + ByteStreamSource::Child(_) => false, + }; + if write { + data.write_to_out_dests(engine_state, stack)?; + PipelineData::Empty + } else { + data + } + } + PipelineData::Empty => PipelineData::Empty, }; - Ok((data, ok)) + Ok((data, failed)) } fn eval_element_with_input( @@ -466,12 +485,18 @@ fn eval_element_with_input( input: PipelineData, ) -> Result<(PipelineData, bool), ShellError> { D::enter_element(engine_state, element); - - let result = eval_element_with_input_inner::(engine_state, stack, element, input); - - D::leave_element(engine_state, element, &result); - - result + match eval_element_with_input_inner::(engine_state, stack, element, input) { + Ok((data, failed)) => { + let res = Ok(data); + D::leave_element(engine_state, element, &res); + res.map(|data| (data, failed)) + } + Err(err) => { + let res = Err(err); + D::leave_element(engine_state, element, &res); + res.map(|data| (data, false)) + } + } } pub fn eval_block_with_early_return( @@ -555,17 +580,20 @@ pub fn eval_block( } input = PipelineData::Empty; match output { - stream @ PipelineData::ExternalStream { .. } => { - let exit_code = stream.drain_with_exit_code()?; - stack.add_env_var( - "LAST_EXIT_CODE".into(), - Value::int(exit_code, last.expr.span), - ); - if exit_code != 0 { - break; + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + let status = stream.drain()?; + if let Some(status) = status { + stack.add_env_var( + "LAST_EXIT_CODE".into(), + Value::int(status.code().into(), span), + ); + if status.code() != 0 { + break; + } } } - PipelineData::ListStream(stream, _) => { + PipelineData::ListStream(stream, ..) => { stream.drain()?; } PipelineData::Value(..) | PipelineData::Empty => {} @@ -684,7 +712,7 @@ impl Eval for EvalRuntime { _: Span, ) -> Result { // FIXME: protect this collect with ctrl-c - Ok(eval_call::(engine_state, stack, call, PipelineData::empty())?.into_value(call.head)) + eval_call::(engine_state, stack, call, PipelineData::empty())?.into_value(call.head) } fn eval_external_call( @@ -696,7 +724,7 @@ impl Eval for EvalRuntime { ) -> Result { let span = head.span; // FIXME: protect this collect with ctrl-c - Ok(eval_external(engine_state, stack, head, args, PipelineData::empty())?.into_value(span)) + eval_external(engine_state, stack, head, args, PipelineData::empty())?.into_value(span) } fn eval_subexpression( @@ -706,12 +734,8 @@ impl Eval for EvalRuntime { span: Span, ) -> Result { let block = engine_state.get_block(block_id); - // FIXME: protect this collect with ctrl-c - Ok( - eval_subexpression::(engine_state, stack, block, PipelineData::empty())? - .into_value(span), - ) + eval_subexpression::(engine_state, stack, block, PipelineData::empty())?.into_value(span) } fn regex_match( diff --git a/crates/nu-explore/src/nu_common/value.rs b/crates/nu-explore/src/nu_common/value.rs index 17b277cac5..8aa71a28bf 100644 --- a/crates/nu-explore/src/nu_common/value.rs +++ b/crates/nu-explore/src/nu_common/value.rs @@ -1,7 +1,7 @@ use super::NuSpan; use anyhow::Result; use nu_engine::get_columns; -use nu_protocol::{record, ListStream, PipelineData, PipelineMetadata, RawStream, Value}; +use nu_protocol::{record, ByteStream, ListStream, PipelineData, PipelineMetadata, Value}; use std::collections::HashMap; pub fn collect_pipeline(input: PipelineData) -> Result<(Vec, Vec>)> { @@ -9,16 +9,7 @@ pub fn collect_pipeline(input: PipelineData) -> Result<(Vec, Vec Ok((vec![], vec![])), PipelineData::Value(value, ..) => collect_input(value), PipelineData::ListStream(stream, ..) => Ok(collect_list_stream(stream)), - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - metadata, - span, - .. - } => Ok(collect_external_stream( - stdout, stderr, exit_code, metadata, span, - )), + PipelineData::ByteStream(stream, metadata) => Ok(collect_byte_stream(stream, metadata)), } } @@ -42,49 +33,60 @@ fn collect_list_stream(stream: ListStream) -> (Vec, Vec>) { (cols, data) } -fn collect_external_stream( - stdout: Option, - stderr: Option, - exit_code: Option, +fn collect_byte_stream( + stream: ByteStream, metadata: Option, - span: NuSpan, ) -> (Vec, Vec>) { + let span = stream.span(); + let mut columns = vec![]; let mut data = vec![]; - if let Some(stdout) = stdout { - let value = stdout.into_string().map_or_else( - |error| Value::error(error, span), - |string| Value::string(string.item, span), - ); - columns.push(String::from("stdout")); - data.push(value); - } - if let Some(stderr) = stderr { - let value = stderr.into_string().map_or_else( - |error| Value::error(error, span), - |string| Value::string(string.item, span), - ); + match stream.into_child() { + Ok(child) => match child.wait_with_output() { + Ok(output) => { + let exit_code = output.exit_status.code(); + if let Some(stdout) = output.stdout { + columns.push(String::from("stdout")); + data.push(string_or_binary(stdout, span)); + } + if let Some(stderr) = output.stderr { + columns.push(String::from("stderr")); + data.push(string_or_binary(stderr, span)); + } + columns.push(String::from("exit_code")); + data.push(Value::int(exit_code.into(), span)); + } + Err(err) => { + columns.push("".into()); + data.push(Value::error(err, span)); + } + }, + Err(stream) => { + let value = stream + .into_value() + .unwrap_or_else(|err| Value::error(err, span)); - columns.push(String::from("stderr")); - data.push(value); + columns.push("".into()); + data.push(value); + } } - if let Some(exit_code) = exit_code { - let list = exit_code.into_iter().collect::>(); - let val = Value::list(list, span); - columns.push(String::from("exit_code")); - data.push(val); - } if metadata.is_some() { let val = Value::record(record! { "data_source" => Value::string("ls", span) }, span); - columns.push(String::from("metadata")); data.push(val); } (columns, vec![data]) } +fn string_or_binary(bytes: Vec, span: NuSpan) -> Value { + match String::from_utf8(bytes) { + Ok(str) => Value::string(str, span), + Err(err) => Value::binary(err.into_bytes(), span), + } +} + /// Try to build column names and a table grid. pub fn collect_input(value: Value) -> Result<(Vec, Vec>)> { let span = value.span(); diff --git a/crates/nu-plugin-core/src/interface/mod.rs b/crates/nu-plugin-core/src/interface/mod.rs index 3fb86aee36..b4a2bc9a25 100644 --- a/crates/nu-plugin-core/src/interface/mod.rs +++ b/crates/nu-plugin-core/src/interface/mod.rs @@ -1,15 +1,10 @@ //! Implements the stream multiplexing interface for both the plugin side and the engine side. -use nu_plugin_protocol::{ - ExternalStreamInfo, ListStreamInfo, PipelineDataHeader, RawStreamInfo, StreamMessage, -}; -use nu_protocol::{ListStream, PipelineData, RawStream, ShellError}; +use nu_plugin_protocol::{ByteStreamInfo, ListStreamInfo, PipelineDataHeader, StreamMessage}; +use nu_protocol::{ByteStream, IntoSpanned, ListStream, PipelineData, Reader, ShellError}; use std::{ - io::Write, - sync::{ - atomic::{AtomicBool, Ordering::Relaxed}, - Arc, Mutex, - }, + io::{Read, Write}, + sync::{atomic::AtomicBool, Arc, Mutex}, thread, }; @@ -185,31 +180,10 @@ pub trait InterfaceManager { let reader = handle.read_stream(info.id, self.get_interface())?; ListStream::new(reader, info.span, ctrlc.cloned()).into() } - PipelineDataHeader::ExternalStream(info) => { + PipelineDataHeader::ByteStream(info) => { let handle = self.stream_manager().get_handle(); - let span = info.span; - let new_raw_stream = |raw_info: RawStreamInfo| { - let reader = handle.read_stream(raw_info.id, self.get_interface())?; - let mut stream = - RawStream::new(Box::new(reader), ctrlc.cloned(), span, raw_info.known_size); - stream.is_binary = raw_info.is_binary; - Ok::<_, ShellError>(stream) - }; - PipelineData::ExternalStream { - stdout: info.stdout.map(new_raw_stream).transpose()?, - stderr: info.stderr.map(new_raw_stream).transpose()?, - exit_code: info - .exit_code - .map(|list_info| { - handle - .read_stream(list_info.id, self.get_interface()) - .map(|reader| ListStream::new(reader, info.span, ctrlc.cloned())) - }) - .transpose()?, - span: info.span, - metadata: None, - trim_end_newline: info.trim_end_newline, - } + let reader = handle.read_stream(info.id, self.get_interface())?; + ByteStream::from_result_iter(reader, info.span, ctrlc.cloned()).into() } }) } @@ -271,11 +245,11 @@ pub trait Interface: Clone + Send { Ok::<_, ShellError>((id, writer)) }; match self.prepare_pipeline_data(data, context)? { - PipelineData::Value(value, _) => { + PipelineData::Value(value, ..) => { Ok((PipelineDataHeader::Value(value), PipelineDataWriter::None)) } PipelineData::Empty => Ok((PipelineDataHeader::Empty, PipelineDataWriter::None)), - PipelineData::ListStream(stream, _) => { + PipelineData::ListStream(stream, ..) => { let (id, writer) = new_stream(LIST_STREAM_HIGH_PRESSURE)?; Ok(( PipelineDataHeader::ListStream(ListStreamInfo { @@ -285,50 +259,15 @@ pub trait Interface: Clone + Send { PipelineDataWriter::ListStream(writer, stream), )) } - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata: _, - trim_end_newline, - } => { - // Create the writers and stream ids - let stdout_stream = stdout - .is_some() - .then(|| new_stream(RAW_STREAM_HIGH_PRESSURE)) - .transpose()?; - let stderr_stream = stderr - .is_some() - .then(|| new_stream(RAW_STREAM_HIGH_PRESSURE)) - .transpose()?; - let exit_code_stream = exit_code - .is_some() - .then(|| new_stream(LIST_STREAM_HIGH_PRESSURE)) - .transpose()?; - // Generate the header, with the stream ids - let header = PipelineDataHeader::ExternalStream(ExternalStreamInfo { - span, - stdout: stdout - .as_ref() - .zip(stdout_stream.as_ref()) - .map(|(stream, (id, _))| RawStreamInfo::new(*id, stream)), - stderr: stderr - .as_ref() - .zip(stderr_stream.as_ref()) - .map(|(stream, (id, _))| RawStreamInfo::new(*id, stream)), - exit_code: exit_code_stream - .as_ref() - .map(|&(id, _)| ListStreamInfo { id, span }), - trim_end_newline, - }); - // Collect the writers - let writer = PipelineDataWriter::ExternalStream { - stdout: stdout_stream.map(|(_, writer)| writer).zip(stdout), - stderr: stderr_stream.map(|(_, writer)| writer).zip(stderr), - exit_code: exit_code_stream.map(|(_, writer)| writer).zip(exit_code), - }; - Ok((header, writer)) + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + if let Some(reader) = stream.reader() { + let (id, writer) = new_stream(RAW_STREAM_HIGH_PRESSURE)?; + let header = PipelineDataHeader::ByteStream(ByteStreamInfo { id, span }); + Ok((header, PipelineDataWriter::ByteStream(writer, reader))) + } else { + Ok((PipelineDataHeader::Empty, PipelineDataWriter::None)) + } } } } @@ -355,11 +294,7 @@ pub enum PipelineDataWriter { #[default] None, ListStream(StreamWriter, ListStream), - ExternalStream { - stdout: Option<(StreamWriter, RawStream)>, - stderr: Option<(StreamWriter, RawStream)>, - exit_code: Option<(StreamWriter, ListStream)>, - }, + ByteStream(StreamWriter, Reader), } impl PipelineDataWriter @@ -376,49 +311,16 @@ where writer.write_all(stream)?; Ok(()) } - // Write all three possible streams of an ExternalStream on separate threads. - PipelineDataWriter::ExternalStream { - stdout, - stderr, - exit_code, - } => { - thread::scope(|scope| { - let stderr_thread = stderr - .map(|(mut writer, stream)| { - thread::Builder::new() - .name("plugin stderr writer".into()) - .spawn_scoped(scope, move || { - writer.write_all(raw_stream_iter(stream)) - }) - }) - .transpose()?; - let exit_code_thread = exit_code - .map(|(mut writer, stream)| { - thread::Builder::new() - .name("plugin exit_code writer".into()) - .spawn_scoped(scope, move || writer.write_all(stream)) - }) - .transpose()?; - // Optimize for stdout: if only stdout is present, don't spawn any other - // threads. - if let Some((mut writer, stream)) = stdout { - writer.write_all(raw_stream_iter(stream))?; - } - let panicked = |thread_name: &str| { - Err(ShellError::NushellFailed { - msg: format!( - "{thread_name} thread panicked in PipelineDataWriter::write" - ), - }) - }; - stderr_thread - .map(|t| t.join().unwrap_or_else(|_| panicked("stderr"))) - .transpose()?; - exit_code_thread - .map(|t| t.join().unwrap_or_else(|_| panicked("exit_code"))) - .transpose()?; - Ok(()) - }) + // Write a byte stream. + PipelineDataWriter::ByteStream(mut writer, mut reader) => { + let span = reader.span(); + let buf = &mut [0; 8192]; + writer.write_all(std::iter::from_fn(move || match reader.read(buf) { + Ok(0) => None, + Ok(len) => Some(Ok(buf[..len].to_vec())), + Err(err) => Some(Err(ShellError::from(err.into_spanned(span)))), + }))?; + Ok(()) } } } @@ -446,11 +348,3 @@ where } } } - -/// Custom iterator for [`RawStream`] that respects ctrlc, but still has binary chunks -fn raw_stream_iter(stream: RawStream) -> impl Iterator, ShellError>> { - let ctrlc = stream.ctrlc; - stream - .stream - .take_while(move |_| ctrlc.as_ref().map(|b| !b.load(Relaxed)).unwrap_or(true)) -} diff --git a/crates/nu-plugin-core/src/interface/tests.rs b/crates/nu-plugin-core/src/interface/tests.rs index ce7be52f30..fb3d737190 100644 --- a/crates/nu-plugin-core/src/interface/tests.rs +++ b/crates/nu-plugin-core/src/interface/tests.rs @@ -6,11 +6,12 @@ use super::{ Interface, InterfaceManager, PluginRead, PluginWrite, }; use nu_plugin_protocol::{ - ExternalStreamInfo, ListStreamInfo, PipelineDataHeader, PluginInput, PluginOutput, - RawStreamInfo, StreamData, StreamMessage, + ByteStreamInfo, ListStreamInfo, PipelineDataHeader, PluginInput, PluginOutput, StreamData, + StreamMessage, }; use nu_protocol::{ - DataSource, ListStream, PipelineData, PipelineMetadata, RawStream, ShellError, Span, Value, + ByteStream, ByteStreamSource, DataSource, ListStream, PipelineData, PipelineMetadata, + ShellError, Span, Value, }; use std::{path::Path, sync::Arc}; @@ -140,9 +141,9 @@ fn read_pipeline_data_value() -> Result<(), ShellError> { let header = PipelineDataHeader::Value(value.clone()); match manager.read_pipeline_data(header, None)? { - PipelineData::Value(read_value, _) => assert_eq!(value, read_value), - PipelineData::ListStream(_, _) => panic!("unexpected ListStream"), - PipelineData::ExternalStream { .. } => panic!("unexpected ExternalStream"), + PipelineData::Value(read_value, ..) => assert_eq!(value, read_value), + PipelineData::ListStream(..) => panic!("unexpected ListStream"), + PipelineData::ByteStream(..) => panic!("unexpected ByteStream"), PipelineData::Empty => panic!("unexpected Empty"), } @@ -188,47 +189,25 @@ fn read_pipeline_data_list_stream() -> Result<(), ShellError> { } #[test] -fn read_pipeline_data_external_stream() -> Result<(), ShellError> { +fn read_pipeline_data_byte_stream() -> Result<(), ShellError> { let test = TestCase::new(); let mut manager = TestInterfaceManager::new(&test); let iterations = 100; let out_pattern = b"hello".to_vec(); - let err_pattern = vec![5, 4, 3, 2]; - test.add(StreamMessage::Data(14, Value::test_int(1).into())); for _ in 0..iterations { test.add(StreamMessage::Data( 12, StreamData::Raw(Ok(out_pattern.clone())), )); - test.add(StreamMessage::Data( - 13, - StreamData::Raw(Ok(err_pattern.clone())), - )); } test.add(StreamMessage::End(12)); - test.add(StreamMessage::End(13)); - test.add(StreamMessage::End(14)); let test_span = Span::new(10, 13); - let header = PipelineDataHeader::ExternalStream(ExternalStreamInfo { + let header = PipelineDataHeader::ByteStream(ByteStreamInfo { + id: 12, span: test_span, - stdout: Some(RawStreamInfo { - id: 12, - is_binary: false, - known_size: Some((out_pattern.len() * iterations) as u64), - }), - stderr: Some(RawStreamInfo { - id: 13, - is_binary: true, - known_size: None, - }), - exit_code: Some(ListStreamInfo { - id: 14, - span: Span::test_data(), - }), - trim_end_newline: true, }); let pipe = manager.read_pipeline_data(header, None)?; @@ -237,52 +216,28 @@ fn read_pipeline_data_external_stream() -> Result<(), ShellError> { manager.consume_all()?; match pipe { - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - } => { - let stdout = stdout.expect("stdout is None"); - let stderr = stderr.expect("stderr is None"); - let exit_code = exit_code.expect("exit_code is None"); - assert_eq!(test_span, span); + PipelineData::ByteStream(stream, metadata) => { + assert_eq!(test_span, stream.span()); assert!( metadata.is_some(), "expected metadata to be Some due to prepare_pipeline_data()" ); - assert!(trim_end_newline); - assert!(!stdout.is_binary); - assert!(stderr.is_binary); - - assert_eq!( - Some((out_pattern.len() * iterations) as u64), - stdout.known_size - ); - assert_eq!(None, stderr.known_size); - - // check the streams - let mut count = 0; - for chunk in stdout.stream { - assert_eq!(out_pattern, chunk?); - count += 1; + match stream.into_source() { + ByteStreamSource::Read(mut read) => { + let mut buf = Vec::new(); + read.read_to_end(&mut buf)?; + let iter = buf.chunks_exact(out_pattern.len()); + assert_eq!(iter.len(), iterations); + for chunk in iter { + assert_eq!(out_pattern, chunk) + } + } + ByteStreamSource::File(..) => panic!("unexpected byte stream source: file"), + ByteStreamSource::Child(..) => { + panic!("unexpected byte stream source: child") + } } - assert_eq!(iterations, count, "stdout length"); - let mut count = 0; - - for chunk in stderr.stream { - assert_eq!(err_pattern, chunk?); - count += 1; - } - assert_eq!(iterations, count, "stderr length"); - - assert_eq!( - vec![Value::test_int(1)], - exit_code.into_iter().collect::>() - ); } _ => panic!("unexpected PipelineData: {pipe:?}"), } @@ -436,120 +391,51 @@ fn write_pipeline_data_list_stream() -> Result<(), ShellError> { } #[test] -fn write_pipeline_data_external_stream() -> Result<(), ShellError> { +fn write_pipeline_data_byte_stream() -> Result<(), ShellError> { let test = TestCase::new(); let manager = TestInterfaceManager::new(&test); let interface = manager.get_interface(); - let stdout_bufs = vec![ - b"hello".to_vec(), - b"world".to_vec(), - b"these are tests".to_vec(), - ]; - let stdout_len = stdout_bufs.iter().map(|b| b.len() as u64).sum::(); - let stderr_bufs = vec![b"error messages".to_vec(), b"go here".to_vec()]; - let exit_code = Value::test_int(7); - + let expected = "hello\nworld\nthese are tests"; let span = Span::new(400, 500); - // Set up pipeline data for an external stream - let pipe = PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(stdout_bufs.clone().into_iter().map(Ok)), - None, - span, - Some(stdout_len), - )), - stderr: Some(RawStream::new( - Box::new(stderr_bufs.clone().into_iter().map(Ok)), - None, - span, - None, - )), - exit_code: Some(ListStream::new( - std::iter::once(exit_code.clone()), - Span::test_data(), - None, - )), - span, - metadata: None, - trim_end_newline: true, - }; + // Set up pipeline data for a byte stream + let data = PipelineData::ByteStream( + ByteStream::read(std::io::Cursor::new(expected), span, None), + None, + ); - let (header, writer) = interface.init_write_pipeline_data(pipe, &())?; + let (header, writer) = interface.init_write_pipeline_data(data, &())?; let info = match header { - PipelineDataHeader::ExternalStream(info) => info, + PipelineDataHeader::ByteStream(info) => info, _ => panic!("unexpected header: {header:?}"), }; writer.write()?; - let stdout_info = info.stdout.as_ref().expect("stdout info is None"); - let stderr_info = info.stderr.as_ref().expect("stderr info is None"); - let exit_code_info = info.exit_code.as_ref().expect("exit code info is None"); - assert_eq!(span, info.span); - assert!(info.trim_end_newline); - - assert_eq!(Some(stdout_len), stdout_info.known_size); - assert_eq!(None, stderr_info.known_size); // Now make sure the stream messages have been written - let mut stdout_iter = stdout_bufs.into_iter(); - let mut stderr_iter = stderr_bufs.into_iter(); - let mut exit_code_iter = std::iter::once(exit_code); + let mut actual = Vec::new(); + let mut ended = false; - let mut stdout_ended = false; - let mut stderr_ended = false; - let mut exit_code_ended = false; - - // There's no specific order these messages must come in with respect to how the streams are - // interleaved, but all of the data for each stream must be in its original order, and the - // End must come after all Data for msg in test.written() { match msg { PluginOutput::Data(id, data) => { - if id == stdout_info.id { - let result: Result, ShellError> = - data.try_into().expect("wrong data in stdout stream"); - assert_eq!( - stdout_iter.next().expect("too much data in stdout"), - result.expect("unexpected error in stdout stream") - ); - } else if id == stderr_info.id { - let result: Result, ShellError> = - data.try_into().expect("wrong data in stderr stream"); - assert_eq!( - stderr_iter.next().expect("too much data in stderr"), - result.expect("unexpected error in stderr stream") - ); - } else if id == exit_code_info.id { - let code: Value = data.try_into().expect("wrong data in stderr stream"); - assert_eq!( - exit_code_iter.next().expect("too much data in stderr"), - code - ); + if id == info.id { + let data: Result, ShellError> = + data.try_into().expect("wrong data in stream"); + + let data = data.expect("unexpected error in stream"); + actual.extend(data); } else { panic!("unrecognized stream id: {id}"); } } PluginOutput::End(id) => { - if id == stdout_info.id { - assert!(!stdout_ended, "double End of stdout"); - assert!(stdout_iter.next().is_none(), "unexpected end of stdout"); - stdout_ended = true; - } else if id == stderr_info.id { - assert!(!stderr_ended, "double End of stderr"); - assert!(stderr_iter.next().is_none(), "unexpected end of stderr"); - stderr_ended = true; - } else if id == exit_code_info.id { - assert!(!exit_code_ended, "double End of exit_code"); - assert!( - exit_code_iter.next().is_none(), - "unexpected end of exit_code" - ); - exit_code_ended = true; + if id == info.id { + ended = true; } else { panic!("unrecognized stream id: {id}"); } @@ -558,9 +444,8 @@ fn write_pipeline_data_external_stream() -> Result<(), ShellError> { } } - assert!(stdout_ended, "stdout did not End"); - assert!(stderr_ended, "stderr did not End"); - assert!(exit_code_ended, "exit_code did not End"); + assert_eq!(expected.as_bytes(), actual); + assert!(ended, "stream did not End"); Ok(()) } diff --git a/crates/nu-plugin-engine/src/context.rs b/crates/nu-plugin-engine/src/context.rs index 0fb7b95b4c..0b1d56c050 100644 --- a/crates/nu-plugin-engine/src/context.rs +++ b/crates/nu-plugin-engine/src/context.rs @@ -108,7 +108,7 @@ impl<'a> PluginExecutionContext for PluginExecutionCommandContext<'a> { Value::Closure { val, .. } => { ClosureEvalOnce::new(&self.engine_state, &self.stack, *val) .run_with_input(PipelineData::Empty) - .map(|data| data.into_value(span)) + .and_then(|data| data.into_value(span)) .unwrap_or_else(|err| Value::error(err, self.call.head)) } _ => value.clone(), diff --git a/crates/nu-plugin-engine/src/init.rs b/crates/nu-plugin-engine/src/init.rs index 0ba70b49c0..198a01cd1c 100644 --- a/crates/nu-plugin-engine/src/init.rs +++ b/crates/nu-plugin-engine/src/init.rs @@ -26,7 +26,7 @@ use crate::{ /// This should be larger than the largest commonly sent message to avoid excessive fragmentation. /// -/// The buffers coming from external streams are typically each 8192 bytes, so double that. +/// The buffers coming from byte streams are typically each 8192 bytes, so double that. pub(crate) const OUTPUT_BUFFER_SIZE: usize = 16384; /// Spawn the command for a plugin, in the given `mode`. After spawning, it can be passed to diff --git a/crates/nu-plugin-engine/src/interface/mod.rs b/crates/nu-plugin-engine/src/interface/mod.rs index 3447d6a907..adab9dc68d 100644 --- a/crates/nu-plugin-engine/src/interface/mod.rs +++ b/crates/nu-plugin-engine/src/interface/mod.rs @@ -519,8 +519,8 @@ impl InterfaceManager for PluginInterfaceManager { .map_data(|data| { let ctrlc = self.get_ctrlc(id)?; - // Register the streams in the response - for stream_id in data.stream_ids() { + // Register the stream in the response + if let Some(stream_id) = data.stream_id() { self.recv_stream_started(id, stream_id); } @@ -602,7 +602,7 @@ impl InterfaceManager for PluginInterfaceManager { meta, )) } - PipelineData::Empty | PipelineData::ExternalStream { .. } => Ok(data), + PipelineData::Empty | PipelineData::ByteStream(..) => Ok(data), } } @@ -953,7 +953,7 @@ impl PluginInterface { let call = PluginCall::CustomValueOp(value.map(|cv| cv.without_source()), op); match self.plugin_call(call, None)? { - PluginCallResponse::PipelineData(out_data) => Ok(out_data.into_value(span)), + PluginCallResponse::PipelineData(out_data) => out_data.into_value(span), PluginCallResponse::Error(err) => Err(err.into()), _ => Err(ShellError::PluginFailedToDecode { msg: format!("Received unexpected response to custom value {op_name}() call"), @@ -1091,7 +1091,7 @@ impl Interface for PluginInterface { meta, )) } - PipelineData::Empty | PipelineData::ExternalStream { .. } => Ok(data), + PipelineData::Empty | PipelineData::ByteStream(..) => Ok(data), } } } diff --git a/crates/nu-plugin-engine/src/interface/tests.rs b/crates/nu-plugin-engine/src/interface/tests.rs index 7548703191..aca59a664e 100644 --- a/crates/nu-plugin-engine/src/interface/tests.rs +++ b/crates/nu-plugin-engine/src/interface/tests.rs @@ -9,10 +9,10 @@ use crate::{ use nu_plugin_core::{interface_test_util::TestCase, Interface, InterfaceManager}; use nu_plugin_protocol::{ test_util::{expected_test_custom_value, test_plugin_custom_value}, - CallInfo, CustomValueOp, EngineCall, EngineCallResponse, EvaluatedCall, ExternalStreamInfo, + ByteStreamInfo, CallInfo, CustomValueOp, EngineCall, EngineCallResponse, EvaluatedCall, ListStreamInfo, PipelineDataHeader, PluginCall, PluginCallId, PluginCallResponse, - PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, RawStreamInfo, - StreamData, StreamMessage, + PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, StreamData, + StreamMessage, }; use nu_protocol::{ ast::{Math, Operator}, @@ -154,16 +154,9 @@ fn manager_consume_all_propagates_message_error_to_readers() -> Result<(), Shell test.add(invalid_output()); let stream = manager.read_pipeline_data( - PipelineDataHeader::ExternalStream(ExternalStreamInfo { + PipelineDataHeader::ByteStream(ByteStreamInfo { + id: 0, span: Span::test_data(), - stdout: Some(RawStreamInfo { - id: 0, - is_binary: false, - known_size: None, - }), - stderr: None, - exit_code: None, - trim_end_newline: false, }), None, )?; @@ -378,7 +371,7 @@ fn manager_consume_call_response_registers_streams() -> Result<(), ShellError> { fake_plugin_call(&mut manager, n); } - // Check list streams, external streams + // Check list streams, byte streams manager.consume(PluginOutput::CallResponse( 0, PluginCallResponse::PipelineData(PipelineDataHeader::ListStream(ListStreamInfo { @@ -388,23 +381,9 @@ fn manager_consume_call_response_registers_streams() -> Result<(), ShellError> { ))?; manager.consume(PluginOutput::CallResponse( 1, - PluginCallResponse::PipelineData(PipelineDataHeader::ExternalStream(ExternalStreamInfo { + PluginCallResponse::PipelineData(PipelineDataHeader::ByteStream(ByteStreamInfo { + id: 1, span: Span::test_data(), - stdout: Some(RawStreamInfo { - id: 1, - is_binary: false, - known_size: None, - }), - stderr: Some(RawStreamInfo { - id: 2, - is_binary: false, - known_size: None, - }), - exit_code: Some(ListStreamInfo { - id: 3, - span: Span::test_data(), - }), - trim_end_newline: false, })), ))?; @@ -423,22 +402,20 @@ fn manager_consume_call_response_registers_streams() -> Result<(), ShellError> { "plugin_call_input_streams[0] should be Some(0)" ); - // ExternalStream should have three + // ByteStream should have one if let Some(sub) = manager.plugin_call_states.get(&1) { assert_eq!( - 3, sub.remaining_streams_to_read, - "ExternalStream remaining_streams_to_read should be 3" + 1, sub.remaining_streams_to_read, + "ByteStream remaining_streams_to_read should be 1" ); } else { - panic!("failed to find subscription for ExternalStream (1), maybe it was removed"); - } - for n in [1, 2, 3] { - assert_eq!( - Some(&1), - manager.plugin_call_input_streams.get(&n), - "plugin_call_input_streams[{n}] should be Some(1)" - ); + panic!("failed to find subscription for ByteStream (1), maybe it was removed"); } + assert_eq!( + Some(&1), + manager.plugin_call_input_streams.get(&1), + "plugin_call_input_streams[1] should be Some(1)" + ); Ok(()) } @@ -1087,7 +1064,7 @@ fn interface_run() -> Result<(), ShellError> { assert_eq!( Value::test_int(number), - result.into_value(Span::test_data()) + result.into_value(Span::test_data())?, ); assert!(test.has_unconsumed_write()); Ok(()) @@ -1136,7 +1113,7 @@ fn interface_prepare_pipeline_data_accepts_normal_values() -> Result<(), ShellEr match interface.prepare_pipeline_data(PipelineData::Value(value.clone(), None), &state) { Ok(data) => assert_eq!( value.get_type(), - data.into_value(Span::test_data()).get_type() + data.into_value(Span::test_data())?.get_type(), ), Err(err) => panic!("failed to accept {value:?}: {err}"), } diff --git a/crates/nu-plugin-protocol/src/lib.rs b/crates/nu-plugin-protocol/src/lib.rs index e40136ca56..ea27f82654 100644 --- a/crates/nu-plugin-protocol/src/lib.rs +++ b/crates/nu-plugin-protocol/src/lib.rs @@ -22,7 +22,7 @@ mod tests; pub mod test_util; use nu_protocol::{ - ast::Operator, engine::Closure, Config, LabeledError, PipelineData, PluginSignature, RawStream, + ast::Operator, engine::Closure, Config, LabeledError, PipelineData, PluginSignature, ShellError, Span, Spanned, Value, }; use serde::{Deserialize, Serialize}; @@ -82,32 +82,20 @@ pub enum PipelineDataHeader { /// /// Items are sent via [`StreamData`] ListStream(ListStreamInfo), - /// Initiate [`nu_protocol::PipelineData::ExternalStream`]. + /// Initiate [`nu_protocol::PipelineData::ByteStream`]. /// /// Items are sent via [`StreamData`] - ExternalStream(ExternalStreamInfo), + ByteStream(ByteStreamInfo), } impl PipelineDataHeader { - /// Return a list of stream IDs embedded in the header - pub fn stream_ids(&self) -> Vec { + /// Return the stream ID, if any, embedded in the header + pub fn stream_id(&self) -> Option { match self { - PipelineDataHeader::Empty => vec![], - PipelineDataHeader::Value(_) => vec![], - PipelineDataHeader::ListStream(info) => vec![info.id], - PipelineDataHeader::ExternalStream(info) => { - let mut out = vec![]; - if let Some(stdout) = &info.stdout { - out.push(stdout.id); - } - if let Some(stderr) = &info.stderr { - out.push(stderr.id); - } - if let Some(exit_code) = &info.exit_code { - out.push(exit_code.id); - } - out - } + PipelineDataHeader::Empty => None, + PipelineDataHeader::Value(_) => None, + PipelineDataHeader::ListStream(info) => Some(info.id), + PipelineDataHeader::ByteStream(info) => Some(info.id), } } } @@ -119,32 +107,11 @@ pub struct ListStreamInfo { pub span: Span, } -/// Additional information about external streams +/// Additional information about byte streams #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] -pub struct ExternalStreamInfo { - pub span: Span, - pub stdout: Option, - pub stderr: Option, - pub exit_code: Option, - pub trim_end_newline: bool, -} - -/// Additional information about raw (byte) streams -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] -pub struct RawStreamInfo { +pub struct ByteStreamInfo { pub id: StreamId, - pub is_binary: bool, - pub known_size: Option, -} - -impl RawStreamInfo { - pub fn new(id: StreamId, stream: &RawStream) -> Self { - RawStreamInfo { - id, - is_binary: stream.is_binary, - known_size: stream.known_size, - } - } + pub span: Span, } /// Calls that a plugin can execute. The type parameter determines the input type. @@ -380,7 +347,7 @@ impl PluginCallResponse { PipelineData::Empty => false, PipelineData::Value(..) => false, PipelineData::ListStream(..) => true, - PipelineData::ExternalStream { .. } => true, + PipelineData::ByteStream(..) => true, }, _ => false, } diff --git a/crates/nu-plugin-test-support/src/lib.rs b/crates/nu-plugin-test-support/src/lib.rs index 8aa675fd1f..caa7cbac1a 100644 --- a/crates/nu-plugin-test-support/src/lib.rs +++ b/crates/nu-plugin-test-support/src/lib.rs @@ -82,7 +82,7 @@ //! let input = vec![Value::test_string("FooBar")].into_pipeline_data(Span::test_data(), None); //! let output = PluginTest::new("lowercase", LowercasePlugin.into())? //! .eval_with("lowercase", input)? -//! .into_value(Span::test_data()); +//! .into_value(Span::test_data())?; //! //! assert_eq!( //! Value::test_list(vec![ diff --git a/crates/nu-plugin-test-support/src/plugin_test.rs b/crates/nu-plugin-test-support/src/plugin_test.rs index 18eee356ef..3d6b3eec23 100644 --- a/crates/nu-plugin-test-support/src/plugin_test.rs +++ b/crates/nu-plugin-test-support/src/plugin_test.rs @@ -93,7 +93,7 @@ impl PluginTest { /// "my-command", /// vec![Value::test_int(42)].into_pipeline_data(Span::test_data(), None) /// )? - /// .into_value(Span::test_data()); + /// .into_value(Span::test_data())?; /// assert_eq!(Value::test_string("42"), result); /// # Ok(()) /// # } @@ -136,33 +136,44 @@ impl PluginTest { // Serialize custom values in the input let source = self.source.clone(); - let input = input.map( - move |mut value| { - let result = PluginCustomValue::serialize_custom_values_in(&mut value) - // Make sure to mark them with the source so they pass correctly, too. - .and_then(|_| PluginCustomValueWithSource::add_source_in(&mut value, &source)); - match result { - Ok(()) => value, - Err(err) => Value::error(err, value.span()), - } - }, - None, - )?; + let input = if matches!(input, PipelineData::ByteStream(..)) { + input + } else { + input.map( + move |mut value| { + let result = PluginCustomValue::serialize_custom_values_in(&mut value) + // Make sure to mark them with the source so they pass correctly, too. + .and_then(|_| { + PluginCustomValueWithSource::add_source_in(&mut value, &source) + }); + match result { + Ok(()) => value, + Err(err) => Value::error(err, value.span()), + } + }, + None, + )? + }; // Eval the block with the input let mut stack = Stack::new().capture(); - eval_block::(&self.engine_state, &mut stack, &block, input)?.map( - |mut value| { - // Make sure to deserialize custom values - let result = PluginCustomValueWithSource::remove_source_in(&mut value) - .and_then(|_| PluginCustomValue::deserialize_custom_values_in(&mut value)); - match result { - Ok(()) => value, - Err(err) => Value::error(err, value.span()), - } - }, - None, - ) + let data = eval_block::(&self.engine_state, &mut stack, &block, input)?; + if matches!(data, PipelineData::ByteStream(..)) { + Ok(data) + } else { + data.map( + |mut value| { + // Make sure to deserialize custom values + let result = PluginCustomValueWithSource::remove_source_in(&mut value) + .and_then(|_| PluginCustomValue::deserialize_custom_values_in(&mut value)); + match result { + Ok(()) => value, + Err(err) => Value::error(err, value.span()), + } + }, + None, + ) + } } /// Evaluate some Nushell source code with the plugin commands in scope. @@ -176,7 +187,7 @@ impl PluginTest { /// # fn test(MyPlugin: impl Plugin + Send + 'static) -> Result<(), ShellError> { /// let result = PluginTest::new("my_plugin", MyPlugin.into())? /// .eval("42 | my-command")? - /// .into_value(Span::test_data()); + /// .into_value(Span::test_data())?; /// assert_eq!(Value::test_string("42"), result); /// # Ok(()) /// # } @@ -219,7 +230,7 @@ impl PluginTest { if let Some(expectation) = &example.result { match self.eval(example.example) { Ok(data) => { - let mut value = data.into_value(Span::test_data()); + let mut value = data.into_value(Span::test_data())?; // Set all of the spans in the value to test_data() to avoid unnecessary // differences when printing diff --git a/crates/nu-plugin-test-support/tests/custom_value/mod.rs b/crates/nu-plugin-test-support/tests/custom_value/mod.rs index aaae5538ff..f703a92e33 100644 --- a/crates/nu-plugin-test-support/tests/custom_value/mod.rs +++ b/crates/nu-plugin-test-support/tests/custom_value/mod.rs @@ -143,7 +143,7 @@ fn test_into_int_from_u32() -> Result<(), ShellError> { "into int from u32", PipelineData::Value(CustomU32(42).into_value(Span::test_data()), None), )? - .into_value(Span::test_data()); + .into_value(Span::test_data())?; assert_eq!(Value::test_int(42), result); Ok(()) } diff --git a/crates/nu-plugin-test-support/tests/hello/mod.rs b/crates/nu-plugin-test-support/tests/hello/mod.rs index 00886f1888..424940f156 100644 --- a/crates/nu-plugin-test-support/tests/hello/mod.rs +++ b/crates/nu-plugin-test-support/tests/hello/mod.rs @@ -80,7 +80,7 @@ fn test_requiring_nu_cmd_lang_commands() -> Result<(), ShellError> { let result = PluginTest::new("hello", HelloPlugin.into())? .eval("do { let greeting = hello; $greeting }")? - .into_value(Span::test_data()); + .into_value(Span::test_data())?; assert_eq!(Value::test_string("Hello, World!"), result); diff --git a/crates/nu-plugin-test-support/tests/lowercase/mod.rs b/crates/nu-plugin-test-support/tests/lowercase/mod.rs index 33446cea86..0072a08aa2 100644 --- a/crates/nu-plugin-test-support/tests/lowercase/mod.rs +++ b/crates/nu-plugin-test-support/tests/lowercase/mod.rs @@ -73,7 +73,7 @@ fn test_lowercase_using_eval_with() -> Result<(), ShellError> { assert_eq!( Value::test_list(vec![Value::test_string("hello world")]), - result.into_value(Span::test_data()) + result.into_value(Span::test_data())? ); Ok(()) diff --git a/crates/nu-plugin/src/plugin/command.rs b/crates/nu-plugin/src/plugin/command.rs index ad8ecd7d9c..5def950b0b 100644 --- a/crates/nu-plugin/src/plugin/command.rs +++ b/crates/nu-plugin/src/plugin/command.rs @@ -313,7 +313,7 @@ where // Unwrap the PipelineData from input, consuming the potential stream, and pass it to the // simpler signature in Plugin let span = input.span().unwrap_or(call.head); - let input_value = input.into_value(span); + let input_value = input.into_value(span)?; // Wrap the output in PipelineData::Value ::run(self, plugin, engine, call, &input_value) .map(|value| PipelineData::Value(value, None)) diff --git a/crates/nu-plugin/src/plugin/interface/mod.rs b/crates/nu-plugin/src/plugin/interface/mod.rs index 70e143ece5..e3e9679471 100644 --- a/crates/nu-plugin/src/plugin/interface/mod.rs +++ b/crates/nu-plugin/src/plugin/interface/mod.rs @@ -345,7 +345,7 @@ impl InterfaceManager for EngineInterfaceManager { }); Ok(PipelineData::ListStream(stream, meta)) } - PipelineData::Empty | PipelineData::ExternalStream { .. } => Ok(data), + PipelineData::Empty | PipelineData::ByteStream(..) => Ok(data), } } } @@ -850,7 +850,7 @@ impl EngineInterface { let input = input.map_or_else(|| PipelineData::Empty, |v| PipelineData::Value(v, None)); let output = self.eval_closure_with_stream(closure, positional, input, true, false)?; // Unwrap an error value - match output.into_value(closure.span) { + match output.into_value(closure.span)? { Value::Error { error, .. } => Err(*error), value => Ok(value), } @@ -920,7 +920,7 @@ impl Interface for EngineInterface { }); Ok(PipelineData::ListStream(stream, meta)) } - PipelineData::Empty | PipelineData::ExternalStream { .. } => Ok(data), + PipelineData::Empty | PipelineData::ByteStream(..) => Ok(data), } } } diff --git a/crates/nu-plugin/src/plugin/interface/tests.rs b/crates/nu-plugin/src/plugin/interface/tests.rs index 17018cbc00..ed04190712 100644 --- a/crates/nu-plugin/src/plugin/interface/tests.rs +++ b/crates/nu-plugin/src/plugin/interface/tests.rs @@ -4,10 +4,9 @@ use super::{EngineInterfaceManager, ReceivedPluginCall}; use nu_plugin_core::{interface_test_util::TestCase, Interface, InterfaceManager}; use nu_plugin_protocol::{ test_util::{expected_test_custom_value, test_plugin_custom_value, TestCustomValue}, - CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, EvaluatedCall, - ExternalStreamInfo, ListStreamInfo, PipelineDataHeader, PluginCall, PluginCallResponse, - PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, RawStreamInfo, - StreamData, + ByteStreamInfo, CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, + EvaluatedCall, ListStreamInfo, PipelineDataHeader, PluginCall, PluginCallResponse, + PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, StreamData, }; use nu_protocol::{ engine::Closure, Config, CustomValue, IntoInterruptiblePipelineData, LabeledError, @@ -158,16 +157,9 @@ fn manager_consume_all_propagates_message_error_to_readers() -> Result<(), Shell test.add(invalid_input()); let stream = manager.read_pipeline_data( - PipelineDataHeader::ExternalStream(ExternalStreamInfo { + PipelineDataHeader::ByteStream(ByteStreamInfo { + id: 0, span: Span::test_data(), - stdout: Some(RawStreamInfo { - id: 0, - is_binary: false, - known_size: None, - }), - stderr: None, - exit_code: None, - trim_end_newline: false, }), None, )?; @@ -1046,7 +1038,7 @@ fn interface_eval_closure_with_stream() -> Result<(), ShellError> { true, false, )? - .into_value(Span::test_data()); + .into_value(Span::test_data())?; assert_eq!(Value::test_int(2), result); diff --git a/crates/nu-plugin/src/plugin/mod.rs b/crates/nu-plugin/src/plugin/mod.rs index 30ed196dc6..85283aadd0 100644 --- a/crates/nu-plugin/src/plugin/mod.rs +++ b/crates/nu-plugin/src/plugin/mod.rs @@ -30,7 +30,7 @@ pub use interface::{EngineInterface, EngineInterfaceManager}; /// This should be larger than the largest commonly sent message to avoid excessive fragmentation. /// -/// The buffers coming from external streams are typically each 8192 bytes, so double that. +/// The buffers coming from byte streams are typically each 8192 bytes, so double that. #[allow(dead_code)] pub(crate) const OUTPUT_BUFFER_SIZE: usize = 16384; diff --git a/crates/nu-protocol/Cargo.toml b/crates/nu-protocol/Cargo.toml index be45738447..ae04c20ddb 100644 --- a/crates/nu-protocol/Cargo.toml +++ b/crates/nu-protocol/Cargo.toml @@ -31,6 +31,10 @@ serde = { workspace = true, default-features = false } serde_json = { workspace = true, optional = true } thiserror = "1.0" typetag = "0.2" +os_pipe = { workspace = true, features = ["io_safety"] } + +[target.'cfg(unix)'.dependencies] +nix = { workspace = true, default-features = false, features = ["signal"] } [features] plugin = [ diff --git a/crates/nu-protocol/src/debugger/debugger_trait.rs b/crates/nu-protocol/src/debugger/debugger_trait.rs index 7a842f6c28..69d395fb98 100644 --- a/crates/nu-protocol/src/debugger/debugger_trait.rs +++ b/crates/nu-protocol/src/debugger/debugger_trait.rs @@ -44,7 +44,7 @@ pub trait DebugContext: Clone + Copy + Debug { fn leave_element( engine_state: &EngineState, element: &PipelineElement, - result: &Result<(PipelineData, bool), ShellError>, + result: &Result, ) { } } @@ -77,7 +77,7 @@ impl DebugContext for WithDebug { fn leave_element( engine_state: &EngineState, element: &PipelineElement, - result: &Result<(PipelineData, bool), ShellError>, + result: &Result, ) { if let Ok(mut debugger) = engine_state.debugger.lock() { debugger @@ -128,7 +128,7 @@ pub trait Debugger: Send + Debug { &mut self, engine_state: &EngineState, element: &PipelineElement, - result: &Result<(PipelineData, bool), ShellError>, + result: &Result, ) { } diff --git a/crates/nu-protocol/src/debugger/profiler.rs b/crates/nu-protocol/src/debugger/profiler.rs index 9d5bece0ab..53b9d0555a 100644 --- a/crates/nu-protocol/src/debugger/profiler.rs +++ b/crates/nu-protocol/src/debugger/profiler.rs @@ -158,7 +158,7 @@ impl Debugger for Profiler { &mut self, _engine_state: &EngineState, element: &PipelineElement, - result: &Result<(PipelineData, bool), ShellError>, + result: &Result, ) { if self.depth > self.max_depth { return; @@ -167,12 +167,10 @@ impl Debugger for Profiler { let element_span = element.expr.span; let out_opt = self.collect_values.then(|| match result { - Ok((pipeline_data, _not_sure_what_this_is)) => match pipeline_data { + Ok(pipeline_data) => match pipeline_data { PipelineData::Value(val, ..) => val.clone(), PipelineData::ListStream(..) => Value::string("list stream", element_span), - PipelineData::ExternalStream { .. } => { - Value::string("external stream", element_span) - } + PipelineData::ByteStream(..) => Value::string("byte stream", element_span), _ => Value::nothing(element_span), }, Err(e) => Value::error(e.clone(), element_span), diff --git a/crates/nu-protocol/src/errors/shell_error.rs b/crates/nu-protocol/src/errors/shell_error.rs index e1d7ade338..525f32e925 100644 --- a/crates/nu-protocol/src/errors/shell_error.rs +++ b/crates/nu-protocol/src/errors/shell_error.rs @@ -1,5 +1,6 @@ use miette::Diagnostic; use serde::{Deserialize, Serialize}; +use std::io; use thiserror::Error; use crate::{ @@ -1374,42 +1375,79 @@ impl ShellError { } } -impl From for ShellError { - fn from(input: std::io::Error) -> ShellError { - ShellError::IOError { - msg: format!("{input:?}"), +impl From for ShellError { + fn from(error: io::Error) -> ShellError { + if error.kind() == io::ErrorKind::Other { + match error.into_inner() { + Some(err) => match err.downcast() { + Ok(err) => *err, + Err(err) => Self::IOError { + msg: err.to_string(), + }, + }, + None => Self::IOError { + msg: "unknown error".into(), + }, + } + } else { + Self::IOError { + msg: error.to_string(), + } } } } -impl From> for ShellError { - fn from(error: Spanned) -> Self { - ShellError::IOErrorSpanned { - msg: error.item.to_string(), - span: error.span, +impl From> for ShellError { + fn from(error: Spanned) -> Self { + let Spanned { item: error, span } = error; + if error.kind() == io::ErrorKind::Other { + match error.into_inner() { + Some(err) => match err.downcast() { + Ok(err) => *err, + Err(err) => Self::IOErrorSpanned { + msg: err.to_string(), + span, + }, + }, + None => Self::IOErrorSpanned { + msg: "unknown error".into(), + span, + }, + } + } else { + Self::IOErrorSpanned { + msg: error.to_string(), + span, + } } } } -impl std::convert::From> for ShellError { - fn from(input: Box) -> ShellError { +impl From for io::Error { + fn from(error: ShellError) -> Self { + io::Error::new(io::ErrorKind::Other, error) + } +} + +impl From> for ShellError { + fn from(error: Box) -> ShellError { ShellError::IOError { - msg: input.to_string(), + msg: error.to_string(), } } } impl From> for ShellError { - fn from(input: Box) -> ShellError { + fn from(error: Box) -> ShellError { ShellError::IOError { - msg: format!("{input:?}"), + msg: format!("{error:?}"), } } } impl From for ShellError { - fn from(value: super::LabeledError) -> Self { - ShellError::LabeledError(Box::new(value)) + fn from(error: super::LabeledError) -> Self { + ShellError::LabeledError(Box::new(error)) } } diff --git a/crates/nu-protocol/src/eval_const.rs b/crates/nu-protocol/src/eval_const.rs index 140a8303d9..4cc7e25324 100644 --- a/crates/nu-protocol/src/eval_const.rs +++ b/crates/nu-protocol/src/eval_const.rs @@ -317,7 +317,7 @@ impl Eval for EvalConst { ) -> Result { // TODO: Allow debugging const eval // TODO: eval.rs uses call.head for the span rather than expr.span - Ok(eval_const_call(working_set, call, PipelineData::empty())?.into_value(span)) + eval_const_call(working_set, call, PipelineData::empty())?.into_value(span) } fn eval_external_call( @@ -339,10 +339,7 @@ impl Eval for EvalConst { ) -> Result { // TODO: Allow debugging const eval let block = working_set.get_block(block_id); - Ok( - eval_const_subexpression(working_set, block, PipelineData::empty(), span)? - .into_value(span), - ) + eval_const_subexpression(working_set, block, PipelineData::empty(), span)?.into_value(span) } fn regex_match( diff --git a/crates/nu-protocol/src/lib.rs b/crates/nu-protocol/src/lib.rs index f5842b5b3a..d09186cf46 100644 --- a/crates/nu-protocol/src/lib.rs +++ b/crates/nu-protocol/src/lib.rs @@ -11,14 +11,14 @@ mod example; mod id; mod lev_distance; mod module; -mod pipeline_data; +mod pipeline; #[cfg(feature = "plugin")] mod plugin; +pub mod process; mod signature; pub mod span; mod syntax_shape; mod ty; -pub mod util; mod value; pub use alias::*; @@ -31,12 +31,11 @@ pub use example::*; pub use id::*; pub use lev_distance::levenshtein_distance; pub use module::*; -pub use pipeline_data::*; +pub use pipeline::*; #[cfg(feature = "plugin")] pub use plugin::*; pub use signature::*; pub use span::*; pub use syntax_shape::*; pub use ty::*; -pub use util::BufferedReader; pub use value::*; diff --git a/crates/nu-protocol/src/pipeline/byte_stream.rs b/crates/nu-protocol/src/pipeline/byte_stream.rs new file mode 100644 index 0000000000..f57aecacba --- /dev/null +++ b/crates/nu-protocol/src/pipeline/byte_stream.rs @@ -0,0 +1,822 @@ +use crate::{ + process::{ChildPipe, ChildProcess, ExitStatus}, + ErrSpan, IntoSpanned, OutDest, PipelineData, ShellError, Span, Value, +}; +#[cfg(unix)] +use std::os::fd::OwnedFd; +#[cfg(windows)] +use std::os::windows::io::OwnedHandle; +use std::{ + fmt::Debug, + fs::File, + io::{self, BufRead, BufReader, Cursor, ErrorKind, Read, Write}, + process::Stdio, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread, +}; + +/// The source of bytes for a [`ByteStream`]. +/// +/// Currently, there are only three possibilities: +/// 1. `Read` (any `dyn` type that implements [`Read`]) +/// 2. [`File`] +/// 3. [`ChildProcess`] +pub enum ByteStreamSource { + Read(Box), + File(File), + Child(Box), +} + +impl ByteStreamSource { + fn reader(self) -> Option { + match self { + ByteStreamSource::Read(read) => Some(SourceReader::Read(read)), + ByteStreamSource::File(file) => Some(SourceReader::File(file)), + ByteStreamSource::Child(mut child) => child.stdout.take().map(|stdout| match stdout { + ChildPipe::Pipe(pipe) => SourceReader::File(convert_file(pipe)), + ChildPipe::Tee(tee) => SourceReader::Read(tee), + }), + } + } +} + +enum SourceReader { + Read(Box), + File(File), +} + +impl Read for SourceReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + SourceReader::Read(reader) => reader.read(buf), + SourceReader::File(file) => file.read(buf), + } + } +} + +/// A potentially infinite, interruptible stream of bytes. +/// +/// The data of a [`ByteStream`] can be accessed using one of the following methods: +/// - [`reader`](ByteStream::reader): returns a [`Read`]-able type to get the raw bytes in the stream. +/// - [`lines`](ByteStream::lines): splits the bytes on lines and returns an [`Iterator`] +/// where each item is a `Result`. +/// - [`chunks`](ByteStream::chunks): returns an [`Iterator`] of [`Value`]s where each value is either a string or binary. +/// Try not to use this method if possible. Rather, please use [`reader`](ByteStream::reader) +/// (or [`lines`](ByteStream::lines) if it matches the situation). +/// +/// Additionally, there are few methods to collect a [`Bytestream`] into memory: +/// - [`into_bytes`](ByteStream::into_bytes): collects all bytes into a [`Vec`]. +/// - [`into_string`](ByteStream::into_string): collects all bytes into a [`String`], erroring if utf-8 decoding failed. +/// - [`into_value`](ByteStream::into_value): collects all bytes into a string [`Value`]. +/// If utf-8 decoding failed, then a binary [`Value`] is returned instead. +/// +/// There are also a few other methods to consume all the data of a [`Bytestream`]: +/// - [`drain`](ByteStream::drain): consumes all bytes and outputs nothing. +/// - [`write_to`](ByteStream::write_to): writes all bytes to the given [`Write`] destination. +/// - [`print`](ByteStream::print): a convenience wrapper around [`write_to`](ByteStream::write_to). +/// It prints all bytes to stdout or stderr. +/// +/// Internally, [`ByteStream`]s currently come in three flavors according to [`ByteStreamSource`]. +/// See its documentation for more information. +pub struct ByteStream { + stream: ByteStreamSource, + span: Span, + ctrlc: Option>, + known_size: Option, +} + +impl ByteStream { + /// Create a new [`ByteStream`] from a [`ByteStreamSource`]. + pub fn new(stream: ByteStreamSource, span: Span, interrupt: Option>) -> Self { + Self { + stream, + span, + ctrlc: interrupt, + known_size: None, + } + } + + /// Create a new [`ByteStream`] from a [`ByteStreamSource::Read`]. + pub fn read( + reader: impl Read + Send + 'static, + span: Span, + interrupt: Option>, + ) -> Self { + Self::new(ByteStreamSource::Read(Box::new(reader)), span, interrupt) + } + + /// Create a new [`ByteStream`] from a [`ByteStreamSource::File`]. + pub fn file(file: File, span: Span, interrupt: Option>) -> Self { + Self::new(ByteStreamSource::File(file), span, interrupt) + } + + /// Create a new [`ByteStream`] from a [`ByteStreamSource::Child`]. + pub fn child(child: ChildProcess, span: Span) -> Self { + Self::new(ByteStreamSource::Child(Box::new(child)), span, None) + } + + /// Create a new [`ByteStream`] that reads from stdin. + pub fn stdin(span: Span) -> Result { + let stdin = os_pipe::dup_stdin().err_span(span)?; + let source = ByteStreamSource::File(convert_file(stdin)); + Ok(Self::new(source, span, None)) + } + + /// Create a new [`ByteStream`] from an [`Iterator`] of bytes slices. + /// + /// The returned [`ByteStream`] will have a [`ByteStreamSource`] of `Read`. + pub fn from_iter(iter: I, span: Span, interrupt: Option>) -> Self + where + I: IntoIterator, + I::IntoIter: Send + 'static, + I::Item: AsRef<[u8]> + Default + Send + 'static, + { + let iter = iter.into_iter(); + let cursor = Some(Cursor::new(I::Item::default())); + Self::read(ReadIterator { iter, cursor }, span, interrupt) + } + + /// Create a new [`ByteStream`] from an [`Iterator`] of [`Result`] bytes slices. + /// + /// The returned [`ByteStream`] will have a [`ByteStreamSource`] of `Read`. + pub fn from_result_iter(iter: I, span: Span, interrupt: Option>) -> Self + where + I: IntoIterator>, + I::IntoIter: Send + 'static, + T: AsRef<[u8]> + Default + Send + 'static, + { + let iter = iter.into_iter(); + let cursor = Some(Cursor::new(T::default())); + Self::read(ReadResultIterator { iter, cursor }, span, interrupt) + } + + /// Set the known size, in number of bytes, of the [`ByteStream`]. + pub fn with_known_size(mut self, size: Option) -> Self { + self.known_size = size; + self + } + + /// Get a reference to the inner [`ByteStreamSource`] of the [`ByteStream`]. + pub fn source(&self) -> &ByteStreamSource { + &self.stream + } + + /// Get a mutable reference to the inner [`ByteStreamSource`] of the [`ByteStream`]. + pub fn source_mut(&mut self) -> &mut ByteStreamSource { + &mut self.stream + } + + /// Returns the [`Span`] associated with the [`ByteStream`]. + pub fn span(&self) -> Span { + self.span + } + + /// Returns the known size, in number of bytes, of the [`ByteStream`]. + pub fn known_size(&self) -> Option { + self.known_size + } + + /// Convert the [`ByteStream`] into its [`Reader`] which allows one to [`Read`] the raw bytes of the stream. + /// + /// [`Reader`] is buffered and also implements [`BufRead`]. + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`] and the child has no stdout, + /// then the stream is considered empty and `None` will be returned. + pub fn reader(self) -> Option { + let reader = self.stream.reader()?; + Some(Reader { + reader: BufReader::new(reader), + span: self.span, + ctrlc: self.ctrlc, + }) + } + + /// Convert the [`ByteStream`] into a [`Lines`] iterator where each element is a `Result`. + /// + /// There is no limit on how large each line will be. Ending new lines (`\n` or `\r\n`) are + /// stripped from each line. If a line fails to be decoded as utf-8, then it will become a [`ShellError`]. + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`] and the child has no stdout, + /// then the stream is considered empty and `None` will be returned. + pub fn lines(self) -> Option { + let reader = self.stream.reader()?; + Some(Lines { + reader: BufReader::new(reader), + span: self.span, + ctrlc: self.ctrlc, + }) + } + + /// Convert the [`ByteStream`] into a [`Chunks`] iterator where each element is a `Result`. + /// + /// Each call to [`next`](Iterator::next) reads the currently available data from the byte stream source, + /// up to a maximum size. If the chunk of bytes, or an expected portion of it, succeeds utf-8 decoding, + /// then it is returned as a [`Value::String`]. Otherwise, it is turned into a [`Value::Binary`]. + /// Any and all newlines are kept intact in each chunk. + /// + /// Where possible, prefer [`reader`](ByteStream::reader) or [`lines`](ByteStream::lines) over this method. + /// Those methods are more likely to be used in a semantically correct way + /// (and [`reader`](ByteStream::reader) is more efficient too). + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`] and the child has no stdout, + /// then the stream is considered empty and `None` will be returned. + pub fn chunks(self) -> Option { + let reader = self.stream.reader()?; + Some(Chunks { + reader: BufReader::new(reader), + span: self.span, + ctrlc: self.ctrlc, + leftover: Vec::new(), + }) + } + + /// Convert the [`ByteStream`] into its inner [`ByteStreamSource`]. + pub fn into_source(self) -> ByteStreamSource { + self.stream + } + + /// Attempt to convert the [`ByteStream`] into a [`Stdio`]. + /// + /// This will succeed if the [`ByteStreamSource`] of the [`ByteStream`] is either: + /// - [`File`](ByteStreamSource::File) + /// - [`Child`](ByteStreamSource::Child) and the child has a stdout that is `Some(ChildPipe::Pipe(..))`. + /// + /// All other cases return an `Err` with the original [`ByteStream`] in it. + pub fn into_stdio(mut self) -> Result { + match self.stream { + ByteStreamSource::Read(..) => Err(self), + ByteStreamSource::File(file) => Ok(file.into()), + ByteStreamSource::Child(child) => { + if let ChildProcess { + stdout: Some(ChildPipe::Pipe(stdout)), + stderr, + .. + } = *child + { + debug_assert!(stderr.is_none(), "stderr should not exist"); + Ok(stdout.into()) + } else { + self.stream = ByteStreamSource::Child(child); + Err(self) + } + } + } + } + + /// Attempt to convert the [`ByteStream`] into a [`ChildProcess`]. + /// + /// This will only succeed if the [`ByteStreamSource`] of the [`ByteStream`] is [`Child`](ByteStreamSource::Child). + /// All other cases return an `Err` with the original [`ByteStream`] in it. + pub fn into_child(self) -> Result { + if let ByteStreamSource::Child(child) = self.stream { + Ok(*child) + } else { + Err(self) + } + } + + /// Collect all the bytes of the [`ByteStream`] into a [`Vec`]. + /// + /// Any trailing new lines are kept in the returned [`Vec`]. + pub fn into_bytes(self) -> Result, ShellError> { + // todo!() ctrlc + match self.stream { + ByteStreamSource::Read(mut read) => { + let mut buf = Vec::new(); + read.read_to_end(&mut buf).err_span(self.span)?; + Ok(buf) + } + ByteStreamSource::File(mut file) => { + let mut buf = Vec::new(); + file.read_to_end(&mut buf).err_span(self.span)?; + Ok(buf) + } + ByteStreamSource::Child(child) => child.into_bytes(), + } + } + + /// Collect all the bytes of the [`ByteStream`] into a [`String`]. + /// + /// The trailing new line (`\n` or `\r\n`), if any, is removed from the [`String`] prior to being returned. + /// + /// If utf-8 decoding fails, an error is returned. + pub fn into_string(self) -> Result { + let span = self.span; + let bytes = self.into_bytes()?; + let mut string = String::from_utf8(bytes).map_err(|_| ShellError::NonUtf8 { span })?; + trim_end_newline(&mut string); + Ok(string) + } + + /// Collect all the bytes of the [`ByteStream`] into a [`Value`]. + /// + /// If the collected bytes are successfully decoded as utf-8, then a [`Value::String`] is returned. + /// The trailing new line (`\n` or `\r\n`), if any, is removed from the [`String`] prior to being returned. + /// Otherwise, a [`Value::Binary`] is returned with any trailing new lines preserved. + pub fn into_value(self) -> Result { + let span = self.span; + let bytes = self.into_bytes()?; + let value = match String::from_utf8(bytes) { + Ok(mut str) => { + trim_end_newline(&mut str); + Value::string(str, span) + } + Err(err) => Value::binary(err.into_bytes(), span), + }; + Ok(value) + } + + /// Consume and drop all bytes of the [`ByteStream`]. + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`], + /// then the [`ExitStatus`] of the [`ChildProcess`] is returned. + pub fn drain(self) -> Result, ShellError> { + match self.stream { + ByteStreamSource::Read(mut read) => { + copy_with_interrupt(&mut read, &mut io::sink(), self.span, self.ctrlc.as_deref())?; + Ok(None) + } + ByteStreamSource::File(_) => Ok(None), + ByteStreamSource::Child(child) => Ok(Some(child.wait()?)), + } + } + + /// Print all bytes of the [`ByteStream`] to stdout or stderr. + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`], + /// then the [`ExitStatus`] of the [`ChildProcess`] is returned. + pub fn print(self, to_stderr: bool) -> Result, ShellError> { + if to_stderr { + self.write_to(&mut io::stderr()) + } else { + self.write_to(&mut io::stdout()) + } + } + + /// Write all bytes of the [`ByteStream`] to `dest`. + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`], + /// then the [`ExitStatus`] of the [`ChildProcess`] is returned. + pub fn write_to(self, dest: &mut impl Write) -> Result, ShellError> { + let span = self.span; + let ctrlc = self.ctrlc.as_deref(); + match self.stream { + ByteStreamSource::Read(mut read) => { + copy_with_interrupt(&mut read, dest, span, ctrlc)?; + Ok(None) + } + ByteStreamSource::File(mut file) => { + copy_with_interrupt(&mut file, dest, span, ctrlc)?; + Ok(None) + } + ByteStreamSource::Child(mut child) => { + // All `OutDest`s except `OutDest::Capture` will cause `stderr` to be `None`. + // Only `save`, `tee`, and `complete` set the stderr `OutDest` to `OutDest::Capture`, + // and those commands have proper simultaneous handling of stdout and stderr. + debug_assert!(child.stderr.is_none(), "stderr should not exist"); + + if let Some(stdout) = child.stdout.take() { + match stdout { + ChildPipe::Pipe(mut pipe) => { + copy_with_interrupt(&mut pipe, dest, span, ctrlc)?; + } + ChildPipe::Tee(mut tee) => { + copy_with_interrupt(&mut tee, dest, span, ctrlc)?; + } + } + } + Ok(Some(child.wait()?)) + } + } + } + + pub(crate) fn write_to_out_dests( + self, + stdout: &OutDest, + stderr: &OutDest, + ) -> Result, ShellError> { + let span = self.span; + let ctrlc = self.ctrlc.as_deref(); + + match self.stream { + ByteStreamSource::Read(read) => { + write_to_out_dest(read, stdout, true, span, ctrlc)?; + Ok(None) + } + ByteStreamSource::File(mut file) => { + match stdout { + OutDest::Pipe | OutDest::Capture | OutDest::Null => {} + OutDest::Inherit => { + copy_with_interrupt(&mut file, &mut io::stdout(), span, ctrlc)?; + } + OutDest::File(f) => { + copy_with_interrupt(&mut file, &mut f.as_ref(), span, ctrlc)?; + } + } + Ok(None) + } + ByteStreamSource::Child(mut child) => { + match (child.stdout.take(), child.stderr.take()) { + (Some(out), Some(err)) => { + // To avoid deadlocks, we must spawn a separate thread to wait on stderr. + thread::scope(|s| { + let err_thread = thread::Builder::new() + .name("stderr writer".into()) + .spawn_scoped(s, || match err { + ChildPipe::Pipe(pipe) => { + write_to_out_dest(pipe, stderr, false, span, ctrlc) + } + ChildPipe::Tee(tee) => { + write_to_out_dest(tee, stderr, false, span, ctrlc) + } + }) + .err_span(span); + + match out { + ChildPipe::Pipe(pipe) => { + write_to_out_dest(pipe, stdout, true, span, ctrlc) + } + ChildPipe::Tee(tee) => { + write_to_out_dest(tee, stdout, true, span, ctrlc) + } + }?; + + if let Ok(result) = err_thread?.join() { + result?; + } else { + // thread panicked, which should not happen + debug_assert!(false) + } + + Ok::<_, ShellError>(()) + })?; + } + (Some(out), None) => { + // single output stream, we can consume directly + write_to_out_dest(out, stdout, true, span, ctrlc)?; + } + (None, Some(err)) => { + // single output stream, we can consume directly + write_to_out_dest(err, stderr, false, span, ctrlc)?; + } + (None, None) => {} + } + Ok(Some(child.wait()?)) + } + } + } +} + +impl Debug for ByteStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ByteStream").finish() + } +} + +impl From for PipelineData { + fn from(stream: ByteStream) -> Self { + Self::ByteStream(stream, None) + } +} + +struct ReadIterator +where + I: Iterator, + I::Item: AsRef<[u8]>, +{ + iter: I, + cursor: Option>, +} + +impl Read for ReadIterator +where + I: Iterator, + I::Item: AsRef<[u8]>, +{ + fn read(&mut self, buf: &mut [u8]) -> io::Result { + while let Some(cursor) = self.cursor.as_mut() { + let read = cursor.read(buf)?; + if read == 0 { + self.cursor = self.iter.next().map(Cursor::new); + } else { + return Ok(read); + } + } + Ok(0) + } +} + +struct ReadResultIterator +where + I: Iterator>, + T: AsRef<[u8]>, +{ + iter: I, + cursor: Option>, +} + +impl Read for ReadResultIterator +where + I: Iterator>, + T: AsRef<[u8]>, +{ + fn read(&mut self, buf: &mut [u8]) -> io::Result { + while let Some(cursor) = self.cursor.as_mut() { + let read = cursor.read(buf)?; + if read == 0 { + self.cursor = self.iter.next().transpose()?.map(Cursor::new); + } else { + return Ok(read); + } + } + Ok(0) + } +} + +pub struct Reader { + reader: BufReader, + span: Span, + ctrlc: Option>, +} + +impl Reader { + pub fn span(&self) -> Span { + self.span + } +} + +impl Read for Reader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { + Err(ShellError::InterruptedByUser { + span: Some(self.span), + } + .into()) + } else { + self.reader.read(buf) + } + } +} + +impl BufRead for Reader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.reader.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.reader.consume(amt) + } +} + +pub struct Lines { + reader: BufReader, + span: Span, + ctrlc: Option>, +} + +impl Lines { + pub fn span(&self) -> Span { + self.span + } +} + +impl Iterator for Lines { + type Item = Result; + + fn next(&mut self) -> Option { + if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { + None + } else { + let mut buf = Vec::new(); + match self.reader.read_until(b'\n', &mut buf) { + Ok(0) => None, + Ok(_) => { + let Ok(mut string) = String::from_utf8(buf) else { + return Some(Err(ShellError::NonUtf8 { span: self.span })); + }; + trim_end_newline(&mut string); + Some(Ok(string)) + } + Err(e) => Some(Err(e.into_spanned(self.span).into())), + } + } + } +} + +pub struct Chunks { + reader: BufReader, + span: Span, + ctrlc: Option>, + leftover: Vec, +} + +impl Chunks { + pub fn span(&self) -> Span { + self.span + } +} + +impl Iterator for Chunks { + type Item = Result; + + fn next(&mut self) -> Option { + if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { + None + } else { + match self.reader.fill_buf() { + Ok(buf) => { + self.leftover.extend_from_slice(buf); + let len = buf.len(); + self.reader.consume(len); + } + Err(err) => return Some(Err(err.into_spanned(self.span).into())), + }; + + if self.leftover.is_empty() { + return None; + } + + match String::from_utf8(std::mem::take(&mut self.leftover)) { + Ok(str) => Some(Ok(Value::string(str, self.span))), + Err(err) => { + if err.utf8_error().error_len().is_some() { + Some(Ok(Value::binary(err.into_bytes(), self.span))) + } else { + let i = err.utf8_error().valid_up_to(); + let mut bytes = err.into_bytes(); + self.leftover = bytes.split_off(i); + let str = String::from_utf8(bytes).expect("valid utf8"); + Some(Ok(Value::string(str, self.span))) + } + } + } + } + } +} + +fn trim_end_newline(string: &mut String) { + if string.ends_with('\n') { + string.pop(); + if string.ends_with('\r') { + string.pop(); + } + } +} + +fn write_to_out_dest( + mut read: impl Read, + stream: &OutDest, + stdout: bool, + span: Span, + ctrlc: Option<&AtomicBool>, +) -> Result<(), ShellError> { + match stream { + OutDest::Pipe | OutDest::Capture => return Ok(()), + OutDest::Null => copy_with_interrupt(&mut read, &mut io::sink(), span, ctrlc), + OutDest::Inherit if stdout => { + copy_with_interrupt(&mut read, &mut io::stdout(), span, ctrlc) + } + OutDest::Inherit => copy_with_interrupt(&mut read, &mut io::stderr(), span, ctrlc), + OutDest::File(file) => copy_with_interrupt(&mut read, &mut file.as_ref(), span, ctrlc), + }?; + Ok(()) +} + +#[cfg(unix)] +pub(crate) fn convert_file>(file: impl Into) -> T { + file.into().into() +} + +#[cfg(windows)] +pub(crate) fn convert_file>(file: impl Into) -> T { + file.into().into() +} + +const DEFAULT_BUF_SIZE: usize = 8192; + +pub fn copy_with_interrupt( + reader: &mut R, + writer: &mut W, + span: Span, + interrupt: Option<&AtomicBool>, +) -> Result +where + R: Read, + W: Write, +{ + if let Some(interrupt) = interrupt { + // #[cfg(any(target_os = "linux", target_os = "android"))] + // { + // return crate::sys::kernel_copy::copy_spec(reader, writer); + // } + match generic_copy(reader, writer, span, interrupt) { + Ok(len) => { + writer.flush().err_span(span)?; + Ok(len) + } + Err(err) => { + let _ = writer.flush(); + Err(err) + } + } + } else { + match io::copy(reader, writer) { + Ok(n) => { + writer.flush().err_span(span)?; + Ok(n) + } + Err(err) => { + let _ = writer.flush(); + Err(err.into_spanned(span).into()) + } + } + } +} + +// Copied from [`std::io::copy`] +fn generic_copy( + reader: &mut R, + writer: &mut W, + span: Span, + interrupt: &AtomicBool, +) -> Result +where + R: Read, + W: Write, +{ + let buf = &mut [0; DEFAULT_BUF_SIZE]; + let mut len = 0; + loop { + if interrupt.load(Ordering::Relaxed) { + return Err(ShellError::InterruptedByUser { span: Some(span) }); + } + let n = match reader.read(buf) { + Ok(0) => break, + Ok(n) => n, + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) => return Err(e.into_spanned(span).into()), + }; + len += n; + writer.write_all(&buf[..n]).err_span(span)?; + } + Ok(len as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_chunks(data: Vec) -> Chunks + where + T: AsRef<[u8]> + Default + Send + 'static, + { + let reader = ReadIterator { + iter: data.into_iter(), + cursor: Some(Cursor::new(T::default())), + }; + Chunks { + reader: BufReader::new(SourceReader::Read(Box::new(reader))), + span: Span::test_data(), + ctrlc: None, + leftover: Vec::new(), + } + } + + #[test] + fn chunks_read_string() { + let data = vec!["Nushell", "が好きです"]; + let chunks = test_chunks(data.clone()); + let actual = chunks.collect::, _>>().unwrap(); + let expected = data.into_iter().map(Value::test_string).collect::>(); + assert_eq!(expected, actual); + } + + #[test] + fn chunks_read_string_split_utf8() { + let expected = "Nushell最高!"; + let chunks = test_chunks(vec![&b"Nushell\xe6"[..], b"\x9c\x80\xe9", b"\xab\x98!"]); + + let actual = chunks + .into_iter() + .map(|value| value.and_then(Value::into_string)) + .collect::>() + .unwrap(); + + assert_eq!(expected, actual); + } + + #[test] + fn chunks_returns_string_or_binary() { + let chunks = test_chunks(vec![b"Nushell".as_slice(), b"\x9c\x80\xe9abcd", b"efgh"]); + let actual = chunks.collect::, _>>().unwrap(); + let expected = vec![ + Value::test_string("Nushell"), + Value::test_binary(b"\x9c\x80\xe9abcd"), + Value::test_string("efgh"), + ]; + assert_eq!(actual, expected) + } +} diff --git a/crates/nu-protocol/src/pipeline_data/list_stream.rs b/crates/nu-protocol/src/pipeline/list_stream.rs similarity index 100% rename from crates/nu-protocol/src/pipeline_data/list_stream.rs rename to crates/nu-protocol/src/pipeline/list_stream.rs diff --git a/crates/nu-protocol/src/pipeline_data/metadata.rs b/crates/nu-protocol/src/pipeline/metadata.rs similarity index 100% rename from crates/nu-protocol/src/pipeline_data/metadata.rs rename to crates/nu-protocol/src/pipeline/metadata.rs diff --git a/crates/nu-protocol/src/pipeline/mod.rs b/crates/nu-protocol/src/pipeline/mod.rs new file mode 100644 index 0000000000..a018a084ed --- /dev/null +++ b/crates/nu-protocol/src/pipeline/mod.rs @@ -0,0 +1,11 @@ +pub mod byte_stream; +pub mod list_stream; +mod metadata; +mod out_dest; +mod pipeline_data; + +pub use byte_stream::*; +pub use list_stream::*; +pub use metadata::*; +pub use out_dest::*; +pub use pipeline_data::*; diff --git a/crates/nu-protocol/src/pipeline_data/out_dest.rs b/crates/nu-protocol/src/pipeline/out_dest.rs similarity index 81% rename from crates/nu-protocol/src/pipeline_data/out_dest.rs rename to crates/nu-protocol/src/pipeline/out_dest.rs index 976123e883..69955e6b0b 100644 --- a/crates/nu-protocol/src/pipeline_data/out_dest.rs +++ b/crates/nu-protocol/src/pipeline/out_dest.rs @@ -5,17 +5,17 @@ use std::{fs::File, io, process::Stdio, sync::Arc}; pub enum OutDest { /// Redirect the stdout and/or stderr of one command as the input for the next command in the pipeline. /// - /// The output pipe will be available as the `stdout` of `PipelineData::ExternalStream`. + /// The output pipe will be available as the `stdout` of [`ChildProcess`](crate::process::ChildProcess). /// /// If stdout and stderr are both set to `Pipe`, - /// then they will combined into the `stdout` of `PipelineData::ExternalStream`. + /// then they will combined into the `stdout` of [`ChildProcess`](crate::process::ChildProcess). Pipe, /// Capture output to later be collected into a [`Value`](crate::Value), `Vec`, or used in some other way. /// - /// The output stream(s) will be available in the `stdout` or `stderr` of `PipelineData::ExternalStream`. + /// The output stream(s) will be available in the `stdout` or `stderr` of [`ChildProcess`](crate::process::ChildProcess). /// /// This is similar to `Pipe` but will never combine stdout and stderr - /// or place an external command's stderr into `stdout` of `PipelineData::ExternalStream`. + /// or place an external command's stderr into `stdout` of [`ChildProcess`](crate::process::ChildProcess). Capture, /// Ignore output. /// diff --git a/crates/nu-protocol/src/pipeline/pipeline_data.rs b/crates/nu-protocol/src/pipeline/pipeline_data.rs new file mode 100644 index 0000000000..d7e58e63a3 --- /dev/null +++ b/crates/nu-protocol/src/pipeline/pipeline_data.rs @@ -0,0 +1,725 @@ +use crate::{ + ast::{Call, PathMember}, + engine::{EngineState, Stack}, + process::{ChildPipe, ChildProcess, ExitStatus}, + ByteStream, Config, ErrSpan, ListStream, OutDest, PipelineMetadata, Range, ShellError, Span, + Value, +}; +use nu_utils::{stderr_write_all_and_flush, stdout_write_all_and_flush}; +use std::{ + io::{Cursor, Read, Write}, + sync::{atomic::AtomicBool, Arc}, +}; + +const LINE_ENDING_PATTERN: &[char] = &['\r', '\n']; + +/// The foundational abstraction for input and output to commands +/// +/// This represents either a single Value or a stream of values coming into the command or leaving a command. +/// +/// A note on implementation: +/// +/// We've tried a few variations of this structure. Listing these below so we have a record. +/// +/// * We tried always assuming a stream in Nushell. This was a great 80% solution, but it had some rough edges. +/// Namely, how do you know the difference between a single string and a list of one string. How do you know +/// when to flatten the data given to you from a data source into the stream or to keep it as an unflattened +/// list? +/// +/// * We tried putting the stream into Value. This had some interesting properties as now commands "just worked +/// on values", but lead to a few unfortunate issues. +/// +/// The first is that you can't easily clone Values in a way that felt largely immutable. For example, if +/// you cloned a Value which contained a stream, and in one variable drained some part of it, then the second +/// variable would see different values based on what you did to the first. +/// +/// To make this kind of mutation thread-safe, we would have had to produce a lock for the stream, which in +/// practice would have meant always locking the stream before reading from it. But more fundamentally, it +/// felt wrong in practice that observation of a value at runtime could affect other values which happen to +/// alias the same stream. By separating these, we don't have this effect. Instead, variables could get +/// concrete list values rather than streams, and be able to view them without non-local effects. +/// +/// * A balance of the two approaches is what we've landed on: Values are thread-safe to pass, and we can stream +/// them into any sources. Streams are still available to model the infinite streams approach of original +/// Nushell. +#[derive(Debug)] +pub enum PipelineData { + Empty, + Value(Value, Option), + ListStream(ListStream, Option), + ByteStream(ByteStream, Option), +} + +impl PipelineData { + pub fn empty() -> PipelineData { + PipelineData::Empty + } + + /// create a `PipelineData::ByteStream` with proper exit_code + /// + /// It's useful to break running without raising error at user level. + pub fn new_external_stream_with_only_exit_code(exit_code: i32) -> PipelineData { + let span = Span::unknown(); + let mut child = ChildProcess::from_raw(None, None, None, span); + child.set_exit_code(exit_code); + PipelineData::ByteStream(ByteStream::child(child, span), None) + } + + pub fn metadata(&self) -> Option { + match self { + PipelineData::Empty => None, + PipelineData::Value(_, meta) + | PipelineData::ListStream(_, meta) + | PipelineData::ByteStream(_, meta) => meta.clone(), + } + } + + pub fn set_metadata(mut self, metadata: Option) -> Self { + match &mut self { + PipelineData::Empty => {} + PipelineData::Value(_, meta) + | PipelineData::ListStream(_, meta) + | PipelineData::ByteStream(_, meta) => *meta = metadata, + } + self + } + + pub fn is_nothing(&self) -> bool { + matches!(self, PipelineData::Value(Value::Nothing { .. }, ..)) + || matches!(self, PipelineData::Empty) + } + + /// PipelineData doesn't always have a Span, but we can try! + pub fn span(&self) -> Option { + match self { + PipelineData::Empty => None, + PipelineData::Value(value, ..) => Some(value.span()), + PipelineData::ListStream(stream, ..) => Some(stream.span()), + PipelineData::ByteStream(stream, ..) => Some(stream.span()), + } + } + + pub fn into_value(self, span: Span) -> Result { + match self { + PipelineData::Empty => Ok(Value::nothing(span)), + PipelineData::Value(value, ..) => Ok(value.with_span(span)), + PipelineData::ListStream(stream, ..) => Ok(stream.into_value()), + PipelineData::ByteStream(stream, ..) => stream.into_value(), + } + } + + /// Writes all values or redirects all output to the current [`OutDest`]s in `stack`. + /// + /// For [`OutDest::Pipe`] and [`OutDest::Capture`], this will return the `PipelineData` as is + /// without consuming input and without writing anything. + /// + /// For the other [`OutDest`]s, the given `PipelineData` will be completely consumed + /// and `PipelineData::Empty` will be returned. + pub fn write_to_out_dests( + self, + engine_state: &EngineState, + stack: &mut Stack, + ) -> Result { + match (self, stack.stdout()) { + (PipelineData::ByteStream(stream, ..), stdout) => { + stream.write_to_out_dests(stdout, stack.stderr())?; + } + (data, OutDest::Pipe | OutDest::Capture) => return Ok(data), + (PipelineData::Empty, ..) => {} + (PipelineData::Value(..), OutDest::Null) => {} + (PipelineData::ListStream(stream, ..), OutDest::Null) => { + // we need to drain the stream in case there are external commands in the pipeline + stream.drain()?; + } + (PipelineData::Value(value, ..), OutDest::File(file)) => { + let bytes = value_to_bytes(value)?; + let mut file = file.as_ref(); + file.write_all(&bytes)?; + file.flush()?; + } + (PipelineData::ListStream(stream, ..), OutDest::File(file)) => { + let mut file = file.as_ref(); + // use BufWriter here? + for value in stream { + let bytes = value_to_bytes(value)?; + file.write_all(&bytes)?; + file.write_all(b"\n")?; + } + file.flush()?; + } + (data @ (PipelineData::Value(..) | PipelineData::ListStream(..)), OutDest::Inherit) => { + data.print(engine_state, stack, false, false)?; + } + } + Ok(PipelineData::Empty) + } + + pub fn drain(self) -> Result, ShellError> { + match self { + PipelineData::Empty => Ok(None), + PipelineData::Value(Value::Error { error, .. }, ..) => Err(*error), + PipelineData::Value(..) => Ok(None), + PipelineData::ListStream(stream, ..) => { + stream.drain()?; + Ok(None) + } + PipelineData::ByteStream(stream, ..) => stream.drain(), + } + } + + /// Try convert from self into iterator + /// + /// It returns Err if the `self` cannot be converted to an iterator. + pub fn into_iter_strict(self, span: Span) -> Result { + Ok(PipelineIterator(match self { + PipelineData::Value(value, ..) => { + let val_span = value.span(); + match value { + Value::List { vals, .. } => PipelineIteratorInner::ListStream( + ListStream::new(vals.into_iter(), val_span, None).into_iter(), + ), + Value::Binary { val, .. } => PipelineIteratorInner::ListStream( + ListStream::new( + val.into_iter().map(move |x| Value::int(x as i64, val_span)), + val_span, + None, + ) + .into_iter(), + ), + Value::Range { val, .. } => PipelineIteratorInner::ListStream( + ListStream::new(val.into_range_iter(val_span, None), val_span, None) + .into_iter(), + ), + // Propagate errors by explicitly matching them before the final case. + Value::Error { error, .. } => return Err(*error), + other => { + return Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary, range, or byte stream".into(), + wrong_type: other.get_type().to_string(), + dst_span: span, + src_span: val_span, + }) + } + } + } + PipelineData::ListStream(stream, ..) => { + PipelineIteratorInner::ListStream(stream.into_iter()) + } + PipelineData::Empty => { + return Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary, range, or byte stream".into(), + wrong_type: "null".into(), + dst_span: span, + src_span: span, + }) + } + PipelineData::ByteStream(stream, ..) => { + if let Some(chunks) = stream.chunks() { + PipelineIteratorInner::ByteStream(chunks) + } else { + PipelineIteratorInner::Empty + } + } + })) + } + + pub fn collect_string(self, separator: &str, config: &Config) -> Result { + match self { + PipelineData::Empty => Ok(String::new()), + PipelineData::Value(value, ..) => Ok(value.to_expanded_string(separator, config)), + PipelineData::ListStream(stream, ..) => Ok(stream.into_string(separator, config)), + PipelineData::ByteStream(stream, ..) => stream.into_string(), + } + } + + /// Retrieves string from pipeline data. + /// + /// As opposed to `collect_string` this raises error rather than converting non-string values. + /// The `span` will be used if `ListStream` is encountered since it doesn't carry a span. + pub fn collect_string_strict( + self, + span: Span, + ) -> Result<(String, Span, Option), ShellError> { + match self { + PipelineData::Empty => Ok((String::new(), span, None)), + PipelineData::Value(Value::String { val, .. }, metadata) => Ok((val, span, metadata)), + PipelineData::Value(val, ..) => Err(ShellError::TypeMismatch { + err_message: "string".into(), + span: val.span(), + }), + PipelineData::ListStream(..) => Err(ShellError::TypeMismatch { + err_message: "string".into(), + span, + }), + PipelineData::ByteStream(stream, metadata) => { + let span = stream.span(); + Ok((stream.into_string()?, span, metadata)) + } + } + } + + pub fn follow_cell_path( + self, + cell_path: &[PathMember], + head: Span, + insensitive: bool, + ) -> Result { + match self { + // FIXME: there are probably better ways of doing this + PipelineData::ListStream(stream, ..) => Value::list(stream.into_iter().collect(), head) + .follow_cell_path(cell_path, insensitive), + PipelineData::Value(v, ..) => v.follow_cell_path(cell_path, insensitive), + PipelineData::Empty => Err(ShellError::IncompatiblePathAccess { + type_name: "empty pipeline".to_string(), + span: head, + }), + PipelineData::ByteStream(stream, ..) => Err(ShellError::IncompatiblePathAccess { + type_name: "byte stream".to_string(), + span: stream.span(), + }), + } + } + + /// Simplified mapper to help with simple values also. For full iterator support use `.into_iter()` instead + pub fn map( + self, + mut f: F, + ctrlc: Option>, + ) -> Result + where + Self: Sized, + F: FnMut(Value) -> Value + 'static + Send, + { + match self { + PipelineData::Value(value, ..) => { + let span = value.span(); + match value { + Value::List { vals, .. } => { + Ok(vals.into_iter().map(f).into_pipeline_data(span, ctrlc)) + } + Value::Range { val, .. } => Ok(val + .into_range_iter(span, ctrlc.clone()) + .map(f) + .into_pipeline_data(span, ctrlc)), + value => match f(value) { + Value::Error { error, .. } => Err(*error), + v => Ok(v.into_pipeline_data()), + }, + } + } + PipelineData::Empty => Ok(PipelineData::Empty), + PipelineData::ListStream(stream, ..) => { + Ok(PipelineData::ListStream(stream.map(f), None)) + } + PipelineData::ByteStream(stream, ..) => { + // TODO: is this behavior desired / correct ? + let span = stream.span(); + match String::from_utf8(stream.into_bytes()?) { + Ok(mut str) => { + str.truncate(str.trim_end_matches(LINE_ENDING_PATTERN).len()); + Ok(f(Value::string(str, span)).into_pipeline_data()) + } + Err(err) => Ok(f(Value::binary(err.into_bytes(), span)).into_pipeline_data()), + } + } + } + } + + /// Simplified flatmapper. For full iterator support use `.into_iter()` instead + pub fn flat_map( + self, + mut f: F, + ctrlc: Option>, + ) -> Result + where + Self: Sized, + U: IntoIterator + 'static, + ::IntoIter: 'static + Send, + F: FnMut(Value) -> U + 'static + Send, + { + match self { + PipelineData::Empty => Ok(PipelineData::Empty), + PipelineData::Value(value, ..) => { + let span = value.span(); + match value { + Value::List { vals, .. } => { + Ok(vals.into_iter().flat_map(f).into_pipeline_data(span, ctrlc)) + } + Value::Range { val, .. } => Ok(val + .into_range_iter(span, ctrlc.clone()) + .flat_map(f) + .into_pipeline_data(span, ctrlc)), + value => Ok(f(value).into_iter().into_pipeline_data(span, ctrlc)), + } + } + PipelineData::ListStream(stream, ..) => { + Ok(stream.modify(|iter| iter.flat_map(f)).into()) + } + PipelineData::ByteStream(stream, ..) => { + // TODO: is this behavior desired / correct ? + let span = stream.span(); + match String::from_utf8(stream.into_bytes()?) { + Ok(mut str) => { + str.truncate(str.trim_end_matches(LINE_ENDING_PATTERN).len()); + Ok(f(Value::string(str, span)) + .into_iter() + .into_pipeline_data(span, ctrlc)) + } + Err(err) => Ok(f(Value::binary(err.into_bytes(), span)) + .into_iter() + .into_pipeline_data(span, ctrlc)), + } + } + } + } + + pub fn filter( + self, + mut f: F, + ctrlc: Option>, + ) -> Result + where + Self: Sized, + F: FnMut(&Value) -> bool + 'static + Send, + { + match self { + PipelineData::Empty => Ok(PipelineData::Empty), + PipelineData::Value(value, ..) => { + let span = value.span(); + match value { + Value::List { vals, .. } => { + Ok(vals.into_iter().filter(f).into_pipeline_data(span, ctrlc)) + } + Value::Range { val, .. } => Ok(val + .into_range_iter(span, ctrlc.clone()) + .filter(f) + .into_pipeline_data(span, ctrlc)), + value => { + if f(&value) { + Ok(value.into_pipeline_data()) + } else { + Ok(Value::nothing(span).into_pipeline_data()) + } + } + } + } + PipelineData::ListStream(stream, ..) => Ok(stream.modify(|iter| iter.filter(f)).into()), + PipelineData::ByteStream(stream, ..) => { + // TODO: is this behavior desired / correct ? + let span = stream.span(); + let value = match String::from_utf8(stream.into_bytes()?) { + Ok(mut str) => { + str.truncate(str.trim_end_matches(LINE_ENDING_PATTERN).len()); + Value::string(str, span) + } + Err(err) => Value::binary(err.into_bytes(), span), + }; + if f(&value) { + Ok(value.into_pipeline_data()) + } else { + Ok(Value::nothing(span).into_pipeline_data()) + } + } + } + } + + /// Try to catch the external command exit status and detect if it failed. + /// + /// This is useful for external commands with semicolon, we can detect errors early to avoid + /// commands after the semicolon running. + /// + /// Returns `self` and a flag that indicates if the external command run failed. If `self` is + /// not [`PipelineData::ByteStream`], the flag will be `false`. + /// + /// Currently this will consume an external command to completion. + pub fn check_external_failed(self) -> Result<(Self, bool), ShellError> { + if let PipelineData::ByteStream(stream, metadata) = self { + let span = stream.span(); + match stream.into_child() { + Ok(mut child) => { + // Only check children without stdout. This means that nothing + // later in the pipeline can possibly consume output from this external command. + if child.stdout.is_none() { + // Note: + // In run-external's implementation detail, the result sender thread + // send out stderr message first, then stdout message, then exit_code. + // + // In this clause, we already make sure that `stdout` is None + // But not the case of `stderr`, so if `stderr` is not None + // We need to consume stderr message before reading external commands' exit code. + // + // Or we'll never have a chance to read exit_code if stderr producer produce too much stderr message. + // So we consume stderr stream and rebuild it. + let stderr = child + .stderr + .take() + .map(|mut stderr| { + let mut buf = Vec::new(); + stderr.read_to_end(&mut buf).err_span(span)?; + Ok::<_, ShellError>(buf) + }) + .transpose()?; + + let code = child.wait()?.code(); + let mut child = ChildProcess::from_raw(None, None, None, span); + if let Some(stderr) = stderr { + child.stderr = Some(ChildPipe::Tee(Box::new(Cursor::new(stderr)))); + } + child.set_exit_code(code); + let stream = ByteStream::child(child, span); + Ok((PipelineData::ByteStream(stream, metadata), code != 0)) + } else { + let stream = ByteStream::child(child, span); + Ok((PipelineData::ByteStream(stream, metadata), false)) + } + } + Err(stream) => Ok((PipelineData::ByteStream(stream, metadata), false)), + } + } else { + Ok((self, false)) + } + } + + /// Try to convert Value from Value::Range to Value::List. + /// This is useful to expand Value::Range into array notation, specifically when + /// converting `to json` or `to nuon`. + /// `1..3 | to XX -> [1,2,3]` + pub fn try_expand_range(self) -> Result { + match self { + PipelineData::Value(v, metadata) => { + let span = v.span(); + match v { + Value::Range { val, .. } => { + match *val { + Range::IntRange(range) => { + if range.is_unbounded() { + return Err(ShellError::GenericError { + error: "Cannot create range".into(), + msg: "Unbounded ranges are not allowed when converting to this format".into(), + span: Some(span), + help: Some("Consider using ranges with valid start and end point.".into()), + inner: vec![], + }); + } + } + Range::FloatRange(range) => { + if range.is_unbounded() { + return Err(ShellError::GenericError { + error: "Cannot create range".into(), + msg: "Unbounded ranges are not allowed when converting to this format".into(), + span: Some(span), + help: Some("Consider using ranges with valid start and end point.".into()), + inner: vec![], + }); + } + } + } + let range_values: Vec = val.into_range_iter(span, None).collect(); + Ok(PipelineData::Value(Value::list(range_values, span), None)) + } + x => Ok(PipelineData::Value(x, metadata)), + } + } + _ => Ok(self), + } + } + + /// Consume and print self data immediately. + /// + /// `no_newline` controls if we need to attach newline character to output. + /// `to_stderr` controls if data is output to stderr, when the value is false, the data is output to stdout. + pub fn print( + self, + engine_state: &EngineState, + stack: &mut Stack, + no_newline: bool, + to_stderr: bool, + ) -> Result, ShellError> { + if let PipelineData::ByteStream(stream, ..) = self { + stream.print(to_stderr) + } else { + // If the table function is in the declarations, then we can use it + // to create the table value that will be printed in the terminal + if let Some(decl_id) = engine_state.table_decl_id { + let command = engine_state.get_decl(decl_id); + if command.get_block_id().is_some() { + self.write_all_and_flush(engine_state, no_newline, to_stderr)?; + } else { + let call = Call::new(Span::new(0, 0)); + let table = command.run(engine_state, stack, &call, self)?; + table.write_all_and_flush(engine_state, no_newline, to_stderr)?; + } + } else { + self.write_all_and_flush(engine_state, no_newline, to_stderr)?; + } + Ok(None) + } + } + + fn write_all_and_flush( + self, + engine_state: &EngineState, + no_newline: bool, + to_stderr: bool, + ) -> Result<(), ShellError> { + let config = engine_state.get_config(); + for item in self { + let mut out = if let Value::Error { error, .. } = item { + return Err(*error); + } else { + item.to_expanded_string("\n", config) + }; + + if !no_newline { + out.push('\n'); + } + + if to_stderr { + stderr_write_all_and_flush(out)? + } else { + stdout_write_all_and_flush(out)? + } + } + + Ok(()) + } +} + +enum PipelineIteratorInner { + Empty, + Value(Value), + ListStream(crate::list_stream::IntoIter), + ByteStream(crate::byte_stream::Chunks), +} + +pub struct PipelineIterator(PipelineIteratorInner); + +impl IntoIterator for PipelineData { + type Item = Value; + + type IntoIter = PipelineIterator; + + fn into_iter(self) -> Self::IntoIter { + PipelineIterator(match self { + PipelineData::Empty => PipelineIteratorInner::Empty, + PipelineData::Value(value, ..) => { + let span = value.span(); + match value { + Value::List { vals, .. } => PipelineIteratorInner::ListStream( + ListStream::new(vals.into_iter(), span, None).into_iter(), + ), + Value::Range { val, .. } => PipelineIteratorInner::ListStream( + ListStream::new(val.into_range_iter(span, None), span, None).into_iter(), + ), + x => PipelineIteratorInner::Value(x), + } + } + PipelineData::ListStream(stream, ..) => { + PipelineIteratorInner::ListStream(stream.into_iter()) + } + PipelineData::ByteStream(stream, ..) => stream.chunks().map_or( + PipelineIteratorInner::Empty, + PipelineIteratorInner::ByteStream, + ), + }) + } +} + +impl Iterator for PipelineIterator { + type Item = Value; + + fn next(&mut self) -> Option { + match &mut self.0 { + PipelineIteratorInner::Empty => None, + PipelineIteratorInner::Value(Value::Nothing { .. }, ..) => None, + PipelineIteratorInner::Value(v, ..) => Some(std::mem::take(v)), + PipelineIteratorInner::ListStream(stream, ..) => stream.next(), + PipelineIteratorInner::ByteStream(stream) => stream.next().map(|x| match x { + Ok(x) => x, + Err(err) => Value::error( + err, + Span::unknown(), //FIXME: unclear where this span should come from + ), + }), + } + } +} + +pub trait IntoPipelineData { + fn into_pipeline_data(self) -> PipelineData; + + fn into_pipeline_data_with_metadata( + self, + metadata: impl Into>, + ) -> PipelineData; +} + +impl IntoPipelineData for V +where + V: Into, +{ + fn into_pipeline_data(self) -> PipelineData { + PipelineData::Value(self.into(), None) + } + + fn into_pipeline_data_with_metadata( + self, + metadata: impl Into>, + ) -> PipelineData { + PipelineData::Value(self.into(), metadata.into()) + } +} + +pub trait IntoInterruptiblePipelineData { + fn into_pipeline_data(self, span: Span, ctrlc: Option>) -> PipelineData; + fn into_pipeline_data_with_metadata( + self, + span: Span, + ctrlc: Option>, + metadata: impl Into>, + ) -> PipelineData; +} + +impl IntoInterruptiblePipelineData for I +where + I: IntoIterator + Send + 'static, + I::IntoIter: Send + 'static, + ::Item: Into, +{ + fn into_pipeline_data(self, span: Span, ctrlc: Option>) -> PipelineData { + ListStream::new(self.into_iter().map(Into::into), span, ctrlc).into() + } + + fn into_pipeline_data_with_metadata( + self, + span: Span, + ctrlc: Option>, + metadata: impl Into>, + ) -> PipelineData { + PipelineData::ListStream( + ListStream::new(self.into_iter().map(Into::into), span, ctrlc), + metadata.into(), + ) + } +} + +fn value_to_bytes(value: Value) -> Result, ShellError> { + let bytes = match value { + Value::String { val, .. } => val.into_bytes(), + Value::Binary { val, .. } => val, + Value::List { vals, .. } => { + let val = vals + .into_iter() + .map(Value::coerce_into_string) + .collect::, ShellError>>()? + .join("\n") + + "\n"; + + val.into_bytes() + } + // Propagate errors by explicitly matching them before the final case. + Value::Error { error, .. } => return Err(*error), + value => value.coerce_into_string()?.into_bytes(), + }; + Ok(bytes) +} diff --git a/crates/nu-protocol/src/pipeline_data/mod.rs b/crates/nu-protocol/src/pipeline_data/mod.rs deleted file mode 100644 index 297eb19c55..0000000000 --- a/crates/nu-protocol/src/pipeline_data/mod.rs +++ /dev/null @@ -1,1185 +0,0 @@ -pub mod list_stream; -mod metadata; -mod out_dest; -mod raw_stream; - -pub use list_stream::{ListStream, ValueIterator}; -pub use metadata::*; -pub use out_dest::*; -pub use raw_stream::*; - -use crate::{ - ast::{Call, PathMember}, - engine::{EngineState, Stack, StateWorkingSet}, - format_error, Config, Range, ShellError, Span, Value, -}; -use nu_utils::{stderr_write_all_and_flush, stdout_write_all_and_flush}; -use std::{ - io::{self, Cursor, Read, Write}, - sync::{atomic::AtomicBool, Arc}, - thread, -}; - -const LINE_ENDING_PATTERN: &[char] = &['\r', '\n']; - -/// The foundational abstraction for input and output to commands -/// -/// This represents either a single Value or a stream of values coming into the command or leaving a command. -/// -/// A note on implementation: -/// -/// We've tried a few variations of this structure. Listing these below so we have a record. -/// -/// * We tried always assuming a stream in Nushell. This was a great 80% solution, but it had some rough edges. -/// Namely, how do you know the difference between a single string and a list of one string. How do you know -/// when to flatten the data given to you from a data source into the stream or to keep it as an unflattened -/// list? -/// -/// * We tried putting the stream into Value. This had some interesting properties as now commands "just worked -/// on values", but lead to a few unfortunate issues. -/// -/// The first is that you can't easily clone Values in a way that felt largely immutable. For example, if -/// you cloned a Value which contained a stream, and in one variable drained some part of it, then the second -/// variable would see different values based on what you did to the first. -/// -/// To make this kind of mutation thread-safe, we would have had to produce a lock for the stream, which in -/// practice would have meant always locking the stream before reading from it. But more fundamentally, it -/// felt wrong in practice that observation of a value at runtime could affect other values which happen to -/// alias the same stream. By separating these, we don't have this effect. Instead, variables could get -/// concrete list values rather than streams, and be able to view them without non-local effects. -/// -/// * A balance of the two approaches is what we've landed on: Values are thread-safe to pass, and we can stream -/// them into any sources. Streams are still available to model the infinite streams approach of original -/// Nushell. -#[derive(Debug)] -pub enum PipelineData { - Value(Value, Option), - ListStream(ListStream, Option), - ExternalStream { - stdout: Option, - stderr: Option, - exit_code: Option, - span: Span, - metadata: Option, - trim_end_newline: bool, - }, - Empty, -} - -impl PipelineData { - pub fn new_with_metadata(metadata: Option, span: Span) -> PipelineData { - PipelineData::Value(Value::nothing(span), metadata) - } - - /// create a `PipelineData::ExternalStream` with proper exit_code - /// - /// It's useful to break running without raising error at user level. - pub fn new_external_stream_with_only_exit_code(exit_code: i64) -> PipelineData { - PipelineData::ExternalStream { - stdout: None, - stderr: None, - exit_code: Some(ListStream::new( - [Value::int(exit_code, Span::unknown())].into_iter(), - Span::unknown(), - None, - )), - span: Span::unknown(), - metadata: None, - trim_end_newline: false, - } - } - - pub fn empty() -> PipelineData { - PipelineData::Empty - } - - pub fn metadata(&self) -> Option { - match self { - PipelineData::ListStream(_, x) => x.clone(), - PipelineData::ExternalStream { metadata: x, .. } => x.clone(), - PipelineData::Value(_, x) => x.clone(), - PipelineData::Empty => None, - } - } - - pub fn set_metadata(mut self, metadata: Option) -> Self { - match &mut self { - PipelineData::ListStream(_, x) => *x = metadata, - PipelineData::ExternalStream { metadata: x, .. } => *x = metadata, - PipelineData::Value(_, x) => *x = metadata, - PipelineData::Empty => {} - } - - self - } - - pub fn is_nothing(&self) -> bool { - matches!(self, PipelineData::Value(Value::Nothing { .. }, ..)) - || matches!(self, PipelineData::Empty) - } - - /// PipelineData doesn't always have a Span, but we can try! - pub fn span(&self) -> Option { - match self { - PipelineData::ListStream(stream, ..) => Some(stream.span()), - PipelineData::ExternalStream { span, .. } => Some(*span), - PipelineData::Value(v, _) => Some(v.span()), - PipelineData::Empty => None, - } - } - - pub fn into_value(self, span: Span) -> Value { - match self { - PipelineData::Empty => Value::nothing(span), - PipelineData::Value(Value::Nothing { .. }, ..) => Value::nothing(span), - PipelineData::Value(v, ..) => v.with_span(span), - PipelineData::ListStream(s, ..) => Value::list( - s.into_iter().collect(), - span, // FIXME? - ), - PipelineData::ExternalStream { - stdout: None, - exit_code, - .. - } => { - // Make sure everything has finished - if let Some(exit_code) = exit_code { - let _: Vec<_> = exit_code.into_iter().collect(); - } - Value::nothing(span) - } - PipelineData::ExternalStream { - stdout: Some(mut s), - exit_code, - trim_end_newline, - .. - } => { - let mut items = vec![]; - - for val in &mut s { - match val { - Ok(val) => { - items.push(val); - } - Err(e) => { - return Value::error(e, span); - } - } - } - - // Make sure everything has finished - if let Some(exit_code) = exit_code { - let _: Vec<_> = exit_code.into_iter().collect(); - } - - // NOTE: currently trim-end-newline only handles for string output. - // For binary, user might need origin data. - if s.is_binary { - let mut output = vec![]; - for item in items { - match item.coerce_into_binary() { - Ok(item) => { - output.extend(item); - } - Err(err) => { - return Value::error(err, span); - } - } - } - - Value::binary( - output, span, // FIXME? - ) - } else { - let mut output = String::new(); - for item in items { - match item.coerce_into_string() { - Ok(s) => output.push_str(&s), - Err(err) => { - return Value::error(err, span); - } - } - } - if trim_end_newline { - output.truncate(output.trim_end_matches(LINE_ENDING_PATTERN).len()) - } - Value::string( - output, span, // FIXME? - ) - } - } - } - } - - /// Writes all values or redirects all output to the current [`OutDest`]s in `stack`. - /// - /// For [`OutDest::Pipe`] and [`OutDest::Capture`], this will return the `PipelineData` as is - /// without consuming input and without writing anything. - /// - /// For the other [`OutDest`]s, the given `PipelineData` will be completely consumed - /// and `PipelineData::Empty` will be returned. - pub fn write_to_out_dests( - self, - engine_state: &EngineState, - stack: &mut Stack, - ) -> Result { - match (self, stack.stdout()) { - ( - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - }, - _, - ) => { - fn needs_redirect( - stream: Option, - out_dest: &OutDest, - ) -> Result> { - match (stream, out_dest) { - (Some(stream), OutDest::Pipe | OutDest::Capture) => Err(Some(stream)), - (Some(stream), _) => Ok(stream), - (None, _) => Err(None), - } - } - - let (stdout, stderr) = match ( - needs_redirect(stdout, stack.stdout()), - needs_redirect(stderr, stack.stderr()), - ) { - (Ok(stdout), Ok(stderr)) => { - // We need to redirect both stdout and stderr - - // To avoid deadlocks, we must spawn a separate thread to wait on stderr. - let err_thread = { - let err = stack.stderr().clone(); - std::thread::Builder::new() - .spawn(move || consume_child_output(stderr, &err)) - }; - - consume_child_output(stdout, stack.stdout())?; - - match err_thread?.join() { - Ok(result) => result?, - Err(err) => { - return Err(ShellError::GenericError { - error: "Error consuming external command stderr".into(), - msg: format! {"{err:?}"}, - span: Some(span), - help: None, - inner: Vec::new(), - }) - } - } - - (None, None) - } - (Ok(stdout), Err(stderr)) => { - // single output stream, we can consume directly - consume_child_output(stdout, stack.stdout())?; - (None, stderr) - } - (Err(stdout), Ok(stderr)) => { - // single output stream, we can consume directly - consume_child_output(stderr, stack.stderr())?; - (stdout, None) - } - (Err(stdout), Err(stderr)) => (stdout, stderr), - }; - - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - }) - } - (data, OutDest::Pipe | OutDest::Capture) => Ok(data), - (PipelineData::Empty, _) => Ok(PipelineData::Empty), - (PipelineData::Value(_, _), OutDest::Null) => Ok(PipelineData::Empty), - (PipelineData::ListStream(stream, _), OutDest::Null) => { - // we need to drain the stream in case there are external commands in the pipeline - stream.drain()?; - Ok(PipelineData::Empty) - } - (PipelineData::Value(value, _), OutDest::File(file)) => { - let bytes = value_to_bytes(value)?; - let mut file = file.try_clone()?; - file.write_all(&bytes)?; - file.flush()?; - Ok(PipelineData::Empty) - } - (PipelineData::ListStream(stream, _), OutDest::File(file)) => { - let mut file = file.try_clone()?; - // use BufWriter here? - for value in stream { - let bytes = value_to_bytes(value)?; - file.write_all(&bytes)?; - file.write_all(b"\n")?; - } - file.flush()?; - Ok(PipelineData::Empty) - } - ( - data @ (PipelineData::Value(_, _) | PipelineData::ListStream(_, _)), - OutDest::Inherit, - ) => { - let config = engine_state.get_config(); - - if let Some(decl_id) = engine_state.table_decl_id { - let command = engine_state.get_decl(decl_id); - if command.get_block_id().is_some() { - data.write_all_and_flush(engine_state, config, false, false)?; - } else { - let call = Call::new(Span::unknown()); - let stack = &mut stack.start_capture(); - let table = command.run(engine_state, stack, &call, data)?; - table.write_all_and_flush(engine_state, config, false, false)?; - } - } else { - data.write_all_and_flush(engine_state, config, false, false)?; - }; - Ok(PipelineData::Empty) - } - } - } - - pub fn drain(self) -> Result<(), ShellError> { - match self { - PipelineData::Value(Value::Error { error, .. }, _) => Err(*error), - PipelineData::Value(_, _) => Ok(()), - PipelineData::ListStream(stream, _) => stream.drain(), - PipelineData::ExternalStream { stdout, stderr, .. } => { - if let Some(stdout) = stdout { - stdout.drain()?; - } - - if let Some(stderr) = stderr { - stderr.drain()?; - } - - Ok(()) - } - PipelineData::Empty => Ok(()), - } - } - - pub fn drain_with_exit_code(self) -> Result { - match self { - PipelineData::Value(Value::Error { error, .. }, _) => Err(*error), - PipelineData::Value(_, _) => Ok(0), - PipelineData::ListStream(stream, _) => { - stream.drain()?; - Ok(0) - } - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - .. - } => { - if let Some(stdout) = stdout { - stdout.drain()?; - } - - if let Some(stderr) = stderr { - stderr.drain()?; - } - - if let Some(exit_code) = exit_code { - let result = drain_exit_code(exit_code)?; - Ok(result) - } else { - Ok(0) - } - } - PipelineData::Empty => Ok(0), - } - } - - /// Try convert from self into iterator - /// - /// It returns Err if the `self` cannot be converted to an iterator. - pub fn into_iter_strict(self, span: Span) -> Result { - Ok(PipelineIterator(match self { - PipelineData::Value(value, ..) => { - let val_span = value.span(); - match value { - Value::List { vals, .. } => PipelineIteratorInner::ListStream( - ListStream::new(vals.into_iter(), val_span, None).into_iter(), - ), - Value::Binary { val, .. } => PipelineIteratorInner::ListStream( - ListStream::new( - val.into_iter().map(move |x| Value::int(x as i64, val_span)), - val_span, - None, - ) - .into_iter(), - ), - Value::Range { ref val, .. } => PipelineIteratorInner::ListStream( - ListStream::new(val.into_range_iter(value.span(), None), val_span, None) - .into_iter(), - ), - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => return Err(*error), - other => { - return Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary, raw data or range".into(), - wrong_type: other.get_type().to_string(), - dst_span: span, - src_span: val_span, - }) - } - } - } - PipelineData::ListStream(stream, ..) => { - PipelineIteratorInner::ListStream(stream.into_iter()) - } - PipelineData::Empty => { - return Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary, raw data or range".into(), - wrong_type: "null".into(), - dst_span: span, - src_span: span, - }) - } - PipelineData::ExternalStream { - stdout: Some(stdout), - .. - } => PipelineIteratorInner::ExternalStream(stdout), - PipelineData::ExternalStream { stdout: None, .. } => PipelineIteratorInner::Empty, - })) - } - - pub fn collect_string(self, separator: &str, config: &Config) -> Result { - match self { - PipelineData::Empty => Ok(String::new()), - PipelineData::Value(v, ..) => Ok(v.to_expanded_string(separator, config)), - PipelineData::ListStream(s, ..) => Ok(s.into_string(separator, config)), - PipelineData::ExternalStream { stdout: None, .. } => Ok(String::new()), - PipelineData::ExternalStream { - stdout: Some(s), - trim_end_newline, - .. - } => { - let mut output = String::new(); - - for val in s { - output.push_str(&val?.coerce_into_string()?); - } - if trim_end_newline { - output.truncate(output.trim_end_matches(LINE_ENDING_PATTERN).len()); - } - Ok(output) - } - } - } - - /// Retrieves string from pipeline data. - /// - /// As opposed to `collect_string` this raises error rather than converting non-string values. - /// The `span` will be used if `ListStream` is encountered since it doesn't carry a span. - pub fn collect_string_strict( - self, - span: Span, - ) -> Result<(String, Span, Option), ShellError> { - match self { - PipelineData::Empty => Ok((String::new(), span, None)), - PipelineData::Value(Value::String { val, .. }, metadata) => Ok((val, span, metadata)), - PipelineData::Value(val, _) => Err(ShellError::TypeMismatch { - err_message: "string".into(), - span: val.span(), - }), - PipelineData::ListStream(_, _) => Err(ShellError::TypeMismatch { - err_message: "string".into(), - span, - }), - PipelineData::ExternalStream { - stdout: None, - metadata, - span, - .. - } => Ok((String::new(), span, metadata)), - PipelineData::ExternalStream { - stdout: Some(stdout), - metadata, - span, - .. - } => Ok((stdout.into_string()?.item, span, metadata)), - } - } - - pub fn follow_cell_path( - self, - cell_path: &[PathMember], - head: Span, - insensitive: bool, - ) -> Result { - match self { - // FIXME: there are probably better ways of doing this - PipelineData::ListStream(stream, ..) => Value::list(stream.into_iter().collect(), head) - .follow_cell_path(cell_path, insensitive), - PipelineData::Value(v, ..) => v.follow_cell_path(cell_path, insensitive), - PipelineData::Empty => Err(ShellError::IncompatiblePathAccess { - type_name: "empty pipeline".to_string(), - span: head, - }), - PipelineData::ExternalStream { span, .. } => Err(ShellError::IncompatiblePathAccess { - type_name: "external stream".to_string(), - span, - }), - } - } - - /// Simplified mapper to help with simple values also. For full iterator support use `.into_iter()` instead - pub fn map( - self, - mut f: F, - ctrlc: Option>, - ) -> Result - where - Self: Sized, - F: FnMut(Value) -> Value + 'static + Send, - { - match self { - PipelineData::Value(value, ..) => { - let span = value.span(); - match value { - Value::List { vals, .. } => { - Ok(vals.into_iter().map(f).into_pipeline_data(span, ctrlc)) - } - Value::Range { val, .. } => Ok(val - .into_range_iter(span, ctrlc.clone()) - .map(f) - .into_pipeline_data(span, ctrlc)), - value => match f(value) { - Value::Error { error, .. } => Err(*error), - v => Ok(v.into_pipeline_data()), - }, - } - } - PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::ListStream(stream, ..) => { - Ok(PipelineData::ListStream(stream.map(f), None)) - } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - trim_end_newline, - .. - } => { - let collected = stream.into_bytes()?; - - if let Ok(mut st) = String::from_utf8(collected.clone().item) { - if trim_end_newline { - st.truncate(st.trim_end_matches(LINE_ENDING_PATTERN).len()); - } - Ok(f(Value::string(st, collected.span)).into_pipeline_data()) - } else { - Ok(f(Value::binary(collected.item, collected.span)).into_pipeline_data()) - } - } - } - } - - /// Simplified flatmapper. For full iterator support use `.into_iter()` instead - pub fn flat_map( - self, - mut f: F, - ctrlc: Option>, - ) -> Result - where - Self: Sized, - U: IntoIterator + 'static, - ::IntoIter: 'static + Send, - F: FnMut(Value) -> U + 'static + Send, - { - match self { - PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::Value(value, ..) => { - let span = value.span(); - match value { - Value::List { vals, .. } => { - Ok(vals.into_iter().flat_map(f).into_pipeline_data(span, ctrlc)) - } - Value::Range { val, .. } => Ok(val - .into_range_iter(span, ctrlc.clone()) - .flat_map(f) - .into_pipeline_data(span, ctrlc)), - value => Ok(f(value).into_iter().into_pipeline_data(span, ctrlc)), - } - } - PipelineData::ListStream(stream, ..) => { - Ok(stream.modify(|iter| iter.flat_map(f)).into()) - } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::Empty), - PipelineData::ExternalStream { - stdout: Some(stream), - span, - trim_end_newline, - .. - } => { - let collected = stream.into_bytes()?; - - if let Ok(mut st) = String::from_utf8(collected.clone().item) { - if trim_end_newline { - st.truncate(st.trim_end_matches(LINE_ENDING_PATTERN).len()) - } - Ok(f(Value::string(st, collected.span)) - .into_iter() - .into_pipeline_data(span, ctrlc)) - } else { - Ok(f(Value::binary(collected.item, collected.span)) - .into_iter() - .into_pipeline_data(span, ctrlc)) - } - } - } - } - - pub fn filter( - self, - mut f: F, - ctrlc: Option>, - ) -> Result - where - Self: Sized, - F: FnMut(&Value) -> bool + 'static + Send, - { - match self { - PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::Value(value, ..) => { - let span = value.span(); - match value { - Value::List { vals, .. } => { - Ok(vals.into_iter().filter(f).into_pipeline_data(span, ctrlc)) - } - Value::Range { val, .. } => Ok(val - .into_range_iter(span, ctrlc.clone()) - .filter(f) - .into_pipeline_data(span, ctrlc)), - value => { - if f(&value) { - Ok(value.into_pipeline_data()) - } else { - Ok(Value::nothing(span).into_pipeline_data()) - } - } - } - } - PipelineData::ListStream(stream, ..) => Ok(stream.modify(|iter| iter.filter(f)).into()), - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::Empty), - PipelineData::ExternalStream { - stdout: Some(stream), - trim_end_newline, - .. - } => { - let collected = stream.into_bytes()?; - - if let Ok(mut st) = String::from_utf8(collected.clone().item) { - if trim_end_newline { - st.truncate(st.trim_end_matches(LINE_ENDING_PATTERN).len()) - } - let v = Value::string(st, collected.span); - - if f(&v) { - Ok(v.into_pipeline_data()) - } else { - Ok(PipelineData::new_with_metadata(None, collected.span)) - } - } else { - let v = Value::binary(collected.item, collected.span); - - if f(&v) { - Ok(v.into_pipeline_data()) - } else { - Ok(PipelineData::new_with_metadata(None, collected.span)) - } - } - } - } - } - - /// Try to catch the external stream exit status and detect if it failed. - /// - /// This is useful for external commands with semicolon, we can detect errors early to avoid - /// commands after the semicolon running. - /// - /// Returns `self` and a flag that indicates if the external stream run failed. If `self` is - /// not [`PipelineData::ExternalStream`], the flag will be `false`. - /// - /// Currently this will consume an external stream to completion. - pub fn check_external_failed(self) -> (Self, bool) { - let mut failed_to_run = false; - // Only need ExternalStream without redirecting output. - // It indicates we have no more commands to execute currently. - if let PipelineData::ExternalStream { - stdout: None, - stderr, - mut exit_code, - span, - metadata, - trim_end_newline, - } = self - { - let exit_code = exit_code.take(); - - // Note: - // In run-external's implementation detail, the result sender thread - // send out stderr message first, then stdout message, then exit_code. - // - // In this clause, we already make sure that `stdout` is None - // But not the case of `stderr`, so if `stderr` is not None - // We need to consume stderr message before reading external commands' exit code. - // - // Or we'll never have a chance to read exit_code if stderr producer produce too much stderr message. - // So we consume stderr stream and rebuild it. - let stderr = stderr.map(|stderr_stream| { - let stderr_ctrlc = stderr_stream.ctrlc.clone(); - let stderr_span = stderr_stream.span; - let stderr_bytes = stderr_stream - .into_bytes() - .map(|bytes| bytes.item) - .unwrap_or_default(); - RawStream::new( - Box::new(std::iter::once(Ok(stderr_bytes))), - stderr_ctrlc, - stderr_span, - None, - ) - }); - - match exit_code { - Some(exit_code_stream) => { - let exit_code: Vec = exit_code_stream.into_iter().collect(); - if let Some(Value::Int { val: code, .. }) = exit_code.last() { - // if exit_code is not 0, it indicates error occurred, return back Err. - if *code != 0 { - failed_to_run = true; - } - } - ( - PipelineData::ExternalStream { - stdout: None, - stderr, - exit_code: Some(ListStream::new(exit_code.into_iter(), span, None)), - span, - metadata, - trim_end_newline, - }, - failed_to_run, - ) - } - None => ( - PipelineData::ExternalStream { - stdout: None, - stderr, - exit_code: None, - span, - metadata, - trim_end_newline, - }, - failed_to_run, - ), - } - } else { - (self, false) - } - } - /// Try to convert Value from Value::Range to Value::List. - /// This is useful to expand Value::Range into array notation, specifically when - /// converting `to json` or `to nuon`. - /// `1..3 | to XX -> [1,2,3]` - pub fn try_expand_range(self) -> Result { - match self { - PipelineData::Value(v, metadata) => { - let span = v.span(); - match v { - Value::Range { val, .. } => { - match *val { - Range::IntRange(range) => { - if range.is_unbounded() { - return Err(ShellError::GenericError { - error: "Cannot create range".into(), - msg: "Unbounded ranges are not allowed when converting to this format".into(), - span: Some(span), - help: Some("Consider using ranges with valid start and end point.".into()), - inner: vec![], - }); - } - } - Range::FloatRange(range) => { - if range.is_unbounded() { - return Err(ShellError::GenericError { - error: "Cannot create range".into(), - msg: "Unbounded ranges are not allowed when converting to this format".into(), - span: Some(span), - help: Some("Consider using ranges with valid start and end point.".into()), - inner: vec![], - }); - } - } - } - let range_values: Vec = val.into_range_iter(span, None).collect(); - Ok(PipelineData::Value(Value::list(range_values, span), None)) - } - x => Ok(PipelineData::Value(x, metadata)), - } - } - _ => Ok(self), - } - } - - /// Consume and print self data immediately. - /// - /// `no_newline` controls if we need to attach newline character to output. - /// `to_stderr` controls if data is output to stderr, when the value is false, the data is output to stdout. - pub fn print( - self, - engine_state: &EngineState, - stack: &mut Stack, - no_newline: bool, - to_stderr: bool, - ) -> Result { - // If the table function is in the declarations, then we can use it - // to create the table value that will be printed in the terminal - - let config = engine_state.get_config(); - - if let PipelineData::ExternalStream { - stdout: stream, - stderr: stderr_stream, - exit_code, - .. - } = self - { - return print_if_stream(stream, stderr_stream, to_stderr, exit_code); - } - - if let Some(decl_id) = engine_state.table_decl_id { - let command = engine_state.get_decl(decl_id); - if command.get_block_id().is_some() { - return self.write_all_and_flush(engine_state, config, no_newline, to_stderr); - } - - let call = Call::new(Span::new(0, 0)); - let table = command.run(engine_state, stack, &call, self)?; - table.write_all_and_flush(engine_state, config, no_newline, to_stderr)?; - } else { - self.write_all_and_flush(engine_state, config, no_newline, to_stderr)?; - }; - - Ok(0) - } - - fn write_all_and_flush( - self, - engine_state: &EngineState, - config: &Config, - no_newline: bool, - to_stderr: bool, - ) -> Result { - for item in self { - let mut is_err = false; - let mut out = if let Value::Error { error, .. } = item { - let working_set = StateWorkingSet::new(engine_state); - // Value::Errors must always go to stderr, not stdout. - is_err = true; - format_error(&working_set, &*error) - } else if no_newline { - item.to_expanded_string("", config) - } else { - item.to_expanded_string("\n", config) - }; - - if !no_newline { - out.push('\n'); - } - - if !to_stderr && !is_err { - stdout_write_all_and_flush(out)? - } else { - stderr_write_all_and_flush(out)? - } - } - - Ok(0) - } -} - -enum PipelineIteratorInner { - Empty, - Value(Value), - ListStream(list_stream::IntoIter), - ExternalStream(RawStream), -} - -pub struct PipelineIterator(PipelineIteratorInner); - -impl IntoIterator for PipelineData { - type Item = Value; - - type IntoIter = PipelineIterator; - - fn into_iter(self) -> Self::IntoIter { - PipelineIterator(match self { - PipelineData::Value(value, ..) => { - let span = value.span(); - match value { - Value::List { vals, .. } => PipelineIteratorInner::ListStream( - ListStream::new(vals.into_iter(), span, None).into_iter(), - ), - Value::Range { val, .. } => PipelineIteratorInner::ListStream( - ListStream::new(val.into_range_iter(span, None), span, None).into_iter(), - ), - x => PipelineIteratorInner::Value(x), - } - } - PipelineData::ListStream(stream, ..) => { - PipelineIteratorInner::ListStream(stream.into_iter()) - } - PipelineData::ExternalStream { - stdout: Some(stdout), - .. - } => PipelineIteratorInner::ExternalStream(stdout), - PipelineData::ExternalStream { stdout: None, .. } => PipelineIteratorInner::Empty, - PipelineData::Empty => PipelineIteratorInner::Empty, - }) - } -} - -pub fn print_if_stream( - stream: Option, - stderr_stream: Option, - to_stderr: bool, - exit_code: Option, -) -> Result { - if let Some(stderr_stream) = stderr_stream { - thread::Builder::new() - .name("stderr consumer".to_string()) - .spawn(move || { - let RawStream { - stream, - leftover, - ctrlc, - .. - } = stderr_stream; - let mut stderr = std::io::stderr(); - let _ = stderr.write_all(&leftover); - drop(leftover); - for bytes in stream { - if nu_utils::ctrl_c::was_pressed(&ctrlc) { - break; - } - match bytes { - Ok(bytes) => { - let _ = stderr.write_all(&bytes); - } - Err(err) => { - // we don't have access to EngineState, but maybe logging the debug - // impl is better than nothing - eprintln!("Error in stderr stream: {err:?}"); - break; - } - } - } - })?; - } - - if let Some(stream) = stream { - for s in stream { - let s_live = s?; - let bin_output = s_live.coerce_into_binary()?; - - if !to_stderr { - stdout_write_all_and_flush(&bin_output)? - } else { - stderr_write_all_and_flush(&bin_output)? - } - } - } - - // Make sure everything has finished - if let Some(exit_code) = exit_code { - return drain_exit_code(exit_code); - } - - Ok(0) -} - -fn drain_exit_code(exit_code: ListStream) -> Result { - let mut exit_codes: Vec<_> = exit_code.into_iter().collect(); - match exit_codes.pop() { - #[cfg(unix)] - Some(Value::Error { error, .. }) => Err(*error), - Some(Value::Int { val, .. }) => Ok(val), - _ => Ok(0), - } -} - -/// Only call this if `output_stream` is not `OutDest::Pipe` or `OutDest::Capture`. -fn consume_child_output(child_output: RawStream, output_stream: &OutDest) -> io::Result<()> { - let mut output = ReadRawStream::new(child_output); - match output_stream { - OutDest::Pipe | OutDest::Capture => { - // The point of `consume_child_output` is to redirect output *right now*, - // but OutDest::Pipe means to redirect output - // into an OS pipe for *future use* (as input for another command). - // So, this branch makes no sense, and will simply drop `output` instead of draining it. - // This could trigger a `SIGPIPE` for the external command, - // since there will be no reader for its pipe. - debug_assert!(false) - } - OutDest::Null => { - io::copy(&mut output, &mut io::sink())?; - } - OutDest::Inherit => { - io::copy(&mut output, &mut io::stdout())?; - } - OutDest::File(file) => { - io::copy(&mut output, &mut file.try_clone()?)?; - } - } - Ok(()) -} - -impl Iterator for PipelineIterator { - type Item = Value; - - fn next(&mut self) -> Option { - match &mut self.0 { - PipelineIteratorInner::Empty => None, - PipelineIteratorInner::Value(Value::Nothing { .. }, ..) => None, - PipelineIteratorInner::Value(v, ..) => Some(std::mem::take(v)), - PipelineIteratorInner::ListStream(stream, ..) => stream.next(), - PipelineIteratorInner::ExternalStream(stream) => stream.next().map(|x| match x { - Ok(x) => x, - Err(err) => Value::error( - err, - Span::unknown(), //FIXME: unclear where this span should come from - ), - }), - } - } -} - -pub trait IntoPipelineData { - fn into_pipeline_data(self) -> PipelineData; - - fn into_pipeline_data_with_metadata( - self, - metadata: impl Into>, - ) -> PipelineData; -} - -impl IntoPipelineData for V -where - V: Into, -{ - fn into_pipeline_data(self) -> PipelineData { - PipelineData::Value(self.into(), None) - } - - fn into_pipeline_data_with_metadata( - self, - metadata: impl Into>, - ) -> PipelineData { - PipelineData::Value(self.into(), metadata.into()) - } -} - -pub trait IntoInterruptiblePipelineData { - fn into_pipeline_data(self, span: Span, ctrlc: Option>) -> PipelineData; - fn into_pipeline_data_with_metadata( - self, - span: Span, - ctrlc: Option>, - metadata: impl Into>, - ) -> PipelineData; -} - -impl IntoInterruptiblePipelineData for I -where - I: IntoIterator + Send + 'static, - I::IntoIter: Send + 'static, - ::Item: Into, -{ - fn into_pipeline_data(self, span: Span, ctrlc: Option>) -> PipelineData { - ListStream::new(self.into_iter().map(Into::into), span, ctrlc).into() - } - - fn into_pipeline_data_with_metadata( - self, - span: Span, - ctrlc: Option>, - metadata: impl Into>, - ) -> PipelineData { - PipelineData::ListStream( - ListStream::new(self.into_iter().map(Into::into), span, ctrlc), - metadata.into(), - ) - } -} - -fn value_to_bytes(value: Value) -> Result, ShellError> { - let bytes = match value { - Value::String { val, .. } => val.into_bytes(), - Value::Binary { val, .. } => val, - Value::List { vals, .. } => { - let val = vals - .into_iter() - .map(Value::coerce_into_string) - .collect::, ShellError>>()? - .join("\n") - + "\n"; - - val.into_bytes() - } - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => return Err(*error), - value => value.coerce_into_string()?.into_bytes(), - }; - Ok(bytes) -} - -struct ReadRawStream { - iter: Box, ShellError>>>, - cursor: Option>>, -} - -impl ReadRawStream { - fn new(stream: RawStream) -> Self { - debug_assert!(stream.leftover.is_empty()); - Self { - iter: stream.stream, - cursor: Some(Cursor::new(Vec::new())), - } - } -} - -impl Read for ReadRawStream { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - while let Some(cursor) = self.cursor.as_mut() { - let read = cursor.read(buf)?; - if read > 0 { - return Ok(read); - } else { - match self.iter.next().transpose() { - Ok(next) => { - self.cursor = next.map(Cursor::new); - } - Err(err) => { - // temporary hack - return Err(io::Error::new(io::ErrorKind::Other, err)); - } - } - } - } - Ok(0) - } -} diff --git a/crates/nu-protocol/src/pipeline_data/raw_stream.rs b/crates/nu-protocol/src/pipeline_data/raw_stream.rs deleted file mode 100644 index 846cdd772b..0000000000 --- a/crates/nu-protocol/src/pipeline_data/raw_stream.rs +++ /dev/null @@ -1,176 +0,0 @@ -use crate::*; -use std::{ - fmt::Debug, - sync::{atomic::AtomicBool, Arc}, -}; - -pub struct RawStream { - pub stream: Box, ShellError>> + Send + 'static>, - pub leftover: Vec, - pub ctrlc: Option>, - pub is_binary: bool, - pub span: Span, - pub known_size: Option, // (bytes) -} - -impl RawStream { - pub fn new( - stream: Box, ShellError>> + Send + 'static>, - ctrlc: Option>, - span: Span, - known_size: Option, - ) -> Self { - Self { - stream, - leftover: vec![], - ctrlc, - is_binary: false, - span, - known_size, - } - } - - pub fn into_bytes(self) -> Result>, ShellError> { - let mut output = vec![]; - - for item in self.stream { - if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { - break; - } - output.extend(item?); - } - - Ok(Spanned { - item: output, - span: self.span, - }) - } - - pub fn into_string(self) -> Result, ShellError> { - let mut output = String::new(); - let span = self.span; - let ctrlc = &self.ctrlc.clone(); - - for item in self { - if nu_utils::ctrl_c::was_pressed(ctrlc) { - break; - } - output.push_str(&item?.coerce_into_string()?); - } - - Ok(Spanned { item: output, span }) - } - - pub fn chain(self, stream: RawStream) -> RawStream { - RawStream { - stream: Box::new(self.stream.chain(stream.stream)), - leftover: self.leftover.into_iter().chain(stream.leftover).collect(), - ctrlc: self.ctrlc, - is_binary: self.is_binary, - span: self.span, - known_size: self.known_size, - } - } - - pub fn drain(self) -> Result<(), ShellError> { - for next in self { - match next { - Ok(val) => { - if let Value::Error { error, .. } = val { - return Err(*error); - } - } - Err(err) => return Err(err), - } - } - Ok(()) - } -} -impl Debug for RawStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RawStream").finish() - } -} -impl Iterator for RawStream { - type Item = Result; - - fn next(&mut self) -> Option { - if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { - return None; - } - - // If we know we're already binary, just output that - if self.is_binary { - self.stream.next().map(|buffer| { - buffer.map(|mut v| { - if !self.leftover.is_empty() { - for b in self.leftover.drain(..).rev() { - v.insert(0, b); - } - } - Value::binary(v, self.span) - }) - }) - } else { - // We *may* be text. We're only going to try utf-8. Other decodings - // needs to be taken as binary first, then passed through `decode`. - if let Some(buffer) = self.stream.next() { - match buffer { - Ok(mut v) => { - if !self.leftover.is_empty() { - while let Some(b) = self.leftover.pop() { - v.insert(0, b); - } - } - - match String::from_utf8(v.clone()) { - Ok(s) => { - // Great, we have a complete string, let's output it - Some(Ok(Value::string(s, self.span))) - } - Err(err) => { - // Okay, we *might* have a string but we've also got some errors - if v.is_empty() { - // We can just end here - None - } else if v.len() > 3 - && (v.len() - err.utf8_error().valid_up_to() > 3) - { - // As UTF-8 characters are max 4 bytes, if we have more than that in error we know - // that it's not just a character spanning two frames. - // We now know we are definitely binary, so switch to binary and stay there. - self.is_binary = true; - Some(Ok(Value::binary(v, self.span))) - } else { - // Okay, we have a tiny bit of error at the end of the buffer. This could very well be - // a character that spans two frames. Since this is the case, remove the error from - // the current frame an dput it in the leftover buffer. - self.leftover = v[err.utf8_error().valid_up_to()..].to_vec(); - - let buf = v[0..err.utf8_error().valid_up_to()].to_vec(); - - match String::from_utf8(buf) { - Ok(s) => Some(Ok(Value::string(s, self.span))), - Err(_) => { - // Something is definitely wrong. Switch to binary, and stay there - self.is_binary = true; - Some(Ok(Value::binary(v, self.span))) - } - } - } - } - } - } - Err(e) => Some(Err(e)), - } - } else if !self.leftover.is_empty() { - let output = Ok(Value::binary(self.leftover.clone(), self.span)); - self.leftover.clear(); - - Some(output) - } else { - None - } - } - } -} diff --git a/crates/nu-protocol/src/process/child.rs b/crates/nu-protocol/src/process/child.rs new file mode 100644 index 0000000000..cc74b40fc1 --- /dev/null +++ b/crates/nu-protocol/src/process/child.rs @@ -0,0 +1,294 @@ +use crate::{ + byte_stream::convert_file, process::ExitStatus, ErrSpan, IntoSpanned, ShellError, Span, +}; +use nu_system::ForegroundChild; +use os_pipe::PipeReader; +use std::{ + fmt::Debug, + io::{self, Read}, + sync::mpsc::{self, Receiver, RecvError, TryRecvError}, + thread, +}; + +#[derive(Debug)] +enum ExitStatusFuture { + Finished(Result>), + Running(Receiver>), +} + +impl ExitStatusFuture { + fn wait(&mut self, span: Span) -> Result { + match self { + ExitStatusFuture::Finished(Ok(status)) => Ok(*status), + ExitStatusFuture::Finished(Err(err)) => Err(err.as_ref().clone()), + ExitStatusFuture::Running(receiver) => { + let code = match receiver.recv() { + Ok(Ok(status)) => Ok(status), + Ok(Err(err)) => Err(ShellError::IOErrorSpanned { + msg: format!("failed to get exit code: {err:?}"), + span, + }), + Err(RecvError) => Err(ShellError::IOErrorSpanned { + msg: "failed to get exit code".into(), + span, + }), + }; + + *self = ExitStatusFuture::Finished(code.clone().map_err(Box::new)); + + code + } + } + } + + fn try_wait(&mut self, span: Span) -> Result, ShellError> { + match self { + ExitStatusFuture::Finished(Ok(code)) => Ok(Some(*code)), + ExitStatusFuture::Finished(Err(err)) => Err(err.as_ref().clone()), + ExitStatusFuture::Running(receiver) => { + let code = match receiver.try_recv() { + Ok(Ok(status)) => Ok(Some(status)), + Ok(Err(err)) => Err(ShellError::IOErrorSpanned { + msg: format!("failed to get exit code: {err:?}"), + span, + }), + Err(TryRecvError::Disconnected) => Err(ShellError::IOErrorSpanned { + msg: "failed to get exit code".into(), + span, + }), + Err(TryRecvError::Empty) => Ok(None), + }; + + if let Some(code) = code.clone().transpose() { + *self = ExitStatusFuture::Finished(code.map_err(Box::new)); + } + + code + } + } + } +} + +pub enum ChildPipe { + Pipe(PipeReader), + Tee(Box), +} + +impl Debug for ChildPipe { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ChildPipe").finish() + } +} + +impl From for ChildPipe { + fn from(pipe: PipeReader) -> Self { + Self::Pipe(pipe) + } +} + +impl Read for ChildPipe { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + ChildPipe::Pipe(pipe) => pipe.read(buf), + ChildPipe::Tee(tee) => tee.read(buf), + } + } +} + +#[derive(Debug)] +pub struct ChildProcess { + pub stdout: Option, + pub stderr: Option, + exit_status: ExitStatusFuture, + span: Span, +} + +impl ChildProcess { + pub fn new( + mut child: ForegroundChild, + reader: Option, + swap: bool, + span: Span, + ) -> Result { + let (stdout, stderr) = if let Some(combined) = reader { + (Some(combined), None) + } else { + let stdout = child.as_mut().stdout.take().map(convert_file); + let stderr = child.as_mut().stderr.take().map(convert_file); + + if swap { + (stderr, stdout) + } else { + (stdout, stderr) + } + }; + + // Create a thread to wait for the exit status. + let (exit_status_sender, exit_status) = mpsc::channel(); + + thread::Builder::new() + .name("exit status waiter".into()) + .spawn(move || exit_status_sender.send(child.wait().map(Into::into))) + .err_span(span)?; + + Ok(Self::from_raw(stdout, stderr, Some(exit_status), span)) + } + + pub fn from_raw( + stdout: Option, + stderr: Option, + exit_status: Option>>, + span: Span, + ) -> Self { + Self { + stdout: stdout.map(Into::into), + stderr: stderr.map(Into::into), + exit_status: exit_status + .map(ExitStatusFuture::Running) + .unwrap_or(ExitStatusFuture::Finished(Ok(ExitStatus::Exited(0)))), + span, + } + } + + pub fn set_exit_code(&mut self, exit_code: i32) { + self.exit_status = ExitStatusFuture::Finished(Ok(ExitStatus::Exited(exit_code))); + } + + pub fn span(&self) -> Span { + self.span + } + + pub fn into_bytes(mut self) -> Result, ShellError> { + if self.stderr.is_some() { + debug_assert!(false, "stderr should not exist"); + return Err(ShellError::IOErrorSpanned { + msg: "internal error".into(), + span: self.span, + }); + } + + let bytes = if let Some(stdout) = self.stdout { + collect_bytes(stdout).err_span(self.span)? + } else { + Vec::new() + }; + + // TODO: check exit_status + self.exit_status.wait(self.span)?; + + Ok(bytes) + } + + pub fn wait(mut self) -> Result { + if let Some(stdout) = self.stdout.take() { + let stderr = self + .stderr + .take() + .map(|stderr| { + thread::Builder::new() + .name("stderr consumer".into()) + .spawn(move || consume_pipe(stderr)) + }) + .transpose() + .err_span(self.span)?; + + let res = consume_pipe(stdout); + + if let Some(handle) = stderr { + handle + .join() + .map_err(|e| match e.downcast::() { + Ok(io) => ShellError::from((*io).into_spanned(self.span)), + Err(err) => ShellError::GenericError { + error: "Unknown error".into(), + msg: format!("{err:?}"), + span: Some(self.span), + help: None, + inner: Vec::new(), + }, + })? + .err_span(self.span)?; + } + + res.err_span(self.span)?; + } else if let Some(stderr) = self.stderr.take() { + consume_pipe(stderr).err_span(self.span)?; + } + + self.exit_status.wait(self.span) + } + + pub fn try_wait(&mut self) -> Result, ShellError> { + self.exit_status.try_wait(self.span) + } + + pub fn wait_with_output(mut self) -> Result { + let (stdout, stderr) = if let Some(stdout) = self.stdout { + let stderr = self + .stderr + .map(|stderr| thread::Builder::new().spawn(move || collect_bytes(stderr))) + .transpose() + .err_span(self.span)?; + + let stdout = collect_bytes(stdout).err_span(self.span)?; + + let stderr = stderr + .map(|handle| { + handle.join().map_err(|e| match e.downcast::() { + Ok(io) => ShellError::from((*io).into_spanned(self.span)), + Err(err) => ShellError::GenericError { + error: "Unknown error".into(), + msg: format!("{err:?}"), + span: Some(self.span), + help: None, + inner: Vec::new(), + }, + }) + }) + .transpose()? + .transpose() + .err_span(self.span)?; + + (Some(stdout), stderr) + } else { + let stderr = self + .stderr + .map(collect_bytes) + .transpose() + .err_span(self.span)?; + + (None, stderr) + }; + + let exit_status = self.exit_status.wait(self.span)?; + + Ok(ProcessOutput { + stdout, + stderr, + exit_status, + }) + } +} + +fn collect_bytes(pipe: ChildPipe) -> io::Result> { + let mut buf = Vec::new(); + match pipe { + ChildPipe::Pipe(mut pipe) => pipe.read_to_end(&mut buf), + ChildPipe::Tee(mut tee) => tee.read_to_end(&mut buf), + }?; + Ok(buf) +} + +fn consume_pipe(pipe: ChildPipe) -> io::Result<()> { + match pipe { + ChildPipe::Pipe(mut pipe) => io::copy(&mut pipe, &mut io::sink()), + ChildPipe::Tee(mut tee) => io::copy(&mut tee, &mut io::sink()), + }?; + Ok(()) +} + +pub struct ProcessOutput { + pub stdout: Option>, + pub stderr: Option>, + pub exit_status: ExitStatus, +} diff --git a/crates/nu-protocol/src/process/exit_status.rs b/crates/nu-protocol/src/process/exit_status.rs new file mode 100644 index 0000000000..8f3794c44f --- /dev/null +++ b/crates/nu-protocol/src/process/exit_status.rs @@ -0,0 +1,64 @@ +use std::process; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExitStatus { + Exited(i32), + #[cfg(unix)] + Signaled { + signal: i32, + core_dumped: bool, + }, +} + +impl ExitStatus { + pub fn code(self) -> i32 { + match self { + ExitStatus::Exited(code) => code, + #[cfg(unix)] + ExitStatus::Signaled { signal, .. } => -signal, + } + } +} + +#[cfg(unix)] +impl From for ExitStatus { + fn from(status: process::ExitStatus) -> Self { + use std::os::unix::process::ExitStatusExt; + + match (status.code(), status.signal()) { + (Some(code), None) => Self::Exited(code), + (None, Some(signal)) => Self::Signaled { + signal, + core_dumped: status.core_dumped(), + }, + (None, None) => { + debug_assert!(false, "ExitStatus should have either a code or a signal"); + Self::Exited(-1) + } + (Some(code), Some(signal)) => { + // Should be unreachable, as `code()` will be `None` if `signal()` is `Some` + // according to the docs for `ExitStatus::code`. + debug_assert!( + false, + "ExitStatus cannot have both a code ({code}) and a signal ({signal})" + ); + Self::Signaled { + signal, + core_dumped: status.core_dumped(), + } + } + } + } +} + +#[cfg(not(unix))] +impl From for ExitStatus { + fn from(status: process::ExitStatus) -> Self { + let code = status.code(); + debug_assert!( + code.is_some(), + "`ExitStatus::code` cannot return `None` on windows" + ); + Self::Exited(code.unwrap_or(-1)) + } +} diff --git a/crates/nu-protocol/src/process/mod.rs b/crates/nu-protocol/src/process/mod.rs new file mode 100644 index 0000000000..2fcf65f56e --- /dev/null +++ b/crates/nu-protocol/src/process/mod.rs @@ -0,0 +1,5 @@ +mod child; +mod exit_status; + +pub use child::*; +pub use exit_status::ExitStatus; diff --git a/crates/nu-protocol/src/util.rs b/crates/nu-protocol/src/util.rs deleted file mode 100644 index 1c17c49e4c..0000000000 --- a/crates/nu-protocol/src/util.rs +++ /dev/null @@ -1,52 +0,0 @@ -use crate::ShellError; -use std::io::{BufRead, BufReader, Read}; - -pub struct BufferedReader { - input: BufReader, - error: bool, -} - -impl BufferedReader { - pub fn new(input: BufReader) -> Self { - Self { - input, - error: false, - } - } - - pub fn into_inner(self) -> BufReader { - self.input - } -} - -impl Iterator for BufferedReader { - type Item = Result, ShellError>; - - fn next(&mut self) -> Option { - // Don't try to read more data if an error occurs - if self.error { - return None; - } - - let buffer = self.input.fill_buf(); - match buffer { - Ok(s) => { - let result = s.to_vec(); - - let buffer_len = s.len(); - - if buffer_len == 0 { - None - } else { - self.input.consume(buffer_len); - - Some(Ok(result)) - } - } - Err(e) => { - self.error = true; - Some(Err(ShellError::IOError { msg: e.to_string() })) - } - } - } -} diff --git a/crates/nu-protocol/tests/test_pipeline_data.rs b/crates/nu-protocol/tests/test_pipeline_data.rs index 6675f6a04a..95941285ad 100644 --- a/crates/nu-protocol/tests/test_pipeline_data.rs +++ b/crates/nu-protocol/tests/test_pipeline_data.rs @@ -11,5 +11,5 @@ fn test_convert_pipeline_data_to_value() { let new_span = Span::new(5, 6); let converted_value = pipeline_data.into_value(new_span); - assert_eq!(converted_value, Value::int(value_val, new_span)); + assert_eq!(converted_value, Ok(Value::int(value_val, new_span))); } diff --git a/crates/nu-system/src/foreground.rs b/crates/nu-system/src/foreground.rs index d54cab1f19..2fe3c4fb29 100644 --- a/crates/nu-system/src/foreground.rs +++ b/crates/nu-system/src/foreground.rs @@ -1,6 +1,6 @@ use std::{ io, - process::{Child, Command}, + process::{Child, Command, ExitStatus}, sync::{atomic::AtomicU32, Arc}, }; @@ -72,6 +72,10 @@ impl ForegroundChild { }) } } + + pub fn wait(&mut self) -> io::Result { + self.as_mut().wait() + } } impl AsMut for ForegroundChild { diff --git a/crates/nu_plugin_example/src/commands/collect_external.rs b/crates/nu_plugin_example/src/commands/collect_bytes.rs similarity index 56% rename from crates/nu_plugin_example/src/commands/collect_external.rs rename to crates/nu_plugin_example/src/commands/collect_bytes.rs index e5c8c61f2e..51ca1d4222 100644 --- a/crates/nu_plugin_example/src/commands/collect_external.rs +++ b/crates/nu_plugin_example/src/commands/collect_bytes.rs @@ -1,22 +1,22 @@ use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand}; use nu_protocol::{ - Category, Example, LabeledError, PipelineData, RawStream, Signature, Type, Value, + ByteStream, Category, Example, LabeledError, PipelineData, Signature, Type, Value, }; use crate::ExamplePlugin; -/// `> | example collect-external` -pub struct CollectExternal; +/// `> | example collect-bytes` +pub struct CollectBytes; -impl PluginCommand for CollectExternal { +impl PluginCommand for CollectBytes { type Plugin = ExamplePlugin; fn name(&self) -> &str { - "example collect-external" + "example collect-bytes" } fn usage(&self) -> &str { - "Example transformer to raw external stream" + "Example transformer to byte stream" } fn search_terms(&self) -> Vec<&str> { @@ -34,7 +34,7 @@ impl PluginCommand for CollectExternal { fn examples(&self) -> Vec { vec![Example { - example: "[a b] | example collect-external", + example: "[a b] | example collect-bytes", description: "collect strings into one stream", result: Some(Value::test_string("ab")), }] @@ -47,26 +47,19 @@ impl PluginCommand for CollectExternal { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let stream = input.into_iter().map(|value| { - value - .as_str() - .map(|str| str.as_bytes()) - .or_else(|_| value.as_binary()) - .map(|bin| bin.to_vec()) - }); - Ok(PipelineData::ExternalStream { - stdout: Some(RawStream::new(Box::new(stream), None, call.head, None)), - stderr: None, - exit_code: None, - span: call.head, - metadata: None, - trim_end_newline: false, - }) + Ok(PipelineData::ByteStream( + ByteStream::from_result_iter( + input.into_iter().map(Value::coerce_into_binary), + call.head, + None, + ), + None, + )) } } #[test] fn test_examples() -> Result<(), nu_protocol::ShellError> { use nu_plugin_test_support::PluginTest; - PluginTest::new("example", ExamplePlugin.into())?.test_command_examples(&CollectExternal) + PluginTest::new("example", ExamplePlugin.into())?.test_command_examples(&CollectBytes) } diff --git a/crates/nu_plugin_example/src/commands/mod.rs b/crates/nu_plugin_example/src/commands/mod.rs index 9425dad4ca..dd808616a9 100644 --- a/crates/nu_plugin_example/src/commands/mod.rs +++ b/crates/nu_plugin_example/src/commands/mod.rs @@ -24,14 +24,14 @@ pub use env::Env; pub use view_span::ViewSpan; // Stream demos -mod collect_external; +mod collect_bytes; mod echo; mod for_each; mod generate; mod seq; mod sum; -pub use collect_external::CollectExternal; +pub use collect_bytes::CollectBytes; pub use echo::Echo; pub use for_each::ForEach; pub use generate::Generate; diff --git a/crates/nu_plugin_example/src/lib.rs b/crates/nu_plugin_example/src/lib.rs index e87c31229d..182bc85121 100644 --- a/crates/nu_plugin_example/src/lib.rs +++ b/crates/nu_plugin_example/src/lib.rs @@ -24,7 +24,7 @@ impl Plugin for ExamplePlugin { Box::new(ViewSpan), Box::new(DisableGc), // Stream demos - Box::new(CollectExternal), + Box::new(CollectBytes), Box::new(Echo), Box::new(ForEach), Box::new(Generate), diff --git a/crates/nu_plugin_polars/src/cache/rm.rs b/crates/nu_plugin_polars/src/cache/rm.rs index b8b814ba60..5918209f32 100644 --- a/crates/nu_plugin_polars/src/cache/rm.rs +++ b/crates/nu_plugin_polars/src/cache/rm.rs @@ -94,7 +94,7 @@ mod test { .add_decl(Box::new(First))? .add_decl(Box::new(Get))? .eval("let df = ([[a b];[1 2] [3 4]] | polars into-df); polars store-ls | get key | first | polars store-rm $in")?; - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let msg = value .as_list()? .first() diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_arrow.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_arrow.rs index 8dad0d195f..dfb331ac46 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_arrow.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_arrow.rs @@ -124,7 +124,7 @@ pub mod test { assert!(tmp_file.exists()); - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let list = value.as_list()?; assert_eq!(list.len(), 1); let msg = list.first().expect("should have a value").as_str()?; diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_avro.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_avro.rs index 7a7197e47a..3a5dc317e7 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_avro.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_avro.rs @@ -153,7 +153,7 @@ pub mod test { assert!(tmp_file.exists()); - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let list = value.as_list()?; assert_eq!(list.len(), 1); let msg = list.first().expect("should have a value").as_str()?; diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_csv.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_csv.rs index ace95d08bb..d55a53f1fc 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_csv.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_csv.rs @@ -171,7 +171,7 @@ pub mod test { assert!(tmp_file.exists()); - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let list = value.as_list()?; assert_eq!(list.len(), 1); let msg = list.first().expect("should have a value").as_str()?; diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_json_lines.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_json_lines.rs index 4140ca199b..88b4a61bbf 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_json_lines.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_json_lines.rs @@ -125,7 +125,7 @@ pub mod test { assert!(tmp_file.exists()); - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let list = value.as_list()?; assert_eq!(list.len(), 1); let msg = list.first().expect("should have a value").as_str()?; diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_nu.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_nu.rs index 9acac7355c..8e3cdffa24 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_nu.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_nu.rs @@ -89,7 +89,7 @@ impl PluginCommand for ToNu { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) { dataframe_command(plugin, call, value) } else { diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_parquet.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_parquet.rs index e53a4ac41d..4a8208ae12 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_parquet.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_parquet.rs @@ -124,7 +124,7 @@ pub mod test { assert!(tmp_file.exists()); - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let list = value.as_list()?; assert_eq!(list.len(), 1); let msg = list.first().expect("should have a value").as_str()?; diff --git a/crates/nu_plugin_polars/src/dataframe/expressions/expressions_macro.rs b/crates/nu_plugin_polars/src/dataframe/expressions/expressions_macro.rs index 577524123c..feb559aba5 100644 --- a/crates/nu_plugin_polars/src/dataframe/expressions/expressions_macro.rs +++ b/crates/nu_plugin_polars/src/dataframe/expressions/expressions_macro.rs @@ -159,7 +159,7 @@ macro_rules! lazy_expr_command { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) { let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value) .map_err(LabeledError::from)?; @@ -239,7 +239,7 @@ macro_rules! lazy_expr_command { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) { let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value) .map_err(LabeledError::from)?; diff --git a/crates/nu_plugin_polars/src/dataframe/expressions/is_in.rs b/crates/nu_plugin_polars/src/dataframe/expressions/is_in.rs index ed4b567983..47be15e2f3 100644 --- a/crates/nu_plugin_polars/src/dataframe/expressions/is_in.rs +++ b/crates/nu_plugin_polars/src/dataframe/expressions/is_in.rs @@ -114,8 +114,7 @@ impl PluginCommand for ExprIsIn { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => command_df(plugin, engine, call, df), PolarsPluginObject::NuLazyFrame(lazy) => { diff --git a/crates/nu_plugin_polars/src/dataframe/expressions/otherwise.rs b/crates/nu_plugin_polars/src/dataframe/expressions/otherwise.rs index 2bdbfefb35..0e84d8fe96 100644 --- a/crates/nu_plugin_polars/src/dataframe/expressions/otherwise.rs +++ b/crates/nu_plugin_polars/src/dataframe/expressions/otherwise.rs @@ -99,7 +99,7 @@ impl PluginCommand for ExprOtherwise { let otherwise_predicate: Value = call.req(0)?; let otherwise_predicate = NuExpression::try_from_value(plugin, &otherwise_predicate)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let complete: NuExpression = match NuWhen::try_from_value(plugin, &value)?.when_type { NuWhenType::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(), NuWhenType::ChainedThen(chained_when) => chained_when diff --git a/crates/nu_plugin_polars/src/dataframe/expressions/when.rs b/crates/nu_plugin_polars/src/dataframe/expressions/when.rs index 158b2ac757..3c1b0eb481 100644 --- a/crates/nu_plugin_polars/src/dataframe/expressions/when.rs +++ b/crates/nu_plugin_polars/src/dataframe/expressions/when.rs @@ -111,7 +111,7 @@ impl PluginCommand for ExprWhen { let then_predicate: Value = call.req(1)?; let then_predicate = NuExpression::try_from_value(plugin, &then_predicate)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let when_then: NuWhen = match value { Value::Nothing { .. } => when(when_predicate.into_polars()) .then(then_predicate.into_polars()) diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/cast.rs b/crates/nu_plugin_polars/src/dataframe/lazy/cast.rs index 559ca27658..9348a9ec82 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/cast.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/cast.rs @@ -90,7 +90,7 @@ impl PluginCommand for CastDF { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuLazyFrame(lazy) => { let (dtype, column_nm) = df_args(call)?; diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/collect.rs b/crates/nu_plugin_polars/src/dataframe/lazy/collect.rs index db62426e83..b6d8909e31 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/collect.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/collect.rs @@ -61,7 +61,7 @@ impl PluginCommand for LazyCollect { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuLazyFrame(lazy) => { let eager = lazy.collect(call.head)?; diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/explode.rs b/crates/nu_plugin_polars/src/dataframe/lazy/explode.rs index b0609d7a3c..787f07fd46 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/explode.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/explode.rs @@ -50,7 +50,7 @@ impl PluginCommand for LazyExplode { result: Some( NuDataFrame::try_from_columns(vec![ Column::new( - "id".to_string(), + "id".to_string(), vec![ Value::test_int(1), Value::test_int(1), @@ -58,7 +58,7 @@ impl PluginCommand for LazyExplode { Value::test_int(2), ]), Column::new( - "name".to_string(), + "name".to_string(), vec![ Value::test_string("Mercy"), Value::test_string("Mercy"), @@ -66,7 +66,7 @@ impl PluginCommand for LazyExplode { Value::test_string("Bob"), ]), Column::new( - "hobbies".to_string(), + "hobbies".to_string(), vec![ Value::test_string("Cycling"), Value::test_string("Knitting"), @@ -84,7 +84,7 @@ impl PluginCommand for LazyExplode { result: Some( NuDataFrame::try_from_columns(vec![ Column::new( - "hobbies".to_string(), + "hobbies".to_string(), vec![ Value::test_string("Cycling"), Value::test_string("Knitting"), @@ -116,8 +116,7 @@ pub(crate) fn explode( call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => { let lazy = df.lazy(); diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/fetch.rs b/crates/nu_plugin_polars/src/dataframe/lazy/fetch.rs index 49d917a393..8fee4cd159 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/fetch.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/fetch.rs @@ -67,7 +67,7 @@ impl PluginCommand for LazyFetch { input: PipelineData, ) -> Result { let rows: i64 = call.req(0)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; let eager: NuDataFrame = lazy diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/fill_nan.rs b/crates/nu_plugin_polars/src/dataframe/lazy/fill_nan.rs index 851be588f9..baeb9da01b 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/fill_nan.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/fill_nan.rs @@ -92,7 +92,7 @@ impl PluginCommand for LazyFillNA { input: PipelineData, ) -> Result { let fill: Value = call.req(0)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => { diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/fill_null.rs b/crates/nu_plugin_polars/src/dataframe/lazy/fill_null.rs index 64e6fd0d3f..c5fb67cd8a 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/fill_null.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/fill_null.rs @@ -69,7 +69,7 @@ impl PluginCommand for LazyFillNull { input: PipelineData, ) -> Result { let fill: Value = call.req(0)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => cmd_lazy(plugin, engine, call, df.lazy(), fill), diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/filter.rs b/crates/nu_plugin_polars/src/dataframe/lazy/filter.rs index 6adabb967a..f8d400ddf4 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/filter.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/filter.rs @@ -72,7 +72,7 @@ impl PluginCommand for LazyFilter { ) -> Result { let expr_value: Value = call.req(0)?; let filter_expr = NuExpression::try_from_value(plugin, &expr_value)?; - let pipeline_value = input.into_value(call.head); + let pipeline_value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &pipeline_value)?; command(plugin, engine, call, lazy, filter_expr).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/filter_with.rs b/crates/nu_plugin_polars/src/dataframe/lazy/filter_with.rs index cd23a3b370..12ccfbc376 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/filter_with.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/filter_with.rs @@ -67,7 +67,7 @@ impl PluginCommand for FilterWith { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/first.rs b/crates/nu_plugin_polars/src/dataframe/lazy/first.rs index 7f32dbf71d..4692a933b0 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/first.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/first.rs @@ -97,7 +97,7 @@ impl PluginCommand for FirstDF { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) || NuDataFrame::can_downcast(&value) { let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command(plugin, engine, call, lazy).map_err(LabeledError::from) diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/groupby.rs b/crates/nu_plugin_polars/src/dataframe/lazy/groupby.rs index 2bc7f578c8..7aaccfead9 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/groupby.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/groupby.rs @@ -138,7 +138,7 @@ impl PluginCommand for ToLazyGroupBy { })?; } - let pipeline_value = input.into_value(call.head); + let pipeline_value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &pipeline_value)?; command(plugin, engine, call, lazy, expressions).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/join.rs b/crates/nu_plugin_polars/src/dataframe/lazy/join.rs index feea8cf308..6db0269403 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/join.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/join.rs @@ -228,7 +228,7 @@ impl PluginCommand for LazyJoin { let suffix: Option = call.get_flag("suffix")?; let suffix = suffix.unwrap_or_else(|| "_x".into()); - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; let lazy = lazy.to_polars(); diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/last.rs b/crates/nu_plugin_polars/src/dataframe/lazy/last.rs index 44095ac44f..0453c71d1e 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/last.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/last.rs @@ -72,7 +72,7 @@ impl PluginCommand for LastDF { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command(plugin, engine, call, df).map_err(|e| e.into()) diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/median.rs b/crates/nu_plugin_polars/src/dataframe/lazy/median.rs index abd55c77c1..ffd69d14e4 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/median.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/median.rs @@ -89,7 +89,7 @@ impl PluginCommand for LazyMedian { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => command(plugin, engine, call, df.lazy()), PolarsPluginObject::NuLazyFrame(lazy) => command(plugin, engine, call, lazy), diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/quantile.rs b/crates/nu_plugin_polars/src/dataframe/lazy/quantile.rs index 46339cc9fc..f6217ff89b 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/quantile.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/quantile.rs @@ -97,7 +97,7 @@ impl PluginCommand for LazyQuantile { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let quantile: f64 = call.req(0)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => { diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/rename.rs b/crates/nu_plugin_polars/src/dataframe/lazy/rename.rs index b678824584..c32b8d9451 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/rename.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/rename.rs @@ -120,7 +120,7 @@ impl PluginCommand for RenameDF { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/select.rs b/crates/nu_plugin_polars/src/dataframe/lazy/select.rs index e49aa8e654..75b3f8f804 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/select.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/select.rs @@ -65,7 +65,7 @@ impl PluginCommand for LazySelect { let expr_value = Value::list(vals, call.head); let expressions = NuExpression::extract_exprs(plugin, expr_value)?; - let pipeline_value = input.into_value(call.head); + let pipeline_value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &pipeline_value)?; let lazy = NuLazyFrame::new(lazy.to_polars().select(&expressions)); lazy.to_pipeline_data(plugin, engine, call.head) diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/sort_by_expr.rs b/crates/nu_plugin_polars/src/dataframe/lazy/sort_by_expr.rs index 4a975afe97..2beba4424c 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/sort_by_expr.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/sort_by_expr.rs @@ -145,7 +145,7 @@ impl PluginCommand for LazySortBy { maintain_order, }; - let pipeline_value = input.into_value(call.head); + let pipeline_value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &pipeline_value)?; let lazy = NuLazyFrame::new(lazy.to_polars().sort_by_exprs(&expressions, sort_options)); lazy.to_pipeline_data(plugin, engine, call.head) diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/with_column.rs b/crates/nu_plugin_polars/src/dataframe/lazy/with_column.rs index e8092231d8..d2f953b068 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/with_column.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/with_column.rs @@ -83,7 +83,7 @@ impl PluginCommand for WithColumn { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/series/masks/is_not_null.rs b/crates/nu_plugin_polars/src/dataframe/series/masks/is_not_null.rs index 218cd116b4..b7e506a67c 100644 --- a/crates/nu_plugin_polars/src/dataframe/series/masks/is_not_null.rs +++ b/crates/nu_plugin_polars/src/dataframe/series/masks/is_not_null.rs @@ -78,8 +78,7 @@ impl PluginCommand for IsNotNull { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => command(plugin, engine, call, df), PolarsPluginObject::NuLazyFrame(lazy) => { diff --git a/crates/nu_plugin_polars/src/dataframe/series/masks/is_null.rs b/crates/nu_plugin_polars/src/dataframe/series/masks/is_null.rs index beb3793661..bc04e7fb76 100644 --- a/crates/nu_plugin_polars/src/dataframe/series/masks/is_null.rs +++ b/crates/nu_plugin_polars/src/dataframe/series/masks/is_null.rs @@ -80,8 +80,7 @@ impl PluginCommand for IsNull { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => command(plugin, engine, call, df), PolarsPluginObject::NuLazyFrame(lazy) => { diff --git a/crates/nu_plugin_polars/src/dataframe/series/n_unique.rs b/crates/nu_plugin_polars/src/dataframe/series/n_unique.rs index 5426ef6d1d..51c6e1bdb3 100644 --- a/crates/nu_plugin_polars/src/dataframe/series/n_unique.rs +++ b/crates/nu_plugin_polars/src/dataframe/series/n_unique.rs @@ -70,8 +70,7 @@ impl PluginCommand for NUnique { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => command(plugin, engine, call, df), PolarsPluginObject::NuLazyFrame(lazy) => { diff --git a/crates/nu_plugin_polars/src/dataframe/series/shift.rs b/crates/nu_plugin_polars/src/dataframe/series/shift.rs index 556b3361c1..c37ba2f2e9 100644 --- a/crates/nu_plugin_polars/src/dataframe/series/shift.rs +++ b/crates/nu_plugin_polars/src/dataframe/series/shift.rs @@ -92,7 +92,7 @@ impl PluginCommand for Shift { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/series/unique.rs b/crates/nu_plugin_polars/src/dataframe/series/unique.rs index 47efd880a6..2475ad026a 100644 --- a/crates/nu_plugin_polars/src/dataframe/series/unique.rs +++ b/crates/nu_plugin_polars/src/dataframe/series/unique.rs @@ -134,7 +134,7 @@ impl PluginCommand for Unique { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let df = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command_lazy(plugin, engine, call, df).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/values/mod.rs b/crates/nu_plugin_polars/src/dataframe/values/mod.rs index a43c8f2412..179c85bf36 100644 --- a/crates/nu_plugin_polars/src/dataframe/values/mod.rs +++ b/crates/nu_plugin_polars/src/dataframe/values/mod.rs @@ -84,7 +84,7 @@ impl PolarsPluginObject { input: PipelineData, span: Span, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(plugin, &value) } @@ -242,7 +242,7 @@ pub trait PolarsPluginCustomValue: CustomValue { /// Handles the ability for a PolarsObjectType implementations to convert between /// their respective CustValue type. /// PolarsPluginObjectType's (NuDataFrame, NuLazyFrame) should -/// implement this trait. +/// implement this trait. pub trait CustomValueSupport: Cacheable { type CV: PolarsPluginCustomValue + CustomValue + 'static; @@ -301,7 +301,7 @@ pub trait CustomValueSupport: Cacheable { input: PipelineData, span: Span, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(plugin, &value) } diff --git a/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/mod.rs b/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/mod.rs index 30a5ea691d..46132133dd 100644 --- a/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/mod.rs +++ b/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/mod.rs @@ -519,7 +519,7 @@ impl NuDataFrame { input: PipelineData, span: Span, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value_coerce(plugin, &value, span) } } diff --git a/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/mod.rs b/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/mod.rs index f3c969b03d..48e296e95e 100644 --- a/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/mod.rs +++ b/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/mod.rs @@ -109,7 +109,7 @@ impl NuLazyFrame { input: PipelineData, span: Span, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value_coerce(plugin, &value) } } diff --git a/src/main.rs b/src/main.rs index db0c80d4f2..d0fc023b68 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,15 +25,13 @@ use nu_cmd_base::util::get_init_cwd; use nu_lsp::LanguageServer; use nu_path::canonicalize_with; use nu_protocol::{ - engine::EngineState, report_error_new, util::BufferedReader, PipelineData, RawStream, - ShellError, Span, Value, + engine::EngineState, report_error_new, ByteStream, PipelineData, ShellError, Span, Value, }; use nu_std::load_standard_library; use nu_utils::utils::perf; use run::{run_commands, run_file, run_repl}; use signals::ctrlc_protection; use std::{ - io::BufReader, path::PathBuf, str::FromStr, sync::{atomic::AtomicBool, Arc}, @@ -345,22 +343,7 @@ fn main() -> Result<()> { start_time = std::time::Instant::now(); let input = if let Some(redirect_stdin) = &parsed_nu_cli_args.redirect_stdin { trace!("redirecting stdin"); - let stdin = std::io::stdin(); - let buf_reader = BufReader::new(stdin); - - PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(BufferedReader::new(buf_reader)), - Some(ctrlc.clone()), - redirect_stdin.span, - None, - )), - stderr: None, - exit_code: None, - span: redirect_stdin.span, - metadata: None, - trim_end_newline: false, - } + PipelineData::ByteStream(ByteStream::stdin(redirect_stdin.span)?, None) } else { trace!("not redirecting stdin"); PipelineData::empty() @@ -450,7 +433,7 @@ fn main() -> Result<()> { ); } - LanguageServer::initialize_stdio_connection()?.serve_requests(engine_state, ctrlc) + LanguageServer::initialize_stdio_connection()?.serve_requests(engine_state, ctrlc)? } else if let Some(commands) = parsed_nu_cli_args.commands.clone() { run_commands( &mut engine_state, @@ -460,7 +443,6 @@ fn main() -> Result<()> { input, entire_start_time, ); - Ok(()) } else if !script_name.is_empty() { run_file( &mut engine_state, @@ -470,8 +452,9 @@ fn main() -> Result<()> { args_to_script, input, ); - Ok(()) } else { - run_repl(&mut engine_state, parsed_nu_cli_args, entire_start_time) + run_repl(&mut engine_state, parsed_nu_cli_args, entire_start_time)? } + + Ok(()) } diff --git a/tests/plugins/stream.rs b/tests/plugins/stream.rs index 8530e5bc32..b8771580f7 100644 --- a/tests/plugins/stream.rs +++ b/tests/plugins/stream.rs @@ -119,40 +119,40 @@ fn sum_big_stream() { } #[test] -fn collect_external_accepts_list_of_string() { +fn collect_bytes_accepts_list_of_string() { let actual = nu_with_plugins!( cwd: "tests/fixtures/formats", plugin: ("nu_plugin_example"), - "[a b] | example collect-external" + "[a b] | example collect-bytes" ); assert_eq!(actual.out, "ab"); } #[test] -fn collect_external_accepts_list_of_binary() { +fn collect_bytes_accepts_list_of_binary() { let actual = nu_with_plugins!( cwd: "tests/fixtures/formats", plugin: ("nu_plugin_example"), - "[0x[41] 0x[42]] | example collect-external" + "[0x[41] 0x[42]] | example collect-bytes" ); assert_eq!(actual.out, "AB"); } #[test] -fn collect_external_produces_raw_input() { +fn collect_bytes_produces_byte_stream() { let actual = nu_with_plugins!( cwd: "tests/fixtures/formats", plugin: ("nu_plugin_example"), - "[a b c] | example collect-external | describe" + "[a b c] | example collect-bytes | describe" ); - assert_eq!(actual.out, "raw input"); + assert_eq!(actual.out, "byte stream"); } #[test] -fn collect_external_big_stream() { +fn collect_bytes_big_stream() { // This in particular helps to ensure that a big stream can be both read and written at the same // time without deadlocking let actual = nu_with_plugins!( @@ -160,9 +160,8 @@ fn collect_external_big_stream() { plugin: ("nu_plugin_example"), r#"( seq 1 10000 | - to text | - each { into string } | - example collect-external | + each {|i| ($i | into string) ++ (char newline) } | + example collect-bytes | lines | length )"# diff --git a/tests/shell/pipeline/commands/internal.rs b/tests/shell/pipeline/commands/internal.rs index 7bc75b07ad..6c2226ff65 100644 --- a/tests/shell/pipeline/commands/internal.rs +++ b/tests/shell/pipeline/commands/internal.rs @@ -1131,13 +1131,13 @@ fn pipe_input_to_print() { #[test] fn err_pipe_input_to_print() { let actual = nu!(r#""foo" e>| print"#); - assert!(actual.err.contains("only works on external streams")); + assert!(actual.err.contains("only works on external commands")); } #[test] fn outerr_pipe_input_to_print() { let actual = nu!(r#""foo" o+e>| print"#); - assert!(actual.err.contains("only works on external streams")); + assert!(actual.err.contains("only works on external commands")); } #[test]