diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 40cda47bbc..23d6a5ff95 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -26,7 +26,7 @@ Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass (on Windows make sure to [enable developer mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging)) -- `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library +- `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 6d18269e89..e8a7f55817 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -19,7 +19,7 @@ jobs: # Prevent sudden announcement of a new advisory from failing ci: continue-on-error: true steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.5 - uses: rustsec/audit-check@v1.4.1 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5aec3f923a..4815491854 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.5 - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 @@ -89,7 +89,7 @@ jobs: runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.5 - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 @@ -121,7 +121,7 @@ jobs: runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.5 - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 @@ -174,7 +174,7 @@ jobs: runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.5 - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index f86d7572a2..ab9f93d97d 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -27,7 +27,7 @@ jobs: # if: github.repository == 'nushell/nightly' steps: - name: Checkout - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 if: github.repository == 'nushell/nightly' with: ref: main @@ -123,7 +123,7 @@ jobs: runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.5 with: ref: main fetch-depth: 0 @@ -174,7 +174,7 @@ jobs: # REF: https://github.com/marketplace/actions/gh-release # Create a release only in nushell/nightly repo - name: Publish Archive - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v2.0.5 if: ${{ startsWith(github.repository, 'nushell/nightly') }} with: prerelease: true @@ -235,7 +235,7 @@ jobs: runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.5 with: ref: main fetch-depth: 0 @@ -286,7 +286,7 @@ jobs: # REF: https://github.com/marketplace/actions/gh-release # Create a release only in nushell/nightly repo - name: Publish Archive - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v2.0.5 if: ${{ startsWith(github.repository, 'nushell/nightly') }} with: draft: false @@ -310,7 +310,7 @@ jobs: - name: Waiting for Release run: sleep 1800 - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.5 with: ref: main diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fce38d9b4b..ffe653bd22 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,7 +73,7 @@ jobs: runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.5 - name: Update Rust Toolchain Target run: | @@ -104,7 +104,7 @@ jobs: # REF: https://github.com/marketplace/actions/gh-release - name: Publish Archive - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v2.0.5 if: ${{ startsWith(github.ref, 'refs/tags/') }} with: draft: true @@ -163,7 +163,7 @@ jobs: runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.5 - name: Update Rust Toolchain Target run: | @@ -194,7 +194,7 @@ jobs: # REF: https://github.com/marketplace/actions/gh-release - name: Publish Archive - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v2.0.5 if: ${{ startsWith(github.ref, 'refs/tags/') }} with: draft: true diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml index a9354ade1d..dacfb83928 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/typos.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Actions Repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Check spelling uses: crate-ci/typos@v1.21.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1ecda698c7..d5cf758b56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,6 @@ It is good practice to cover your changes with a test. Also, try to think about Tests can be found in different places: * `/tests` -* `src/tests` * command examples * crate-specific tests diff --git a/Cargo.lock b/Cargo.lock index c29386e03f..b008026a09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -377,7 +377,7 @@ dependencies = [ "bitflags 2.5.0", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.11.0", "lazy_static", "lazycell", "proc-macro2", @@ -2043,9 +2043,9 @@ dependencies = [ [[package]] name = "interprocess" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7fb8583fab9503654385e2bafda123376445a77027a1b106dd7e44cf51122f" +checksum = "7b4d0250d41da118226e55b3d50ca3f0d9e0a0f6829b92f543ac0054aeea1572" dependencies = [ "libc", "recvmsg", @@ -2861,6 +2861,7 @@ dependencies = [ "reedline", "rstest", "sysinfo", + "tempfile", "unicode-segmentation", "uuid", "which", @@ -3263,11 +3264,13 @@ dependencies = [ "indexmap", "lru", "miette", + "nix", "nu-path", "nu-system", "nu-test-support", "nu-utils", "num-format", + "os_pipe", "pretty_assertions", "rmp-serde", "rstest", @@ -4850,8 +4853,7 @@ dependencies = [ [[package]] name = "reedline" version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf59e4c97b5049ba96b052cdb652368305a2eddcbce9bf1c16f9d003139eeea" +source = "git+https://github.com/nushell/reedline?branch=main#a580ea56d4e5a889468b2969d2a1534379504ab6" dependencies = [ "arboard", "chrono", @@ -5065,9 +5067,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "8.3.0" +version = "8.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb78f46d0066053d16d4ca7b898e9343bc3530f71c61d5ad84cd404ada068745" +checksum = "19549741604902eb99a7ed0ee177a0663ee1eda51a29f71401f166e47e77806a" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -5076,9 +5078,9 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "8.3.0" +version = "8.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91ac2a3c6c0520a3fb3dd89321177c3c692937c4eb21893378219da10c44fc8" +checksum = "cb9f96e283ec64401f30d3df8ee2aaeb2561f34c824381efa24a35f79bf40ee4" dependencies = [ "proc-macro2", "quote", @@ -5089,9 +5091,9 @@ dependencies = [ [[package]] name = "rust-embed-utils" -version = "8.3.0" +version = "8.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f69089032567ffff4eada41c573fc43ff466c7db7c5688b2e7969584345581" +checksum = "38c74a686185620830701348de757fd36bef4aa9680fd23c49fc539ddcc1af32" dependencies = [ "sha2", "walkdir", diff --git a/Cargo.toml b/Cargo.toml index beabd26e20..2e9c7e0b0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ heck = "0.5.0" human-date-parser = "0.1.1" indexmap = "2.2" indicatif = "0.17" -interprocess = "2.0.1" +interprocess = "2.1.0" is_executable = "1.0" itertools = "0.12" libc = "0.2" @@ -119,7 +119,7 @@ num-traits = "0.2" omnipath = "0.1" once_cell = "1.18" open = "5.1" -os_pipe = "1.1" +os_pipe = { version = "1.1", features = ["io_safety"] } pathdiff = "0.2" percent-encoding = "2" pretty_assertions = "1.4" @@ -140,7 +140,7 @@ ropey = "1.6.1" roxmltree = "0.19" rstest = { version = "0.18", default-features = false } rusqlite = "0.31" -rust-embed = "8.3.0" +rust-embed = "8.4.0" same-file = "1.0" serde = { version = "1.0", default-features = false } serde_json = "1.0" @@ -305,7 +305,7 @@ bench = false # To use a development version of a dependency please use a global override here # changing versions in each sub-crate of the workspace is tedious [patch.crates-io] -# reedline = { git = "https://github.com/nushell/reedline", branch = "main" } +reedline = { git = "https://github.com/nushell/reedline", branch = "main" } # nu-ansi-term = {git = "https://github.com/nushell/nu-ansi-term.git", branch = "main"} # Run all benchmarks with `cargo bench` diff --git a/benches/benchmarks.rs b/benches/benchmarks.rs index e291eeebcc..84552daef9 100644 --- a/benches/benchmarks.rs +++ b/benches/benchmarks.rs @@ -4,15 +4,11 @@ use nu_plugin_protocol::{PluginCallResponse, PluginOutput}; use nu_protocol::{ engine::{EngineState, Stack}, - eval_const::create_nu_constant, - PipelineData, Span, Spanned, Value, NU_VARIABLE_ID, + PipelineData, Span, Spanned, Value, }; use nu_std::load_standard_library; use nu_utils::{get_default_config, get_default_env}; -use std::{ - path::{Path, PathBuf}, - rc::Rc, -}; +use std::rc::Rc; use std::hint::black_box; @@ -22,38 +18,18 @@ fn load_bench_commands() -> EngineState { nu_command::add_shell_command_context(nu_cmd_lang::create_default_context()) } -fn canonicalize_path(engine_state: &EngineState, path: &Path) -> PathBuf { - let cwd = engine_state.cwd_as_string(None).unwrap(); - - if path.exists() { - match nu_path::canonicalize_with(path, cwd) { - Ok(canon_path) => canon_path, - Err(_) => path.to_owned(), - } - } else { - path.to_owned() - } -} - -fn get_home_path(engine_state: &EngineState) -> PathBuf { - nu_path::home_dir() - .map(|path| canonicalize_path(engine_state, &path)) - .unwrap_or_default() -} - fn setup_engine() -> EngineState { let mut engine_state = load_bench_commands(); - let home_path = get_home_path(&engine_state); + let cwd = std::env::current_dir() + .unwrap() + .into_os_string() + .into_string() + .unwrap(); // parsing config.nu breaks without PWD set, so set a valid path - engine_state.add_env_var( - "PWD".into(), - Value::string(home_path.to_string_lossy(), Span::test_data()), - ); + engine_state.add_env_var("PWD".into(), Value::string(cwd, Span::test_data())); - let nu_const = create_nu_constant(&engine_state, Span::unknown()) - .expect("Failed to create nushell constant."); - engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const); + engine_state.generate_nu_constant(); engine_state } @@ -107,6 +83,7 @@ fn bench_command( b.iter(move || { let mut stack = stack.clone(); let mut engine = engine.clone(); + #[allow(clippy::unit_arg)] black_box( evaluate_commands( &commands, diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index 3423450cf3..e631f7ccf6 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -15,6 +15,7 @@ nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" } nu-command = { path = "../nu-command", version = "0.93.1" } nu-test-support = { path = "../nu-test-support", version = "0.93.1" } rstest = { workspace = true, default-features = false } +tempfile = { workspace = true } [dependencies] nu-cmd-base = { path = "../nu-cmd-base", version = "0.93.1" } diff --git a/crates/nu-cli/src/completions/base.rs b/crates/nu-cli/src/completions/base.rs index c4290b8767..0debabe688 100644 --- a/crates/nu-cli/src/completions/base.rs +++ b/crates/nu-cli/src/completions/base.rs @@ -1,13 +1,18 @@ use crate::completions::{CompletionOptions, SortBy}; -use nu_protocol::{engine::StateWorkingSet, levenshtein_distance, Span}; +use nu_protocol::{ + engine::{Stack, StateWorkingSet}, + levenshtein_distance, Span, +}; use reedline::Suggestion; // Completer trait represents the three stages of the completion // fetch, filter and sort pub trait Completer { + #[allow(clippy::too_many_arguments)] fn fetch( &mut self, working_set: &StateWorkingSet, + stack: &Stack, prefix: Vec, span: Span, offset: usize, diff --git a/crates/nu-cli/src/completions/command_completions.rs b/crates/nu-cli/src/completions/command_completions.rs index 42094f9c97..2549854540 100644 --- a/crates/nu-cli/src/completions/command_completions.rs +++ b/crates/nu-cli/src/completions/command_completions.rs @@ -4,16 +4,14 @@ use crate::{ }; use nu_parser::FlatShape; use nu_protocol::{ - engine::{CachedFile, EngineState, StateWorkingSet}, + engine::{CachedFile, Stack, StateWorkingSet}, Span, }; use reedline::Suggestion; -use std::sync::Arc; use super::SemanticSuggestion; pub struct CommandCompletion { - engine_state: Arc, flattened: Vec<(Span, FlatShape)>, flat_shape: FlatShape, force_completion_after_space: bool, @@ -21,14 +19,11 @@ pub struct CommandCompletion { impl CommandCompletion { pub fn new( - engine_state: Arc, - _: &StateWorkingSet, flattened: Vec<(Span, FlatShape)>, flat_shape: FlatShape, force_completion_after_space: bool, ) -> Self { Self { - engine_state, flattened, flat_shape, force_completion_after_space, @@ -37,13 +32,14 @@ impl CommandCompletion { fn external_command_completion( &self, + working_set: &StateWorkingSet, prefix: &str, match_algorithm: MatchAlgorithm, ) -> Vec { let mut executables = vec![]; // os agnostic way to get the PATH env var - let paths = self.engine_state.get_path_env_var(); + let paths = working_set.permanent_state.get_path_env_var(); if let Some(paths) = paths { if let Ok(paths) = paths.as_list() { @@ -52,7 +48,10 @@ impl CommandCompletion { if let Ok(mut contents) = std::fs::read_dir(path.as_ref()) { while let Some(Ok(item)) = contents.next() { - if self.engine_state.config.max_external_completion_results + if working_set + .permanent_state + .config + .max_external_completion_results > executables.len() as i64 && !executables.contains( &item @@ -114,7 +113,7 @@ impl CommandCompletion { if find_externals { let results_external = self - .external_command_completion(&partial, match_algorithm) + .external_command_completion(working_set, &partial, match_algorithm) .into_iter() .map(move |x| SemanticSuggestion { suggestion: Suggestion { @@ -161,6 +160,7 @@ impl Completer for CommandCompletion { fn fetch( &mut self, working_set: &StateWorkingSet, + _stack: &Stack, _prefix: Vec, span: Span, offset: usize, @@ -266,6 +266,8 @@ pub fn is_passthrough_command(working_set_file_contents: &[CachedFile]) -> bool #[cfg(test)] mod command_completions_tests { use super::*; + use nu_protocol::engine::EngineState; + use std::sync::Arc; #[test] fn test_find_non_whitespace_index() { diff --git a/crates/nu-cli/src/completions/completer.rs b/crates/nu-cli/src/completions/completer.rs index 5837772d54..007a0e288a 100644 --- a/crates/nu-cli/src/completions/completer.rs +++ b/crates/nu-cli/src/completions/completer.rs @@ -22,10 +22,10 @@ pub struct NuCompleter { } impl NuCompleter { - pub fn new(engine_state: Arc, stack: Stack) -> Self { + pub fn new(engine_state: Arc, stack: Arc) -> Self { Self { engine_state, - stack: stack.reset_out_dest().capture(), + stack: Stack::with_parent(stack).reset_out_dest().capture(), } } @@ -52,8 +52,15 @@ impl NuCompleter { }; // Fetch - let mut suggestions = - completer.fetch(working_set, prefix.clone(), new_span, offset, pos, &options); + let mut suggestions = completer.fetch( + working_set, + &self.stack, + prefix.clone(), + new_span, + offset, + pos, + &options, + ); // Sort suggestions = completer.sort(suggestions, prefix); @@ -96,9 +103,8 @@ impl NuCompleter { PipelineData::empty(), ); - match result { - Ok(pd) => { - let value = pd.into_value(span); + match result.and_then(|data| data.into_value(span)) { + Ok(value) => { if let Value::List { vals, .. } = value { let result = map_value_completions(vals.iter(), Span::new(span.start, span.end), offset); @@ -175,11 +181,8 @@ impl NuCompleter { // Variables completion if prefix.starts_with(b"$") || most_left_var.is_some() { - let mut completer = VariableCompletion::new( - self.engine_state.clone(), - self.stack.clone(), - most_left_var.unwrap_or((vec![], vec![])), - ); + let mut completer = + VariableCompletion::new(most_left_var.unwrap_or((vec![], vec![]))); return self.process_completion( &mut completer, @@ -224,8 +227,6 @@ impl NuCompleter { || (flat_idx == 0 && working_set.get_span_contents(new_span).is_empty()) { let mut completer = CommandCompletion::new( - self.engine_state.clone(), - &working_set, flattened.clone(), // flat_idx, FlatShape::String, @@ -253,10 +254,7 @@ impl NuCompleter { || prev_expr_str == b"overlay use" || prev_expr_str == b"source-env" { - let mut completer = DotNuCompletion::new( - self.engine_state.clone(), - self.stack.clone(), - ); + let mut completer = DotNuCompletion::new(); return self.process_completion( &mut completer, @@ -267,10 +265,7 @@ impl NuCompleter { pos, ); } else if prev_expr_str == b"ls" { - let mut completer = FileCompletion::new( - self.engine_state.clone(), - self.stack.clone(), - ); + let mut completer = FileCompletion::new(); return self.process_completion( &mut completer, @@ -288,7 +283,6 @@ impl NuCompleter { match &flat.1 { FlatShape::Custom(decl_id) => { let mut completer = CustomCompletion::new( - self.engine_state.clone(), self.stack.clone(), *decl_id, initial_line, @@ -304,10 +298,7 @@ impl NuCompleter { ); } FlatShape::Directory => { - let mut completer = DirectoryCompletion::new( - self.engine_state.clone(), - self.stack.clone(), - ); + let mut completer = DirectoryCompletion::new(); return self.process_completion( &mut completer, @@ -319,10 +310,7 @@ impl NuCompleter { ); } FlatShape::Filepath | FlatShape::GlobPattern => { - let mut completer = FileCompletion::new( - self.engine_state.clone(), - self.stack.clone(), - ); + let mut completer = FileCompletion::new(); return self.process_completion( &mut completer, @@ -335,8 +323,6 @@ impl NuCompleter { } flat_shape => { let mut completer = CommandCompletion::new( - self.engine_state.clone(), - &working_set, flattened.clone(), // flat_idx, flat_shape.clone(), @@ -369,10 +355,7 @@ impl NuCompleter { } // Check for file completion - let mut completer = FileCompletion::new( - self.engine_state.clone(), - self.stack.clone(), - ); + let mut completer = FileCompletion::new(); out = self.process_completion( &mut completer, &working_set, @@ -557,7 +540,7 @@ mod completer_tests { result.err().unwrap() ); - let mut completer = NuCompleter::new(engine_state.into(), Stack::new()); + let mut completer = NuCompleter::new(engine_state.into(), Arc::new(Stack::new())); let dataset = [ ("sudo", false, "", Vec::new()), ("sudo l", true, "l", vec!["ls", "let", "lines", "loop"]), diff --git a/crates/nu-cli/src/completions/custom_completions.rs b/crates/nu-cli/src/completions/custom_completions.rs index 12a7762e94..17c8e6a924 100644 --- a/crates/nu-cli/src/completions/custom_completions.rs +++ b/crates/nu-cli/src/completions/custom_completions.rs @@ -6,14 +6,13 @@ use nu_engine::eval_call; use nu_protocol::{ ast::{Argument, Call, Expr, Expression}, debugger::WithoutDebug, - engine::{EngineState, Stack, StateWorkingSet}, + engine::{Stack, StateWorkingSet}, PipelineData, Span, Type, Value, }; use nu_utils::IgnoreCaseExt; -use std::{collections::HashMap, sync::Arc}; +use std::collections::HashMap; pub struct CustomCompletion { - engine_state: Arc, stack: Stack, decl_id: usize, line: String, @@ -21,10 +20,9 @@ pub struct CustomCompletion { } impl CustomCompletion { - pub fn new(engine_state: Arc, stack: Stack, decl_id: usize, line: String) -> Self { + pub fn new(stack: Stack, decl_id: usize, line: String) -> Self { Self { - engine_state, - stack: stack.reset_out_dest().capture(), + stack, decl_id, line, sort_by: SortBy::None, @@ -35,7 +33,8 @@ impl CustomCompletion { impl Completer for CustomCompletion { fn fetch( &mut self, - _: &StateWorkingSet, + working_set: &StateWorkingSet, + _stack: &Stack, prefix: Vec, span: Span, offset: usize, @@ -47,7 +46,7 @@ impl Completer for CustomCompletion { // Call custom declaration let result = eval_call::( - &self.engine_state, + working_set.permanent_state, &mut self.stack, &Call { decl_id: self.decl_id, @@ -75,55 +74,53 @@ impl Completer for CustomCompletion { // Parse result let suggestions = result - .map(|pd| { - let value = pd.into_value(span); - match &value { - Value::Record { val, .. } => { - let completions = val - .get("completions") - .and_then(|val| { - val.as_list() - .ok() - .map(|it| map_value_completions(it.iter(), span, offset)) - }) - .unwrap_or_default(); - let options = val.get("options"); + .and_then(|data| data.into_value(span)) + .map(|value| match &value { + Value::Record { val, .. } => { + let completions = val + .get("completions") + .and_then(|val| { + val.as_list() + .ok() + .map(|it| map_value_completions(it.iter(), span, offset)) + }) + .unwrap_or_default(); + let options = val.get("options"); - if let Some(Value::Record { val: options, .. }) = &options { - let should_sort = options - .get("sort") - .and_then(|val| val.as_bool().ok()) - .unwrap_or(false); + if let Some(Value::Record { val: options, .. }) = &options { + let should_sort = options + .get("sort") + .and_then(|val| val.as_bool().ok()) + .unwrap_or(false); - if should_sort { - self.sort_by = SortBy::Ascending; - } - - custom_completion_options = Some(CompletionOptions { - case_sensitive: options - .get("case_sensitive") - .and_then(|val| val.as_bool().ok()) - .unwrap_or(true), - positional: options - .get("positional") - .and_then(|val| val.as_bool().ok()) - .unwrap_or(true), - match_algorithm: match options.get("completion_algorithm") { - Some(option) => option - .coerce_string() - .ok() - .and_then(|option| option.try_into().ok()) - .unwrap_or(MatchAlgorithm::Prefix), - None => completion_options.match_algorithm, - }, - }); + if should_sort { + self.sort_by = SortBy::Ascending; } - completions + custom_completion_options = Some(CompletionOptions { + case_sensitive: options + .get("case_sensitive") + .and_then(|val| val.as_bool().ok()) + .unwrap_or(true), + positional: options + .get("positional") + .and_then(|val| val.as_bool().ok()) + .unwrap_or(true), + match_algorithm: match options.get("completion_algorithm") { + Some(option) => option + .coerce_string() + .ok() + .and_then(|option| option.try_into().ok()) + .unwrap_or(MatchAlgorithm::Prefix), + None => completion_options.match_algorithm, + }, + }); } - Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset), - _ => vec![], + + completions } + Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset), + _ => vec![], }) .unwrap_or_default(); diff --git a/crates/nu-cli/src/completions/directory_completions.rs b/crates/nu-cli/src/completions/directory_completions.rs index e8d463c19f..024322f997 100644 --- a/crates/nu-cli/src/completions/directory_completions.rs +++ b/crates/nu-cli/src/completions/directory_completions.rs @@ -8,25 +8,16 @@ use nu_protocol::{ levenshtein_distance, Span, }; use reedline::Suggestion; -use std::{ - path::{Path, MAIN_SEPARATOR as SEP}, - sync::Arc, -}; +use std::path::{Path, MAIN_SEPARATOR as SEP}; use super::SemanticSuggestion; -#[derive(Clone)] -pub struct DirectoryCompletion { - engine_state: Arc, - stack: Stack, -} +#[derive(Clone, Default)] +pub struct DirectoryCompletion {} impl DirectoryCompletion { - pub fn new(engine_state: Arc, stack: Stack) -> Self { - Self { - engine_state, - stack, - } + pub fn new() -> Self { + Self::default() } } @@ -34,10 +25,11 @@ impl Completer for DirectoryCompletion { fn fetch( &mut self, working_set: &StateWorkingSet, + stack: &Stack, prefix: Vec, span: Span, offset: usize, - _: usize, + _pos: usize, options: &CompletionOptions, ) -> Vec { let AdjustView { prefix, span, .. } = adjust_if_intermediate(&prefix, working_set, span); @@ -47,10 +39,10 @@ impl Completer for DirectoryCompletion { let output: Vec<_> = directory_completion( span, &prefix, - &self.engine_state.current_work_dir(), + &working_set.permanent_state.current_work_dir(), options, - self.engine_state.as_ref(), - &self.stack, + working_set.permanent_state, + stack, ) .into_iter() .map(move |x| SemanticSuggestion { diff --git a/crates/nu-cli/src/completions/dotnu_completions.rs b/crates/nu-cli/src/completions/dotnu_completions.rs index 8927738491..c939578b41 100644 --- a/crates/nu-cli/src/completions/dotnu_completions.rs +++ b/crates/nu-cli/src/completions/dotnu_completions.rs @@ -1,39 +1,31 @@ use crate::completions::{file_path_completion, Completer, CompletionOptions, SortBy}; use nu_protocol::{ - engine::{EngineState, Stack, StateWorkingSet}, + engine::{Stack, StateWorkingSet}, Span, }; use reedline::Suggestion; -use std::{ - path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR}, - sync::Arc, -}; +use std::path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR}; use super::SemanticSuggestion; -#[derive(Clone)] -pub struct DotNuCompletion { - engine_state: Arc, - stack: Stack, -} +#[derive(Clone, Default)] +pub struct DotNuCompletion {} impl DotNuCompletion { - pub fn new(engine_state: Arc, stack: Stack) -> Self { - Self { - engine_state, - stack, - } + pub fn new() -> Self { + Self::default() } } impl Completer for DotNuCompletion { fn fetch( &mut self, - _: &StateWorkingSet, + working_set: &StateWorkingSet, + stack: &Stack, prefix: Vec, span: Span, offset: usize, - _: usize, + _pos: usize, options: &CompletionOptions, ) -> Vec { let prefix_str = String::from_utf8_lossy(&prefix).replace('`', ""); @@ -49,26 +41,25 @@ impl Completer for DotNuCompletion { let mut is_current_folder = false; // Fetch the lib dirs - let lib_dirs: Vec = - if let Some(lib_dirs) = self.engine_state.get_env_var("NU_LIB_DIRS") { - lib_dirs - .as_list() - .into_iter() - .flat_map(|it| { - it.iter().map(|x| { - x.to_path() - .expect("internal error: failed to convert lib path") - }) + let lib_dirs: Vec = if let Some(lib_dirs) = working_set.get_env_var("NU_LIB_DIRS") { + lib_dirs + .as_list() + .into_iter() + .flat_map(|it| { + it.iter().map(|x| { + x.to_path() + .expect("internal error: failed to convert lib path") }) - .map(|it| { - it.into_os_string() - .into_string() - .expect("internal error: failed to convert OS path") - }) - .collect() - } else { - vec![] - }; + }) + .map(|it| { + it.into_os_string() + .into_string() + .expect("internal error: failed to convert OS path") + }) + .collect() + } else { + vec![] + }; // Check if the base_dir is a folder // rsplit_once removes the separator @@ -85,7 +76,7 @@ impl Completer for DotNuCompletion { } else { // Fetch the current folder #[allow(deprecated)] - let current_folder = self.engine_state.current_work_dir(); + let current_folder = working_set.permanent_state.current_work_dir(); is_current_folder = true; // Add the current folder and the lib dirs into the @@ -104,8 +95,8 @@ impl Completer for DotNuCompletion { &partial, &search_dir, options, - self.engine_state.as_ref(), - &self.stack, + working_set.permanent_state, + stack, ); completions .into_iter() diff --git a/crates/nu-cli/src/completions/file_completions.rs b/crates/nu-cli/src/completions/file_completions.rs index 1a99c995db..f6205f6792 100644 --- a/crates/nu-cli/src/completions/file_completions.rs +++ b/crates/nu-cli/src/completions/file_completions.rs @@ -9,25 +9,16 @@ use nu_protocol::{ }; use nu_utils::IgnoreCaseExt; use reedline::Suggestion; -use std::{ - path::{Path, MAIN_SEPARATOR as SEP}, - sync::Arc, -}; +use std::path::{Path, MAIN_SEPARATOR as SEP}; use super::SemanticSuggestion; -#[derive(Clone)] -pub struct FileCompletion { - engine_state: Arc, - stack: Stack, -} +#[derive(Clone, Default)] +pub struct FileCompletion {} impl FileCompletion { - pub fn new(engine_state: Arc, stack: Stack) -> Self { - Self { - engine_state, - stack, - } + pub fn new() -> Self { + Self::default() } } @@ -35,10 +26,11 @@ impl Completer for FileCompletion { fn fetch( &mut self, working_set: &StateWorkingSet, + stack: &Stack, prefix: Vec, span: Span, offset: usize, - _: usize, + _pos: usize, options: &CompletionOptions, ) -> Vec { let AdjustView { @@ -52,10 +44,10 @@ impl Completer for FileCompletion { readjusted, span, &prefix, - &self.engine_state.current_work_dir(), + &working_set.permanent_state.current_work_dir(), options, - self.engine_state.as_ref(), - &self.stack, + working_set.permanent_state, + stack, ) .into_iter() .map(move |x| SemanticSuggestion { diff --git a/crates/nu-cli/src/completions/flag_completions.rs b/crates/nu-cli/src/completions/flag_completions.rs index 07cd89dc0a..b0dcc0963b 100644 --- a/crates/nu-cli/src/completions/flag_completions.rs +++ b/crates/nu-cli/src/completions/flag_completions.rs @@ -1,7 +1,7 @@ use crate::completions::{Completer, CompletionOptions}; use nu_protocol::{ ast::{Expr, Expression}, - engine::StateWorkingSet, + engine::{Stack, StateWorkingSet}, Span, }; use reedline::Suggestion; @@ -23,10 +23,11 @@ impl Completer for FlagCompletion { fn fetch( &mut self, working_set: &StateWorkingSet, + _stack: &Stack, prefix: Vec, span: Span, offset: usize, - _: usize, + _pos: usize, options: &CompletionOptions, ) -> Vec { // Check if it's a flag diff --git a/crates/nu-cli/src/completions/variable_completions.rs b/crates/nu-cli/src/completions/variable_completions.rs index b869e9f972..0572fe93c1 100644 --- a/crates/nu-cli/src/completions/variable_completions.rs +++ b/crates/nu-cli/src/completions/variable_completions.rs @@ -3,30 +3,20 @@ use crate::completions::{ }; use nu_engine::{column::get_columns, eval_variable}; use nu_protocol::{ - engine::{EngineState, Stack, StateWorkingSet}, + engine::{Stack, StateWorkingSet}, Span, Value, }; use reedline::Suggestion; -use std::{str, sync::Arc}; +use std::str; #[derive(Clone)] pub struct VariableCompletion { - engine_state: Arc, // TODO: Is engine state necessary? It's already a part of working set in fetch() - stack: Stack, var_context: (Vec, Vec>), // tuple with $var and the sublevels (.b.c.d) } impl VariableCompletion { - pub fn new( - engine_state: Arc, - stack: Stack, - var_context: (Vec, Vec>), - ) -> Self { - Self { - engine_state, - stack, - var_context, - } + pub fn new(var_context: (Vec, Vec>)) -> Self { + Self { var_context } } } @@ -34,10 +24,11 @@ impl Completer for VariableCompletion { fn fetch( &mut self, working_set: &StateWorkingSet, + stack: &Stack, prefix: Vec, span: Span, offset: usize, - _: usize, + _pos: usize, options: &CompletionOptions, ) -> Vec { let mut output = vec![]; @@ -54,7 +45,7 @@ impl Completer for VariableCompletion { if !var_str.is_empty() { // Completion for $env. if var_str == "$env" { - let env_vars = self.stack.get_env_vars(&self.engine_state); + let env_vars = stack.get_env_vars(working_set.permanent_state); // Return nested values if sublevels_count > 0 { @@ -110,8 +101,8 @@ impl Completer for VariableCompletion { if var_str == "$nu" { // Eval nu var if let Ok(nuval) = eval_variable( - &self.engine_state, - &self.stack, + working_set.permanent_state, + stack, nu_protocol::NU_VARIABLE_ID, nu_protocol::Span::new(current_span.start, current_span.end), ) { @@ -133,7 +124,7 @@ impl Completer for VariableCompletion { // Completion other variable types if let Some(var_id) = var_id { // Extract the variable value from the stack - let var = self.stack.get_var(var_id, Span::new(span.start, span.end)); + let var = stack.get_var(var_id, Span::new(span.start, span.end)); // If the value exists and it's of type Record if let Ok(value) = var { @@ -207,7 +198,11 @@ impl Completer for VariableCompletion { // Permanent state vars // for scope in &self.engine_state.scope { - for overlay_frame in self.engine_state.active_overlays(&removed_overlays).rev() { + for overlay_frame in working_set + .permanent_state + .active_overlays(&removed_overlays) + .rev() + { for v in &overlay_frame.vars { if options.match_algorithm.matches_u8_insensitive( options.case_sensitive, diff --git a/crates/nu-cli/src/config_files.rs b/crates/nu-cli/src/config_files.rs index 091fe7daa3..ec7ad2f412 100644 --- a/crates/nu-cli/src/config_files.rs +++ b/crates/nu-cli/src/config_files.rs @@ -1,12 +1,12 @@ use crate::util::eval_source; #[cfg(feature = "plugin")] use nu_path::canonicalize_with; -use nu_protocol::{ - engine::{EngineState, Stack, StateWorkingSet}, - report_error, HistoryFileFormat, PipelineData, -}; #[cfg(feature = "plugin")] -use nu_protocol::{ParseError, PluginRegistryFile, Spanned}; +use nu_protocol::{engine::StateWorkingSet, report_error, ParseError, PluginRegistryFile, Spanned}; +use nu_protocol::{ + engine::{EngineState, Stack}, + report_error_new, HistoryFileFormat, PipelineData, +}; #[cfg(feature = "plugin")] use nu_utils::utils::perf; use std::path::PathBuf; @@ -25,10 +25,9 @@ pub fn read_plugin_file( plugin_file: Option>, storage_path: &str, ) { + use nu_protocol::ShellError; use std::path::Path; - use nu_protocol::{report_error_new, ShellError}; - let span = plugin_file.as_ref().map(|s| s.span); // Check and warn + abort if this is a .nu plugin file @@ -239,13 +238,11 @@ pub fn eval_config_contents( match engine_state.cwd(Some(stack)) { Ok(cwd) => { if let Err(e) = engine_state.merge_env(stack, cwd) { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &e); + report_error_new(engine_state, &e); } } Err(e) => { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &e); + report_error_new(engine_state, &e); } } } @@ -266,8 +263,8 @@ pub(crate) fn get_history_path(storage_path: &str, mode: HistoryFileFormat) -> O #[cfg(feature = "plugin")] pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -> bool { use nu_protocol::{ - report_error_new, PluginExample, PluginIdentity, PluginRegistryItem, - PluginRegistryItemData, PluginSignature, ShellError, + PluginExample, PluginIdentity, PluginRegistryItem, PluginRegistryItemData, PluginSignature, + ShellError, }; use std::collections::BTreeMap; @@ -309,14 +306,15 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) - let mut engine_state = engine_state.clone(); let mut stack = Stack::new(); - if !eval_source( + if eval_source( &mut engine_state, &mut stack, &old_contents, &old_plugin_file_path.to_string_lossy(), PipelineData::Empty, false, - ) { + ) != 0 + { return false; } diff --git a/crates/nu-cli/src/eval_cmds.rs b/crates/nu-cli/src/eval_cmds.rs index 1e3cc70348..8fa3bf30e5 100644 --- a/crates/nu-cli/src/eval_cmds.rs +++ b/crates/nu-cli/src/eval_cmds.rs @@ -1,12 +1,12 @@ use log::info; -use miette::Result; use nu_engine::{convert_env_values, eval_block}; use nu_parser::parse; use nu_protocol::{ debugger::WithoutDebug, engine::{EngineState, Stack, StateWorkingSet}, - report_error, PipelineData, Spanned, Value, + report_error, PipelineData, ShellError, Spanned, Value, }; +use std::sync::Arc; /// Run a command (or commands) given to us by the user pub fn evaluate_commands( @@ -16,13 +16,9 @@ pub fn evaluate_commands( input: PipelineData, table_mode: Option, no_newline: bool, -) -> Result> { +) -> Result<(), ShellError> { // Translate environment variables from Strings to Values - if let Some(e) = convert_env_values(engine_state, stack) { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &e); - std::process::exit(1); - } + convert_env_values(engine_state, stack)?; // Parse the source code let (block, delta) = { @@ -41,7 +37,6 @@ pub fn evaluate_commands( if let Some(err) = working_set.parse_errors.first() { report_error(&working_set, err); - std::process::exit(1); } @@ -49,35 +44,27 @@ pub fn evaluate_commands( }; // Update permanent state - if let Err(err) = engine_state.merge_delta(delta) { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &err); - } + engine_state.merge_delta(delta)?; // Run the block - let exit_code = match eval_block::(engine_state, stack, &block, input) { - Ok(pipeline_data) => { - let mut config = engine_state.get_config().clone(); - if let Some(t_mode) = table_mode { - config.table_mode = t_mode.coerce_str()?.parse().unwrap_or_default(); - } - crate::eval_file::print_table_or_error( - engine_state, - stack, - pipeline_data, - &mut config, - no_newline, - ) - } - Err(err) => { - let working_set = StateWorkingSet::new(engine_state); + let pipeline = eval_block::(engine_state, stack, &block, input)?; - report_error(&working_set, &err); - std::process::exit(1); + if let PipelineData::Value(Value::Error { error, .. }, ..) = pipeline { + return Err(*error); + } + + if let Some(t_mode) = table_mode { + Arc::make_mut(&mut engine_state.config).table_mode = + t_mode.coerce_str()?.parse().unwrap_or_default(); + } + + if let Some(status) = pipeline.print(engine_state, stack, no_newline, false)? { + if status.code() != 0 { + std::process::exit(status.code()) } - }; + } info!("evaluate {}:{}:{}", file!(), line!(), column!()); - Ok(exit_code) + Ok(()) } diff --git a/crates/nu-cli/src/eval_file.rs b/crates/nu-cli/src/eval_file.rs index 8107de71a5..ff6ba36fe3 100644 --- a/crates/nu-cli/src/eval_file.rs +++ b/crates/nu-cli/src/eval_file.rs @@ -1,15 +1,14 @@ use crate::util::eval_source; use log::{info, trace}; -use miette::{IntoDiagnostic, Result}; use nu_engine::{convert_env_values, eval_block}; use nu_parser::parse; use nu_path::canonicalize_with; use nu_protocol::{ debugger::WithoutDebug, engine::{EngineState, Stack, StateWorkingSet}, - report_error, Config, PipelineData, ShellError, Span, Value, + report_error, PipelineData, ShellError, Span, Value, }; -use std::{io::Write, sync::Arc}; +use std::sync::Arc; /// Entry point for evaluating a file. /// @@ -21,73 +20,40 @@ pub fn evaluate_file( engine_state: &mut EngineState, stack: &mut Stack, input: PipelineData, -) -> Result<()> { +) -> Result<(), ShellError> { // Convert environment variables from Strings to Values and store them in the engine state. - if let Some(e) = convert_env_values(engine_state, stack) { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &e); - std::process::exit(1); - } + convert_env_values(engine_state, stack)?; let cwd = engine_state.cwd_as_string(Some(stack))?; - let file_path = canonicalize_with(&path, cwd).unwrap_or_else(|e| { - let working_set = StateWorkingSet::new(engine_state); - report_error( - &working_set, - &ShellError::FileNotFoundCustom { - msg: format!("Could not access file '{}': {:?}", path, e.to_string()), - span: Span::unknown(), - }, - ); - std::process::exit(1); - }); + let file_path = + canonicalize_with(&path, cwd).map_err(|err| ShellError::FileNotFoundCustom { + msg: format!("Could not access file '{path}': {err}"), + span: Span::unknown(), + })?; - let file_path_str = file_path.to_str().unwrap_or_else(|| { - let working_set = StateWorkingSet::new(engine_state); - report_error( - &working_set, - &ShellError::NonUtf8Custom { - msg: format!( - "Input file name '{}' is not valid UTF8", - file_path.to_string_lossy() - ), - span: Span::unknown(), - }, - ); - std::process::exit(1); - }); + let file_path_str = file_path + .to_str() + .ok_or_else(|| ShellError::NonUtf8Custom { + msg: format!( + "Input file name '{}' is not valid UTF8", + file_path.to_string_lossy() + ), + span: Span::unknown(), + })?; - let file = std::fs::read(&file_path) - .into_diagnostic() - .unwrap_or_else(|e| { - let working_set = StateWorkingSet::new(engine_state); - report_error( - &working_set, - &ShellError::FileNotFoundCustom { - msg: format!( - "Could not read file '{}': {:?}", - file_path_str, - e.to_string() - ), - span: Span::unknown(), - }, - ); - std::process::exit(1); - }); + let file = std::fs::read(&file_path).map_err(|err| ShellError::FileNotFoundCustom { + msg: format!("Could not read file '{file_path_str}': {err}"), + span: Span::unknown(), + })?; engine_state.file = Some(file_path.clone()); - let parent = file_path.parent().unwrap_or_else(|| { - let working_set = StateWorkingSet::new(engine_state); - report_error( - &working_set, - &ShellError::FileNotFoundCustom { - msg: format!("The file path '{file_path_str}' does not have a parent"), - span: Span::unknown(), - }, - ); - std::process::exit(1); - }); + let parent = file_path + .parent() + .ok_or_else(|| ShellError::FileNotFoundCustom { + msg: format!("The file path '{file_path_str}' does not have a parent"), + span: Span::unknown(), + })?; stack.add_env_var( "FILE_PWD".to_string(), @@ -127,119 +93,48 @@ pub fn evaluate_file( } // Merge the changes into the engine state. - engine_state - .merge_delta(working_set.delta) - .expect("merging delta into engine_state should succeed"); + engine_state.merge_delta(working_set.delta)?; // Check if the file contains a main command. - if engine_state.find_decl(b"main", &[]).is_some() { + let exit_code = if engine_state.find_decl(b"main", &[]).is_some() { // Evaluate the file, but don't run main yet. - let pipeline_data = - eval_block::(engine_state, stack, &block, PipelineData::empty()); - let pipeline_data = match pipeline_data { - Err(ShellError::Return { .. }) => { - // Allow early return before main is run. - return Ok(()); - } - x => x, - } - .unwrap_or_else(|e| { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &e); - std::process::exit(1); - }); - - // Print the pipeline output of the file. - // The pipeline output of a file is the pipeline output of its last command. - let result = pipeline_data.print(engine_state, stack, true, false); - match result { - Err(err) => { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &err); - std::process::exit(1); - } - Ok(exit_code) => { - if exit_code != 0 { - std::process::exit(exit_code as i32); + let pipeline = + match eval_block::(engine_state, stack, &block, PipelineData::empty()) { + Ok(data) => data, + Err(ShellError::Return { .. }) => { + // Allow early return before main is run. + return Ok(()); } + Err(err) => return Err(err), + }; + + // Print the pipeline output of the last command of the file. + if let Some(status) = pipeline.print(engine_state, stack, true, false)? { + if status.code() != 0 { + std::process::exit(status.code()) } } // Invoke the main command with arguments. // Arguments with whitespaces are quoted, thus can be safely concatenated by whitespace. let args = format!("main {}", args.join(" ")); - if !eval_source( + eval_source( engine_state, stack, args.as_bytes(), "", input, true, - ) { - std::process::exit(1); - } - } else if !eval_source(engine_state, stack, &file, file_path_str, input, true) { - std::process::exit(1); + ) + } else { + eval_source(engine_state, stack, &file, file_path_str, input, true) + }; + + if exit_code != 0 { + std::process::exit(exit_code) } info!("evaluate {}:{}:{}", file!(), line!(), column!()); Ok(()) } - -pub(crate) fn print_table_or_error( - engine_state: &mut EngineState, - stack: &mut Stack, - mut pipeline_data: PipelineData, - config: &mut Config, - no_newline: bool, -) -> Option { - let exit_code = match &mut pipeline_data { - PipelineData::ExternalStream { exit_code, .. } => exit_code.take(), - _ => None, - }; - - // Change the engine_state config to use the passed in configuration - engine_state.set_config(config.clone()); - - if let PipelineData::Value(Value::Error { error, .. }, ..) = &pipeline_data { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &**error); - std::process::exit(1); - } - - // We don't need to do anything special to print a table because print() handles it - print_or_exit(pipeline_data, engine_state, stack, no_newline); - - // Make sure everything has finished - if let Some(exit_code) = exit_code { - let mut exit_code: Vec<_> = exit_code.into_iter().collect(); - exit_code - .pop() - .and_then(|last_exit_code| match last_exit_code { - Value::Int { val: code, .. } => Some(code), - _ => None, - }) - } else { - None - } -} - -fn print_or_exit( - pipeline_data: PipelineData, - engine_state: &EngineState, - stack: &mut Stack, - no_newline: bool, -) { - let result = pipeline_data.print(engine_state, stack, no_newline, false); - - let _ = std::io::stdout().flush(); - let _ = std::io::stderr().flush(); - - if let Err(error) = result { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &error); - let _ = std::io::stderr().flush(); - std::process::exit(1); - } -} diff --git a/crates/nu-cli/src/menus/menu_completions.rs b/crates/nu-cli/src/menus/menu_completions.rs index fbfc598225..c65f0bd100 100644 --- a/crates/nu-cli/src/menus/menu_completions.rs +++ b/crates/nu-cli/src/menus/menu_completions.rs @@ -59,8 +59,7 @@ impl Completer for NuMenuCompleter { let res = eval_block::(&self.engine_state, &mut self.stack, block, input); - if let Ok(values) = res { - let values = values.into_value(self.span); + if let Ok(values) = res.and_then(|data| data.into_value(self.span)) { convert_to_suggestions(values, line, pos, self.only_buffer_difference) } else { Vec::new() diff --git a/crates/nu-cli/src/prompt.rs b/crates/nu-cli/src/prompt.rs index a2045a201c..744640b76f 100644 --- a/crates/nu-cli/src/prompt.rs +++ b/crates/nu-cli/src/prompt.rs @@ -129,9 +129,11 @@ impl Prompt for NushellPrompt { { // We're in vscode and we have osc633 enabled format!("{VSCODE_PRE_PROMPT_MARKER}{prompt}{VSCODE_POST_PROMPT_MARKER}").into() - } else { - // If we're in VSCode but we don't find the env var, just return the regular markers + } else if self.shell_integration_osc133 { + // If we're in VSCode but we don't find the env var, but we have osc133 set, then use it format!("{PRE_PROMPT_MARKER}{prompt}{POST_PROMPT_MARKER}").into() + } else { + prompt.into() } } else if self.shell_integration_osc133 { format!("{PRE_PROMPT_MARKER}{prompt}{POST_PROMPT_MARKER}").into() diff --git a/crates/nu-cli/src/prompt_update.rs b/crates/nu-cli/src/prompt_update.rs index 0c5641378b..0c24b8282a 100644 --- a/crates/nu-cli/src/prompt_update.rs +++ b/crates/nu-cli/src/prompt_update.rs @@ -2,8 +2,8 @@ use crate::NushellPrompt; use log::trace; use nu_engine::ClosureEvalOnce; use nu_protocol::{ - engine::{EngineState, Stack, StateWorkingSet}, - report_error, Config, PipelineData, Value, + engine::{EngineState, Stack}, + report_error_new, Config, PipelineData, Value, }; use reedline::Prompt; @@ -65,7 +65,7 @@ fn get_prompt_string( .get_env_var(engine_state, prompt) .and_then(|v| match v { Value::Closure { val, .. } => { - let result = ClosureEvalOnce::new(engine_state, stack, val) + let result = ClosureEvalOnce::new(engine_state, stack, *val) .run_with_input(PipelineData::Empty); trace!( @@ -77,8 +77,7 @@ fn get_prompt_string( result .map_err(|err| { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &err); + report_error_new(engine_state, &err); }) .ok() } @@ -108,50 +107,34 @@ pub(crate) fn update_prompt( stack: &mut Stack, nu_prompt: &mut NushellPrompt, ) { - let left_prompt_string = get_prompt_string(PROMPT_COMMAND, config, engine_state, stack); + let configured_left_prompt_string = + match get_prompt_string(PROMPT_COMMAND, config, engine_state, stack) { + Some(s) => s, + None => "".to_string(), + }; // Now that we have the prompt string lets ansify it. // <133 A><133 B><133 C> - let left_prompt_string_133 = if config.shell_integration_osc133 { - if let Some(prompt_string) = left_prompt_string.clone() { + let left_prompt_string = if config.shell_integration_osc633 { + if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) { + // We're in vscode and we have osc633 enabled Some(format!( - "{PRE_PROMPT_MARKER}{prompt_string}{POST_PROMPT_MARKER}" + "{VSCODE_PRE_PROMPT_MARKER}{configured_left_prompt_string}{VSCODE_POST_PROMPT_MARKER}" + )) + } else if config.shell_integration_osc133 { + // If we're in VSCode but we don't find the env var, but we have osc133 set, then use it + Some(format!( + "{PRE_PROMPT_MARKER}{configured_left_prompt_string}{POST_PROMPT_MARKER}" )) } else { - left_prompt_string.clone() + configured_left_prompt_string.into() } + } else if config.shell_integration_osc133 { + Some(format!( + "{PRE_PROMPT_MARKER}{configured_left_prompt_string}{POST_PROMPT_MARKER}" + )) } else { - left_prompt_string.clone() - }; - - let left_prompt_string_633 = if config.shell_integration_osc633 { - if let Some(prompt_string) = left_prompt_string.clone() { - if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) - { - // If the user enabled osc633 and we're in vscode, use the vscode markers - Some(format!( - "{VSCODE_PRE_PROMPT_MARKER}{prompt_string}{VSCODE_POST_PROMPT_MARKER}" - )) - } else { - // otherwise, use the regular osc133 markers - Some(format!( - "{PRE_PROMPT_MARKER}{prompt_string}{POST_PROMPT_MARKER}" - )) - } - } else { - left_prompt_string.clone() - } - } else { - left_prompt_string.clone() - }; - - let left_prompt_string = match (left_prompt_string_133, left_prompt_string_633) { - (None, None) => left_prompt_string, - (None, Some(l633)) => Some(l633), - (Some(l133), None) => Some(l133), - // If both are set, it means we're in vscode, so use the vscode markers - // and even if we're not actually in vscode atm, the regular 133 markers are used - (Some(_l133), Some(l633)) => Some(l633), + configured_left_prompt_string.into() }; let right_prompt_string = get_prompt_string(PROMPT_COMMAND_RIGHT, config, engine_state, stack); diff --git a/crates/nu-cli/src/repl.rs b/crates/nu-cli/src/repl.rs index 30cccebbde..29c2f62734 100644 --- a/crates/nu-cli/src/repl.rs +++ b/crates/nu-cli/src/repl.rs @@ -26,9 +26,8 @@ use nu_parser::{lex, parse, trim_quotes_str}; use nu_protocol::{ config::NuCursorShape, engine::{EngineState, Stack, StateWorkingSet}, - eval_const::create_nu_constant, report_error_new, HistoryConfig, HistoryFileFormat, PipelineData, ShellError, Span, Spanned, - Value, NU_VARIABLE_ID, + Value, }; use nu_utils::{ filesystem::{have_permission, PermissionResult}, @@ -87,7 +86,7 @@ pub fn evaluate_repl( let start_time = std::time::Instant::now(); // Translate environment variables from Strings to Values - if let Some(e) = convert_env_values(engine_state, &unique_stack) { + if let Err(e) = convert_env_values(engine_state, &unique_stack) { report_error_new(engine_state, &e); } perf( @@ -145,8 +144,7 @@ pub fn evaluate_repl( engine_state.set_startup_time(entire_start_time.elapsed().as_nanos() as i64); // Regenerate the $nu constant to contain the startup time and any other potential updates - let nu_const = create_nu_constant(engine_state, Span::unknown())?; - engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const); + engine_state.generate_nu_constant(); if load_std_lib.is_none() && engine_state.get_config().show_banner { eval_source( @@ -389,7 +387,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { .with_completer(Box::new(NuCompleter::new( engine_reference.clone(), // STACK-REFERENCE 2 - Stack::with_parent(stack_arc.clone()), + stack_arc.clone(), ))) .with_quick_completions(config.quick_completions) .with_partial_completions(config.partial_completions) @@ -544,7 +542,9 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { let shell_integration_osc633 = config.shell_integration_osc633; let shell_integration_reset_application_mode = config.shell_integration_reset_application_mode; - let mut stack = Stack::unwrap_unique(stack_arc); + // TODO: we may clone the stack, this can lead to major performance issues + // so we should avoid it or making stack cheaper to clone. + let mut stack = Arc::unwrap_or_clone(stack_arc); perf( "line_editor setup", @@ -620,7 +620,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { column!(), use_color, ); - } else { + } else if shell_integration_osc133 { start_time = Instant::now(); run_ansi_sequence(PRE_EXECUTION_MARKER); @@ -660,9 +660,9 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { run_finaliziation_ansi_sequence( &stack, engine_state, + use_color, shell_integration_osc633, shell_integration_osc133, - use_color, ); } ReplOperation::RunCommand(cmd) => { @@ -679,9 +679,9 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { run_finaliziation_ansi_sequence( &stack, engine_state, + use_color, shell_integration_osc633, shell_integration_osc133, - use_color, ); } // as the name implies, we do nothing in this case @@ -731,9 +731,9 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { run_finaliziation_ansi_sequence( &stack, engine_state, + use_color, shell_integration_osc633, shell_integration_osc133, - use_color, ); } Ok(Signal::CtrlD) => { @@ -742,9 +742,9 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { run_finaliziation_ansi_sequence( &stack, engine_state, + use_color, shell_integration_osc633, shell_integration_osc133, - use_color, ); println!(); @@ -763,9 +763,9 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { run_finaliziation_ansi_sequence( &stack, engine_state, + use_color, shell_integration_osc633, shell_integration_osc133, - use_color, ); } } @@ -872,7 +872,7 @@ fn parse_operation( let tokens = lex(s.as_bytes(), 0, &[], &[], false); // Check if this is a single call to a directory, if so auto-cd #[allow(deprecated)] - let cwd = nu_engine::env::current_dir_str(engine_state, stack)?; + let cwd = nu_engine::env::current_dir_str(engine_state, stack).unwrap_or_default(); let mut orig = s.clone(); if orig.starts_with('`') { orig = trim_quotes_str(&orig).to_string() @@ -929,7 +929,10 @@ fn do_auto_cd( //FIXME: this only changes the current scope, but instead this environment variable //should probably be a block that loads the information from the state in the overlay - stack.add_env_var("PWD".into(), Value::string(path.clone(), Span::unknown())); + if let Err(err) = stack.set_cwd(&path) { + report_error_new(engine_state, &err); + return; + }; let cwd = Value::string(cwd, span); let shells = stack.get_env_var(engine_state, "NUSHELL_SHELLS"); @@ -1298,27 +1301,46 @@ fn map_nucursorshape_to_cursorshape(shape: NuCursorShape) -> Option String { +fn get_command_finished_marker( + stack: &Stack, + engine_state: &EngineState, + shell_integration_osc633: bool, + shell_integration_osc133: bool, +) -> String { let exit_code = stack .get_env_var(engine_state, "LAST_EXIT_CODE") .and_then(|e| e.as_i64().ok()); - if vscode { - // format!("\x1b]633;D;{}\x1b\\", exit_code.unwrap_or(0)) - format!( - "{}{}{}", - VSCODE_POST_EXECUTION_MARKER_PREFIX, - exit_code.unwrap_or(0), - VSCODE_POST_EXECUTION_MARKER_SUFFIX - ) - } else { - // format!("\x1b]133;D;{}\x1b\\", exit_code.unwrap_or(0)) + if shell_integration_osc633 { + if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) { + // We're in vscode and we have osc633 enabled + format!( + "{}{}{}", + VSCODE_POST_EXECUTION_MARKER_PREFIX, + exit_code.unwrap_or(0), + VSCODE_POST_EXECUTION_MARKER_SUFFIX + ) + } else if shell_integration_osc133 { + // If we're in VSCode but we don't find the env var, just return the regular markers + format!( + "{}{}{}", + POST_EXECUTION_MARKER_PREFIX, + exit_code.unwrap_or(0), + POST_EXECUTION_MARKER_SUFFIX + ) + } else { + // We're not in vscode, so we don't need to do anything special + "\x1b[0m".to_string() + } + } else if shell_integration_osc133 { format!( "{}{}{}", POST_EXECUTION_MARKER_PREFIX, exit_code.unwrap_or(0), POST_EXECUTION_MARKER_SUFFIX ) + } else { + "\x1b[0m".to_string() } } @@ -1342,7 +1364,12 @@ fn run_finaliziation_ansi_sequence( if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) { let start_time = Instant::now(); - run_ansi_sequence(&get_command_finished_marker(stack, engine_state, true)); + run_ansi_sequence(&get_command_finished_marker( + stack, + engine_state, + shell_integration_osc633, + shell_integration_osc133, + )); perf( "post_execute_marker (633;D) ansi escape sequences", @@ -1352,10 +1379,15 @@ fn run_finaliziation_ansi_sequence( column!(), use_color, ); - } else { + } else if shell_integration_osc133 { let start_time = Instant::now(); - run_ansi_sequence(&get_command_finished_marker(stack, engine_state, false)); + run_ansi_sequence(&get_command_finished_marker( + stack, + engine_state, + shell_integration_osc633, + shell_integration_osc133, + )); perf( "post_execute_marker (133;D) ansi escape sequences", @@ -1369,7 +1401,12 @@ fn run_finaliziation_ansi_sequence( } else if shell_integration_osc133 { let start_time = Instant::now(); - run_ansi_sequence(&get_command_finished_marker(stack, engine_state, false)); + run_ansi_sequence(&get_command_finished_marker( + stack, + engine_state, + shell_integration_osc633, + shell_integration_osc133, + )); perf( "post_execute_marker (133;D) ansi escape sequences", @@ -1447,3 +1484,136 @@ fn are_session_ids_in_sync() { engine_state.history_session_id ); } + +#[cfg(test)] +mod test_auto_cd { + use super::{do_auto_cd, parse_operation, ReplOperation}; + use nu_protocol::engine::{EngineState, Stack}; + use std::path::Path; + use tempfile::tempdir; + + /// Create a symlink. Works on both Unix and Windows. + #[cfg(any(unix, windows))] + fn symlink(original: impl AsRef, link: impl AsRef) -> std::io::Result<()> { + #[cfg(unix)] + { + std::os::unix::fs::symlink(original, link) + } + #[cfg(windows)] + { + if original.as_ref().is_dir() { + std::os::windows::fs::symlink_dir(original, link) + } else { + std::os::windows::fs::symlink_file(original, link) + } + } + } + + /// Run one test case on the auto-cd feature. PWD is initially set to + /// `before`, and after `input` is parsed and evaluated, PWD should be + /// changed to `after`. + #[track_caller] + fn check(before: impl AsRef, input: &str, after: impl AsRef) { + // Setup EngineState and Stack. + let mut engine_state = EngineState::new(); + let mut stack = Stack::new(); + stack.set_cwd(before).unwrap(); + + // Parse the input. It must be an auto-cd operation. + let op = parse_operation(input.to_string(), &engine_state, &stack).unwrap(); + let ReplOperation::AutoCd { cwd, target, span } = op else { + panic!("'{}' was not parsed into an auto-cd operation", input) + }; + + // Perform the auto-cd operation. + do_auto_cd(target, cwd, &mut stack, &mut engine_state, span); + let updated_cwd = engine_state.cwd(Some(&stack)).unwrap(); + + // Check that `updated_cwd` and `after` point to the same place. They + // don't have to be byte-wise equal (on Windows, the 8.3 filename + // conversion messes things up), + let updated_cwd = std::fs::canonicalize(updated_cwd).unwrap(); + let after = std::fs::canonicalize(after).unwrap(); + assert_eq!(updated_cwd, after); + } + + #[test] + fn auto_cd_root() { + let tempdir = tempdir().unwrap(); + let root = if cfg!(windows) { r"C:\" } else { "/" }; + check(&tempdir, root, root); + } + + #[test] + fn auto_cd_tilde() { + let tempdir = tempdir().unwrap(); + let home = nu_path::home_dir().unwrap(); + check(&tempdir, "~", home); + } + + #[test] + fn auto_cd_dot() { + let tempdir = tempdir().unwrap(); + check(&tempdir, ".", &tempdir); + } + + #[test] + fn auto_cd_double_dot() { + let tempdir = tempdir().unwrap(); + let dir = tempdir.path().join("foo"); + std::fs::create_dir_all(&dir).unwrap(); + check(dir, "..", &tempdir); + } + + #[test] + fn auto_cd_triple_dot() { + let tempdir = tempdir().unwrap(); + let dir = tempdir.path().join("foo").join("bar"); + std::fs::create_dir_all(&dir).unwrap(); + check(dir, "...", &tempdir); + } + + #[test] + fn auto_cd_relative() { + let tempdir = tempdir().unwrap(); + let foo = tempdir.path().join("foo"); + let bar = tempdir.path().join("bar"); + std::fs::create_dir_all(&foo).unwrap(); + std::fs::create_dir_all(&bar).unwrap(); + + let input = if cfg!(windows) { r"..\bar" } else { "../bar" }; + check(foo, input, bar); + } + + #[test] + fn auto_cd_trailing_slash() { + let tempdir = tempdir().unwrap(); + let dir = tempdir.path().join("foo"); + std::fs::create_dir_all(&dir).unwrap(); + + let input = if cfg!(windows) { r"foo\" } else { "foo/" }; + check(&tempdir, input, dir); + } + + #[test] + fn auto_cd_symlink() { + let tempdir = tempdir().unwrap(); + let dir = tempdir.path().join("foo"); + std::fs::create_dir_all(&dir).unwrap(); + let link = tempdir.path().join("link"); + symlink(&dir, &link).unwrap(); + + let input = if cfg!(windows) { r".\link" } else { "./link" }; + check(&tempdir, input, link); + } + + #[test] + #[should_panic(expected = "was not parsed into an auto-cd operation")] + fn auto_cd_nonexistent_directory() { + let tempdir = tempdir().unwrap(); + let dir = tempdir.path().join("foo"); + + let input = if cfg!(windows) { r"foo\" } else { "foo/" }; + check(&tempdir, input, dir); + } +} diff --git a/crates/nu-cli/src/util.rs b/crates/nu-cli/src/util.rs index 8ff4ef35ea..7ebea0deb2 100644 --- a/crates/nu-cli/src/util.rs +++ b/crates/nu-cli/src/util.rs @@ -4,7 +4,7 @@ use nu_parser::{escape_quote_string, lex, parse, unescape_unquote_string, Token, use nu_protocol::{ debugger::WithoutDebug, engine::{EngineState, Stack, StateWorkingSet}, - print_if_stream, report_error, report_error_new, PipelineData, ShellError, Span, Value, + report_error, report_error_new, PipelineData, ShellError, Span, Value, }; #[cfg(windows)] use nu_utils::enable_vt_processing; @@ -39,9 +39,8 @@ fn gather_env_vars( init_cwd: &Path, ) { fn report_capture_error(engine_state: &EngineState, env_str: &str, msg: &str) { - let working_set = StateWorkingSet::new(engine_state); - report_error( - &working_set, + report_error_new( + engine_state, &ShellError::GenericError { error: format!("Environment variable was not captured: {env_str}"), msg: "".into(), @@ -71,9 +70,8 @@ fn gather_env_vars( } None => { // Could not capture current working directory - let working_set = StateWorkingSet::new(engine_state); - report_error( - &working_set, + report_error_new( + engine_state, &ShellError::GenericError { error: "Current directory is not a valid utf-8 path".into(), msg: "".into(), @@ -208,9 +206,48 @@ pub fn eval_source( fname: &str, input: PipelineData, allow_return: bool, -) -> bool { +) -> i32 { let start_time = std::time::Instant::now(); + let exit_code = match evaluate_source(engine_state, stack, source, fname, input, allow_return) { + Ok(code) => code.unwrap_or(0), + Err(err) => { + report_error_new(engine_state, &err); + 1 + } + }; + + stack.add_env_var( + "LAST_EXIT_CODE".to_string(), + Value::int(exit_code.into(), Span::unknown()), + ); + + // reset vt processing, aka ansi because illbehaved externals can break it + #[cfg(windows)] + { + let _ = enable_vt_processing(); + } + + perf( + &format!("eval_source {}", &fname), + start_time, + file!(), + line!(), + column!(), + engine_state.get_config().use_ansi_coloring, + ); + + exit_code +} + +fn evaluate_source( + engine_state: &mut EngineState, + stack: &mut Stack, + source: &[u8], + fname: &str, + input: PipelineData, + allow_return: bool, +) -> Result, ShellError> { let (block, delta) = { let mut working_set = StateWorkingSet::new(engine_state); let output = parse( @@ -224,104 +261,40 @@ pub fn eval_source( } if let Some(err) = working_set.parse_errors.first() { - set_last_exit_code(stack, 1); report_error(&working_set, err); - return false; + return Ok(Some(1)); } (output, working_set.render()) }; - if let Err(err) = engine_state.merge_delta(delta) { - set_last_exit_code(stack, 1); - report_error_new(engine_state, &err); - return false; - } + engine_state.merge_delta(delta)?; - let b = if allow_return { + let pipeline = if allow_return { eval_block_with_early_return::(engine_state, stack, &block, input) } else { eval_block::(engine_state, stack, &block, input) + }?; + + let status = if let PipelineData::ByteStream(stream, ..) = pipeline { + stream.print(false)? + } else { + if let Some(hook) = engine_state.get_config().hooks.display_output.clone() { + let pipeline = eval_hook( + engine_state, + stack, + Some(pipeline), + vec![], + &hook, + "display_output", + )?; + pipeline.print(engine_state, stack, false, false) + } else { + pipeline.print(engine_state, stack, true, false) + }? }; - match b { - Ok(pipeline_data) => { - let config = engine_state.get_config(); - let result; - if let PipelineData::ExternalStream { - stdout: stream, - stderr: stderr_stream, - exit_code, - .. - } = pipeline_data - { - result = print_if_stream(stream, stderr_stream, false, exit_code); - } else if let Some(hook) = config.hooks.display_output.clone() { - match eval_hook( - engine_state, - stack, - Some(pipeline_data), - vec![], - &hook, - "display_output", - ) { - Err(err) => { - result = Err(err); - } - Ok(val) => { - result = val.print(engine_state, stack, false, false); - } - } - } else { - result = pipeline_data.print(engine_state, stack, true, false); - } - - match result { - Err(err) => { - let working_set = StateWorkingSet::new(engine_state); - - report_error(&working_set, &err); - - return false; - } - Ok(exit_code) => { - set_last_exit_code(stack, exit_code); - } - } - - // reset vt processing, aka ansi because illbehaved externals can break it - #[cfg(windows)] - { - let _ = enable_vt_processing(); - } - } - Err(err) => { - set_last_exit_code(stack, 1); - - let working_set = StateWorkingSet::new(engine_state); - - report_error(&working_set, &err); - - return false; - } - } - perf( - &format!("eval_source {}", &fname), - start_time, - file!(), - line!(), - column!(), - engine_state.get_config().use_ansi_coloring, - ); - - true -} - -fn set_last_exit_code(stack: &mut Stack, exit_code: i64) { - stack.add_env_var( - "LAST_EXIT_CODE".to_string(), - Value::int(exit_code, Span::unknown()), - ); + Ok(status.map(|status| status.code())) } #[cfg(test)] diff --git a/crates/nu-cli/tests/commands/mod.rs b/crates/nu-cli/tests/commands/mod.rs new file mode 100644 index 0000000000..00488f0b9e --- /dev/null +++ b/crates/nu-cli/tests/commands/mod.rs @@ -0,0 +1 @@ +mod nu_highlight; diff --git a/crates/nu-cli/tests/commands/nu_highlight.rs b/crates/nu-cli/tests/commands/nu_highlight.rs new file mode 100644 index 0000000000..bd185a8634 --- /dev/null +++ b/crates/nu-cli/tests/commands/nu_highlight.rs @@ -0,0 +1,7 @@ +use nu_test_support::nu; + +#[test] +fn nu_highlight_not_expr() { + let actual = nu!("'not false' | nu-highlight | ansi strip"); + assert_eq!(actual.out, "not false"); +} diff --git a/crates/nu-cli/tests/completions.rs b/crates/nu-cli/tests/completions/mod.rs similarity index 94% rename from crates/nu-cli/tests/completions.rs rename to crates/nu-cli/tests/completions/mod.rs index a22d770010..cb883b67db 100644 --- a/crates/nu-cli/tests/completions.rs +++ b/crates/nu-cli/tests/completions/mod.rs @@ -6,7 +6,10 @@ use nu_parser::parse; use nu_protocol::{debugger::WithoutDebug, engine::StateWorkingSet, PipelineData}; use reedline::{Completer, Suggestion}; use rstest::{fixture, rstest}; -use std::path::{PathBuf, MAIN_SEPARATOR}; +use std::{ + path::{PathBuf, MAIN_SEPARATOR}, + sync::Arc, +}; use support::{ completions_helpers::{new_partial_engine, new_quote_engine}, file, folder, match_suggestions, new_engine, @@ -22,7 +25,7 @@ fn completer() -> NuCompleter { assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok()); // Instantiate a new completer - NuCompleter::new(std::sync::Arc::new(engine), stack) + NuCompleter::new(Arc::new(engine), Arc::new(stack)) } #[fixture] @@ -36,7 +39,7 @@ fn completer_strings() -> NuCompleter { assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok()); // Instantiate a new completer - NuCompleter::new(std::sync::Arc::new(engine), stack) + NuCompleter::new(Arc::new(engine), Arc::new(stack)) } #[fixture] @@ -56,7 +59,7 @@ fn extern_completer() -> NuCompleter { assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok()); // Instantiate a new completer - NuCompleter::new(std::sync::Arc::new(engine), stack) + NuCompleter::new(Arc::new(engine), Arc::new(stack)) } #[fixture] @@ -79,14 +82,14 @@ fn custom_completer() -> NuCompleter { assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok()); // Instantiate a new completer - NuCompleter::new(std::sync::Arc::new(engine), stack) + NuCompleter::new(Arc::new(engine), Arc::new(stack)) } #[test] fn variables_dollar_sign_with_varialblecompletion() { let (_, _, engine, stack) = new_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let target_dir = "$ "; let suggestions = completer.complete(target_dir, target_dir.len()); @@ -138,7 +141,7 @@ fn dotnu_completions() { let (_, _, engine, stack) = new_engine(); // Instantiate a new completer - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); // Test source completion let completion_str = "source-env ".to_string(); @@ -217,7 +220,7 @@ fn file_completions() { let (dir, dir_str, engine, stack) = new_engine(); // Instantiate a new completer - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); // Test completions for the current folder let target_dir = format!("cp {dir_str}{MAIN_SEPARATOR}"); @@ -265,7 +268,7 @@ fn partial_completions() { let (dir, _, engine, stack) = new_partial_engine(); // Instantiate a new completer - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); // Test completions for a folder's name let target_dir = format!("cd {}", file(dir.join("pa"))); @@ -363,7 +366,7 @@ fn partial_completions() { fn command_ls_with_filecompletion() { let (_, _, engine, stack) = new_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let target_dir = "ls "; let suggestions = completer.complete(target_dir, target_dir.len()); @@ -397,7 +400,7 @@ fn command_ls_with_filecompletion() { fn command_open_with_filecompletion() { let (_, _, engine, stack) = new_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let target_dir = "open "; let suggestions = completer.complete(target_dir, target_dir.len()); @@ -432,7 +435,7 @@ fn command_open_with_filecompletion() { fn command_rm_with_globcompletion() { let (_, _, engine, stack) = new_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let target_dir = "rm "; let suggestions = completer.complete(target_dir, target_dir.len()); @@ -467,7 +470,7 @@ fn command_rm_with_globcompletion() { fn command_cp_with_globcompletion() { let (_, _, engine, stack) = new_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let target_dir = "cp "; let suggestions = completer.complete(target_dir, target_dir.len()); @@ -502,7 +505,7 @@ fn command_cp_with_globcompletion() { fn command_save_with_filecompletion() { let (_, _, engine, stack) = new_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let target_dir = "save "; let suggestions = completer.complete(target_dir, target_dir.len()); @@ -537,7 +540,7 @@ fn command_save_with_filecompletion() { fn command_touch_with_filecompletion() { let (_, _, engine, stack) = new_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let target_dir = "touch "; let suggestions = completer.complete(target_dir, target_dir.len()); @@ -572,7 +575,7 @@ fn command_touch_with_filecompletion() { fn command_watch_with_filecompletion() { let (_, _, engine, stack) = new_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let target_dir = "watch "; let suggestions = completer.complete(target_dir, target_dir.len()); @@ -607,7 +610,7 @@ fn command_watch_with_filecompletion() { fn file_completion_quoted() { let (_, _, engine, stack) = new_quote_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let target_dir = "open "; let suggestions = completer.complete(target_dir, target_dir.len()); @@ -645,7 +648,7 @@ fn flag_completions() { let (_, _, engine, stack) = new_engine(); // Instantiate a new completer - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); // Test completions for the 'ls' flags let suggestions = completer.complete("ls -", 4); @@ -680,7 +683,7 @@ fn folder_with_directorycompletions() { let (dir, dir_str, engine, stack) = new_engine(); // Instantiate a new completer - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); // Test completions for the current folder let target_dir = format!("cd {dir_str}{MAIN_SEPARATOR}"); @@ -709,7 +712,7 @@ fn variables_completions() { assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok()); // Instantiate a new completer - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); // Test completions for $nu let suggestions = completer.complete("$nu.", 4); @@ -815,7 +818,7 @@ fn alias_of_command_and_flags() { let alias = r#"alias ll = ls -l"#; assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok()); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let suggestions = completer.complete("ll t", 4); #[cfg(windows)] @@ -834,7 +837,7 @@ fn alias_of_basic_command() { let alias = r#"alias ll = ls "#; assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok()); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let suggestions = completer.complete("ll t", 4); #[cfg(windows)] @@ -856,7 +859,7 @@ fn alias_of_another_alias() { let alias = r#"alias lf = ll -f"#; assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok()); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let suggestions = completer.complete("lf t", 4); #[cfg(windows)] @@ -890,7 +893,7 @@ fn run_external_completion(completer: &str, input: &str) -> Vec { assert!(engine_state.merge_env(&mut stack, &dir).is_ok()); // Instantiate a new completer - let mut completer = NuCompleter::new(std::sync::Arc::new(engine_state), stack); + let mut completer = NuCompleter::new(Arc::new(engine_state), Arc::new(stack)); completer.complete(input, input.len()) } @@ -899,7 +902,7 @@ fn run_external_completion(completer: &str, input: &str) -> Vec { fn unknown_command_completion() { let (_, _, engine, stack) = new_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let target_dir = "thiscommanddoesnotexist "; let suggestions = completer.complete(target_dir, target_dir.len()); @@ -962,7 +965,7 @@ fn flagcompletion_triggers_after_cursor_piped(mut completer: NuCompleter) { fn filecompletions_triggers_after_cursor() { let (_, _, engine, stack) = new_engine(); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); let suggestions = completer.complete("cp test_c", 3); @@ -1071,7 +1074,7 @@ fn alias_offset_bug_7648() { let alias = r#"alias ea = ^$env.EDITOR /tmp/test.s"#; assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok()); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); // Issue #7648 // Nushell crashes when an alias name is shorter than the alias command @@ -1090,7 +1093,7 @@ fn alias_offset_bug_7754() { let alias = r#"alias ll = ls -l"#; assert!(support::merge_input(alias.as_bytes(), &mut engine, &mut stack, dir).is_ok()); - let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack); + let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); // Issue #7754 // Nushell crashes when an alias name is shorter than the alias command diff --git a/crates/nu-cli/tests/support/completions_helpers.rs b/crates/nu-cli/tests/completions/support/completions_helpers.rs similarity index 95% rename from crates/nu-cli/tests/support/completions_helpers.rs rename to crates/nu-cli/tests/completions/support/completions_helpers.rs index bdc52739ee..47f46ab00e 100644 --- a/crates/nu-cli/tests/support/completions_helpers.rs +++ b/crates/nu-cli/tests/completions/support/completions_helpers.rs @@ -3,8 +3,7 @@ use nu_parser::parse; use nu_protocol::{ debugger::WithoutDebug, engine::{EngineState, Stack, StateWorkingSet}, - eval_const::create_nu_constant, - PipelineData, ShellError, Span, Value, NU_VARIABLE_ID, + PipelineData, ShellError, Span, Value, }; use nu_test_support::fs; use reedline::Suggestion; @@ -28,9 +27,7 @@ pub fn new_engine() -> (PathBuf, String, EngineState, Stack) { let mut engine_state = create_default_context(); // Add $nu - let nu_const = - create_nu_constant(&engine_state, Span::test_data()).expect("Failed creating $nu"); - engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const); + engine_state.generate_nu_constant(); // New stack let mut stack = Stack::new(); diff --git a/crates/nu-cli/tests/support/mod.rs b/crates/nu-cli/tests/completions/support/mod.rs similarity index 100% rename from crates/nu-cli/tests/support/mod.rs rename to crates/nu-cli/tests/completions/support/mod.rs diff --git a/crates/nu-cli/tests/main.rs b/crates/nu-cli/tests/main.rs new file mode 100644 index 0000000000..a040a731f3 --- /dev/null +++ b/crates/nu-cli/tests/main.rs @@ -0,0 +1,2 @@ +mod commands; +mod completions; diff --git a/crates/nu-cmd-base/src/util.rs b/crates/nu-cmd-base/src/util.rs index 619237a21c..9c63dec836 100644 --- a/crates/nu-cmd-base/src/util.rs +++ b/crates/nu-cmd-base/src/util.rs @@ -1,6 +1,6 @@ use nu_protocol::{ - engine::{EngineState, Stack, StateWorkingSet}, - report_error, Range, ShellError, Span, Value, + engine::{EngineState, Stack}, + Range, ShellError, Span, Value, }; use std::{ops::Bound, path::PathBuf}; @@ -13,11 +13,9 @@ pub fn get_init_cwd() -> PathBuf { } pub fn get_guaranteed_cwd(engine_state: &EngineState, stack: &Stack) -> PathBuf { - engine_state.cwd(Some(stack)).unwrap_or_else(|e| { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &e); - crate::util::get_init_cwd() - }) + engine_state + .cwd(Some(stack)) + .unwrap_or(crate::util::get_init_cwd()) } type MakeRangeError = fn(&str, Span) -> ShellError; diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs index c170f8db8a..be9c33a229 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs @@ -79,7 +79,7 @@ impl Command for CastDF { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let (dtype, column_nm) = df_args(engine_state, stack, call)?; let df = NuLazyFrame::try_from_value(value)?; diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs index 3793945181..e0e94d10a0 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs @@ -72,8 +72,7 @@ impl Command for FilterWith { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; command_lazy(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs index 70160b3005..14c86e8c40 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs @@ -86,7 +86,7 @@ impl Command for FirstDF { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuDataFrame::try_from_value(value)?; command(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs index a0a188471d..ff2c4f98a2 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs @@ -61,7 +61,7 @@ impl Command for LastDF { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuDataFrame::try_from_value(value)?; command(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs index 5167a0c968..0cb75f34f2 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs @@ -109,8 +109,7 @@ impl Command for RenameDF { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; command_lazy(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs index 73dadacb2b..a6ab42052c 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs @@ -76,7 +76,7 @@ impl Command for ToNu { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { dataframe_command(engine_state, stack, call, value) } else { diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs index 52ceefceb4..79d3427e8a 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs @@ -102,8 +102,7 @@ impl Command for WithColumn { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; command_lazy(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs index b2d79be010..4cc56e030b 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs @@ -172,7 +172,7 @@ macro_rules! lazy_expr_command { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let lazy = NuLazyFrame::try_from_value(value)?; let lazy = NuLazyFrame::new( @@ -271,7 +271,7 @@ macro_rules! lazy_expr_command { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let lazy = NuLazyFrame::try_from_value(value)?; let lazy = NuLazyFrame::new( diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs index 0ba507f97f..eb97c575b7 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs @@ -91,7 +91,7 @@ impl Command for ExprOtherwise { let otherwise_predicate: Value = call.req(engine_state, stack, 0)?; let otherwise_predicate = NuExpression::try_from_value(otherwise_predicate)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let complete: NuExpression = match NuWhen::try_from_value(value)? { NuWhen::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(), NuWhen::ChainedThen(chained_when) => chained_when diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs index d82a0faf0a..aaa1029ee9 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs @@ -67,7 +67,7 @@ impl Command for ExprQuantile { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let quantile: f64 = call.req(engine_state, stack, 0)?; let expr = NuExpression::try_from_value(value)?; diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs index d70fd00825..5a6aad2de7 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs @@ -103,7 +103,7 @@ impl Command for ExprWhen { let then_predicate: Value = call.req(engine_state, stack, 1)?; let then_predicate = NuExpression::try_from_value(then_predicate)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let when_then: NuWhen = match value { Value::Nothing { .. } => when(when_predicate.into_polars()) .then(then_predicate.into_polars()) diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs index 8e32ae8040..a027e84d36 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs @@ -100,7 +100,7 @@ impl Command for LazyExplode { } pub(crate) fn explode(call: &Call, input: PipelineData) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; let columns: Vec = call diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs index a9a1eb1590..4c75f1d9a3 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs @@ -82,7 +82,7 @@ impl Command for LazyFillNA { input: PipelineData, ) -> Result { let fill: Value = call.req(engine_state, stack, 0)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuExpression::can_downcast(&value) { let expr = NuExpression::try_from_value(value)?; diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs index b3d35d2b8d..88be2a9e88 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs @@ -59,7 +59,7 @@ impl Command for LazyFillNull { input: PipelineData, ) -> Result { let fill: Value = call.req(engine_state, stack, 0)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuExpression::can_downcast(&value) { let expr = NuExpression::try_from_value(value)?; diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs index 7f7d1ab66b..4ae297acfd 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs @@ -219,7 +219,7 @@ impl Command for LazyJoin { let suffix: Option = call.get_flag(engine_state, stack, "suffix")?; let suffix = suffix.unwrap_or_else(|| "_x".into()); - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value(value)?; let from_eager = lazy.from_eager; let lazy = lazy.into_polars(); diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs index d17a444f49..ac8ec590c6 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs @@ -54,7 +54,7 @@ impl Command for LazyQuantile { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let quantile: f64 = call.req(engine_state, stack, 0)?; let lazy = NuLazyFrame::try_from_value(value)?; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs index ce66f69877..4ed33ce951 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs @@ -68,7 +68,7 @@ impl Command for IsNotNull { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuDataFrame::try_from_value(value)?; command(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs index d7921da347..b99d48af66 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs @@ -68,7 +68,7 @@ impl Command for IsNull { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuDataFrame::try_from_value(value)?; command(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs index b23ab4e20d..c6d6e829f8 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs @@ -60,7 +60,7 @@ impl Command for NUnique { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) { let df = NuDataFrame::try_from_value(value)?; command(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs b/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs index bf842840b4..2f40cf0a45 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs @@ -56,8 +56,7 @@ impl Command for Shift { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; command_lazy(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs index 13012b4fb3..1bc2e0dc1b 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs @@ -72,8 +72,7 @@ impl Command for Unique { call: &Call, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value(value)?; command_lazy(engine_state, stack, call, df) diff --git a/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs b/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs index d6febf7e43..39c30be9dd 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs @@ -80,7 +80,8 @@ pub fn test_dataframe_example(engine_state: &mut Box, example: &Exa let result = eval_block::(engine_state, &mut stack, &block, PipelineData::empty()) .unwrap_or_else(|err| panic!("test eval error in `{}`: {:?}", example.example, err)) - .into_value(Span::test_data()); + .into_value(Span::test_data()) + .expect("ok value"); println!("input: {}", example.example); println!("result: {result:?}"); diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/between_values.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/between_values.rs index fab201cf9a..74a484825a 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/between_values.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/between_values.rs @@ -1,7 +1,7 @@ use super::{operations::Axis, NuDataFrame}; use nu_protocol::{ ast::{Boolean, Comparison, Math, Operator}, - span, ShellError, Span, Spanned, Value, + ShellError, Span, Spanned, Value, }; use num::Zero; use polars::prelude::{ @@ -17,7 +17,7 @@ pub(super) fn between_dataframes( right: &Value, rhs: &NuDataFrame, ) -> Result { - let operation_span = span(&[left.span(), right.span()]); + let operation_span = Span::merge(left.span(), right.span()); match operator.item { Operator::Math(Math::Plus) => match lhs.append_df(rhs, Axis::Row, operation_span) { Ok(df) => Ok(df.into_value(operation_span)), @@ -40,7 +40,7 @@ pub(super) fn compute_between_series( right: &Value, rhs: &Series, ) -> Result { - let operation_span = span(&[left.span(), right.span()]); + let operation_span = Span::merge(left.span(), right.span()); match operator.item { Operator::Math(Math::Plus) => { let mut res = lhs + rhs; diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs index 9327c3bbc6..1ab13271a3 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs @@ -295,7 +295,7 @@ impl NuDataFrame { } pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(value) } diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs index 8646cdefd0..cee31d7b53 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs @@ -84,7 +84,7 @@ impl NuExpression { } pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(value) } diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs index f03d9f0cc8..355516d340 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs @@ -134,7 +134,7 @@ impl NuLazyFrame { } pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(value) } diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs index e942e3be97..e1bcb30069 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs @@ -107,7 +107,7 @@ impl NuLazyGroupBy { } pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(value) } } diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/utils.rs b/crates/nu-cmd-dataframe/src/dataframe/values/utils.rs index 3ccf2e6d77..0dc43399a3 100644 --- a/crates/nu-cmd-dataframe/src/dataframe/values/utils.rs +++ b/crates/nu-cmd-dataframe/src/dataframe/values/utils.rs @@ -1,4 +1,4 @@ -use nu_protocol::{span as span_join, ShellError, Span, Spanned, Value}; +use nu_protocol::{ShellError, Span, Spanned, Value}; // Default value used when selecting rows from dataframe pub const DEFAULT_ROWS: usize = 5; @@ -27,7 +27,7 @@ pub(crate) fn convert_columns( let span = value.span(); match value { Value::String { val, .. } => { - col_span = span_join(&[col_span, span]); + col_span = col_span.merge(span); Ok(Spanned { item: val, span }) } _ => Err(ShellError::GenericError { @@ -68,7 +68,7 @@ pub(crate) fn convert_columns_string( let span = value.span(); match value { Value::String { val, .. } => { - col_span = span_join(&[col_span, span]); + col_span = col_span.merge(span); Ok(val) } _ => Err(ShellError::GenericError { diff --git a/crates/nu-cmd-extra/src/extra/bits/into.rs b/crates/nu-cmd-extra/src/extra/bits/into.rs index c7fd09b728..cf85f92ac5 100644 --- a/crates/nu-cmd-extra/src/extra/bits/into.rs +++ b/crates/nu-cmd-extra/src/extra/bits/into.rs @@ -118,22 +118,12 @@ fn into_bits( let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - match input { - PipelineData::ExternalStream { stdout: None, .. } => { - Ok(Value::binary(vec![], head).into_pipeline_data()) - } - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - // TODO: in the future, we may want this to stream out, converting each to bytes - let output = stream.into_bytes()?; - Ok(Value::binary(output.item, head).into_pipeline_data()) - } - _ => { - let args = Arguments { cell_paths }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) - } + if let PipelineData::ByteStream(stream, ..) = input { + // TODO: in the future, we may want this to stream out, converting each to bytes + Ok(Value::binary(stream.into_bytes()?, head).into_pipeline_data()) + } else { + let args = Arguments { cell_paths }; + operate(action, args, input, call.head, engine_state.ctrlc.clone()) } } diff --git a/crates/nu-cmd-extra/src/extra/filters/each_while.rs b/crates/nu-cmd-extra/src/extra/filters/each_while.rs index 2dda815d43..58679c8eea 100644 --- a/crates/nu-cmd-extra/src/extra/filters/each_while.rs +++ b/crates/nu-cmd-extra/src/extra/filters/each_while.rs @@ -10,7 +10,7 @@ impl Command for EachWhile { } fn usage(&self) -> &str { - "Run a block on each row of the input list until a null is found, then create a new list with the results." + "Run a closure on each row of the input list until a null is found, then create a new list with the results." } fn search_terms(&self) -> Vec<&str> { @@ -78,38 +78,40 @@ impl Command for EachWhile { | PipelineData::ListStream(..) => { let mut closure = ClosureEval::new(engine_state, stack, closure); Ok(input - .into_iter() - .map_while(move |value| match closure.run_with_value(value) { - Ok(data) => { - let value = data.into_value(head); - (!value.is_nothing()).then_some(value) - } - Err(_) => None, - }) - .fuse() - .into_pipeline_data(head, engine_state.ctrlc.clone())) - } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - let mut closure = ClosureEval::new(engine_state, stack, closure); - Ok(stream .into_iter() .map_while(move |value| { - let value = value.ok()?; - match closure.run_with_value(value) { - Ok(data) => { - let value = data.into_value(head); - (!value.is_nothing()).then_some(value) - } + match closure + .run_with_value(value) + .and_then(|data| data.into_value(head)) + { + Ok(value) => (!value.is_nothing()).then_some(value), Err(_) => None, } }) .fuse() .into_pipeline_data(head, engine_state.ctrlc.clone())) } + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + if let Some(chunks) = stream.chunks() { + let mut closure = ClosureEval::new(engine_state, stack, closure); + Ok(chunks + .map_while(move |value| { + let value = value.ok()?; + match closure + .run_with_value(value) + .and_then(|data| data.into_value(span)) + { + Ok(value) => (!value.is_nothing()).then_some(value), + Err(_) => None, + } + }) + .fuse() + .into_pipeline_data(head, engine_state.ctrlc.clone())) + } else { + Ok(PipelineData::Empty) + } + } // This match allows non-iterables to be accepted, // which is currently considered undesirable (Nov 2022). PipelineData::Value(value, ..) => { diff --git a/crates/nu-cmd-extra/src/extra/filters/roll/roll_down.rs b/crates/nu-cmd-extra/src/extra/filters/roll/roll_down.rs index 465b9f1f4c..24ea1bc309 100644 --- a/crates/nu-cmd-extra/src/extra/filters/roll/roll_down.rs +++ b/crates/nu-cmd-extra/src/extra/filters/roll/roll_down.rs @@ -56,7 +56,7 @@ impl Command for RollDown { let by: Option = call.get_flag(engine_state, stack, "by")?; let metadata = input.metadata(); - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let rotated_value = vertical_rotate_value(value, by, VerticalDirection::Down)?; Ok(rotated_value.into_pipeline_data().set_metadata(metadata)) diff --git a/crates/nu-cmd-extra/src/extra/filters/roll/roll_left.rs b/crates/nu-cmd-extra/src/extra/filters/roll/roll_left.rs index ff69f23268..789b70830d 100644 --- a/crates/nu-cmd-extra/src/extra/filters/roll/roll_left.rs +++ b/crates/nu-cmd-extra/src/extra/filters/roll/roll_left.rs @@ -94,7 +94,7 @@ impl Command for RollLeft { let metadata = input.metadata(); let cells_only = call.has_flag(engine_state, stack, "cells-only")?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let rotated_value = horizontal_rotate_value(value, by, cells_only, &HorizontalDirection::Left)?; diff --git a/crates/nu-cmd-extra/src/extra/filters/roll/roll_right.rs b/crates/nu-cmd-extra/src/extra/filters/roll/roll_right.rs index d190960581..55a1e42158 100644 --- a/crates/nu-cmd-extra/src/extra/filters/roll/roll_right.rs +++ b/crates/nu-cmd-extra/src/extra/filters/roll/roll_right.rs @@ -94,7 +94,7 @@ impl Command for RollRight { let metadata = input.metadata(); let cells_only = call.has_flag(engine_state, stack, "cells-only")?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let rotated_value = horizontal_rotate_value(value, by, cells_only, &HorizontalDirection::Right)?; diff --git a/crates/nu-cmd-extra/src/extra/filters/roll/roll_up.rs b/crates/nu-cmd-extra/src/extra/filters/roll/roll_up.rs index 1cd74fe247..7b9480599d 100644 --- a/crates/nu-cmd-extra/src/extra/filters/roll/roll_up.rs +++ b/crates/nu-cmd-extra/src/extra/filters/roll/roll_up.rs @@ -56,7 +56,7 @@ impl Command for RollUp { let by: Option = call.get_flag(engine_state, stack, "by")?; let metadata = input.metadata(); - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let rotated_value = vertical_rotate_value(value, by, VerticalDirection::Up)?; Ok(rotated_value.into_pipeline_data().set_metadata(metadata)) diff --git a/crates/nu-cmd-extra/src/extra/filters/update_cells.rs b/crates/nu-cmd-extra/src/extra/filters/update_cells.rs index b35736370d..452b91dc0a 100644 --- a/crates/nu-cmd-extra/src/extra/filters/update_cells.rs +++ b/crates/nu-cmd-extra/src/extra/filters/update_cells.rs @@ -151,7 +151,7 @@ impl Iterator for UpdateCellIterator { fn eval_value(closure: &mut ClosureEval, span: Span, value: Value) -> Value { closure .run_with_value(value) - .map(|data| data.into_value(span)) + .and_then(|data| data.into_value(span)) .unwrap_or_else(|err| Value::error(err, span)) } diff --git a/crates/nu-cmd-extra/src/extra/strings/format/command.rs b/crates/nu-cmd-extra/src/extra/strings/format/command.rs index 932b5ccb7f..1c72627779 100644 --- a/crates/nu-cmd-extra/src/extra/strings/format/command.rs +++ b/crates/nu-cmd-extra/src/extra/strings/format/command.rs @@ -39,7 +39,7 @@ impl Command for FormatPattern { let mut working_set = StateWorkingSet::new(engine_state); let specified_pattern: Result = call.req(engine_state, stack, 0); - let input_val = input.into_value(call.head); + let input_val = input.into_value(call.head)?; // add '$it' variable to support format like this: $it.column1.column2. let it_id = working_set.add_variable(b"$it".to_vec(), call.head, Type::Any, false); stack.add_var(it_id, input_val.clone()); diff --git a/crates/nu-cmd-extra/tests/commands/bytes/starts_with.rs b/crates/nu-cmd-extra/tests/commands/bytes/starts_with.rs index e7d57698b5..c3ad1ec448 100644 --- a/crates/nu-cmd-extra/tests/commands/bytes/starts_with.rs +++ b/crates/nu-cmd-extra/tests/commands/bytes/starts_with.rs @@ -19,102 +19,102 @@ fn basic_string_fails() { assert_eq!(actual.out, ""); } -#[test] -fn short_stream_binary() { - let actual = nu!(r#" - nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101] - "#); +// #[test] +// fn short_stream_binary() { +// let actual = nu!(r#" +// nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101] +// "#); - assert_eq!(actual.out, "true"); -} +// assert_eq!(actual.out, "true"); +// } -#[test] -fn short_stream_mismatch() { - let actual = nu!(r#" - nu --testbin repeater (0x[010203]) 5 | bytes starts-with 0x[010204] - "#); +// #[test] +// fn short_stream_mismatch() { +// let actual = nu!(r#" +// nu --testbin repeater (0x[010203]) 5 | bytes starts-with 0x[010204] +// "#); - assert_eq!(actual.out, "false"); -} +// assert_eq!(actual.out, "false"); +// } -#[test] -fn short_stream_binary_overflow() { - let actual = nu!(r#" - nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101010101] - "#); +// #[test] +// fn short_stream_binary_overflow() { +// let actual = nu!(r#" +// nu --testbin repeater (0x[01]) 5 | bytes starts-with 0x[010101010101] +// "#); - assert_eq!(actual.out, "false"); -} +// assert_eq!(actual.out, "false"); +// } -#[test] -fn long_stream_binary() { - let actual = nu!(r#" - nu --testbin repeater (0x[01]) 32768 | bytes starts-with 0x[010101] - "#); +// #[test] +// fn long_stream_binary() { +// let actual = nu!(r#" +// nu --testbin repeater (0x[01]) 32768 | bytes starts-with 0x[010101] +// "#); - assert_eq!(actual.out, "true"); -} +// assert_eq!(actual.out, "true"); +// } -#[test] -fn long_stream_binary_overflow() { - // .. ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow - let actual = nu!(r#" - nu --testbin repeater (0x[01]) 32768 | bytes starts-with (0..32768 | each {|| 0x[01] } | bytes collect) - "#); +// #[test] +// fn long_stream_binary_overflow() { +// // .. ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow +// let actual = nu!(r#" +// nu --testbin repeater (0x[01]) 32768 | bytes starts-with (0..32768 | each {|| 0x[01] } | bytes collect) +// "#); - assert_eq!(actual.out, "false"); -} +// assert_eq!(actual.out, "false"); +// } -#[test] -fn long_stream_binary_exact() { - // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow - let actual = nu!(r#" - nu --testbin repeater (0x[01020304]) 8192 | bytes starts-with (0..<8192 | each {|| 0x[01020304] } | bytes collect) - "#); +// #[test] +// fn long_stream_binary_exact() { +// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow +// let actual = nu!(r#" +// nu --testbin repeater (0x[01020304]) 8192 | bytes starts-with (0..<8192 | each {|| 0x[01020304] } | bytes collect) +// "#); - assert_eq!(actual.out, "true"); -} +// assert_eq!(actual.out, "true"); +// } -#[test] -fn long_stream_string_exact() { - // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow - let actual = nu!(r#" - nu --testbin repeater hell 8192 | bytes starts-with (0..<8192 | each {|| "hell" | into binary } | bytes collect) - "#); +// #[test] +// fn long_stream_string_exact() { +// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow +// let actual = nu!(r#" +// nu --testbin repeater hell 8192 | bytes starts-with (0..<8192 | each {|| "hell" | into binary } | bytes collect) +// "#); - assert_eq!(actual.out, "true"); -} +// assert_eq!(actual.out, "true"); +// } -#[test] -fn long_stream_mixed_exact() { - // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow - let actual = nu!(r#" - let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect) - let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect) +// #[test] +// fn long_stream_mixed_exact() { +// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow +// let actual = nu!(r#" +// let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect) +// let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect) - nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg) - "#); +// nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg) +// "#); - assert_eq!( - actual.err, "", - "invocation failed. command line limit likely reached" - ); - assert_eq!(actual.out, "true"); -} +// assert_eq!( +// actual.err, "", +// "invocation failed. command line limit likely reached" +// ); +// assert_eq!(actual.out, "true"); +// } -#[test] -fn long_stream_mixed_overflow() { - // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow - let actual = nu!(r#" - let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect) - let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect) +// #[test] +// fn long_stream_mixed_overflow() { +// // ranges are inclusive..inclusive, so we don't need to +1 to check for an overflow +// let actual = nu!(r#" +// let binseg = (0..<2048 | each {|| 0x[003d9fbf] } | bytes collect) +// let strseg = (0..<2048 | each {|| "hell" | into binary } | bytes collect) - nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg 0x[01]) - "#); +// nu --testbin repeat_bytes 003d9fbf 2048 68656c6c 2048 | bytes starts-with (bytes build $binseg $strseg 0x[01]) +// "#); - assert_eq!( - actual.err, "", - "invocation failed. command line limit likely reached" - ); - assert_eq!(actual.out, "false"); -} +// assert_eq!( +// actual.err, "", +// "invocation failed. command line limit likely reached" +// ); +// assert_eq!(actual.out, "false"); +// } diff --git a/crates/nu-cmd-lang/src/core_commands/collect.rs b/crates/nu-cmd-lang/src/core_commands/collect.rs index eae41e8690..404aa568da 100644 --- a/crates/nu-cmd-lang/src/core_commands/collect.rs +++ b/crates/nu-cmd-lang/src/core_commands/collect.rs @@ -43,7 +43,7 @@ impl Command for Collect { stack.captures_to_stack_preserve_out_dest(closure.captures.clone()); let metadata = input.metadata(); - let input = input.into_value(call.head); + let input = input.into_value(call.head)?; let mut saved_positional = None; if let Some(var) = block.signature.get_positional(0) { diff --git a/crates/nu-cmd-lang/src/core_commands/describe.rs b/crates/nu-cmd-lang/src/core_commands/describe.rs index cdbc09357e..1787fc199a 100644 --- a/crates/nu-cmd-lang/src/core_commands/describe.rs +++ b/crates/nu-cmd-lang/src/core_commands/describe.rs @@ -1,5 +1,5 @@ use nu_engine::command_prelude::*; -use nu_protocol::{engine::StateWorkingSet, PipelineMetadata}; +use nu_protocol::{engine::StateWorkingSet, ByteStreamSource, PipelineMetadata}; #[derive(Clone)] pub struct Describe; @@ -162,73 +162,38 @@ fn run( let metadata = input.metadata(); let description = match input { - PipelineData::ExternalStream { - ref stdout, - ref stderr, - ref exit_code, - .. - } => { - if options.detailed { - let stdout = if stdout.is_some() { - Value::record( - record! { - "type" => Value::string("stream", head), - "origin" => Value::string("external", head), - "subtype" => Value::string("any", head), - }, - head, - ) - } else { - Value::nothing(head) - }; - - let stderr = if stderr.is_some() { - Value::record( - record! { - "type" => Value::string("stream", head), - "origin" => Value::string("external", head), - "subtype" => Value::string("any", head), - }, - head, - ) - } else { - Value::nothing(head) - }; - - let exit_code = if exit_code.is_some() { - Value::record( - record! { - "type" => Value::string("stream", head), - "origin" => Value::string("external", head), - "subtype" => Value::string("int", head), - }, - head, - ) - } else { - Value::nothing(head) + PipelineData::ByteStream(stream, ..) => { + let description = if options.detailed { + let origin = match stream.source() { + ByteStreamSource::Read(_) => "unknown", + ByteStreamSource::File(_) => "file", + ByteStreamSource::Child(_) => "external", }; Value::record( record! { - "type" => Value::string("stream", head), - "origin" => Value::string("external", head), - "stdout" => stdout, - "stderr" => stderr, - "exit_code" => exit_code, + "type" => Value::string("byte stream", head), + "origin" => Value::string(origin, head), "metadata" => metadata_to_value(metadata, head), }, head, ) } else { - Value::string("raw input", head) + Value::string("byte stream", head) + }; + + if !options.no_collect { + stream.drain()?; } + + description } - PipelineData::ListStream(_, _) => { + PipelineData::ListStream(stream, ..) => { if options.detailed { let subtype = if options.no_collect { Value::string("any", head) } else { - describe_value(input.into_value(head), head, engine_state) + describe_value(stream.into_value(), head, engine_state) }; Value::record( record! { @@ -242,19 +207,19 @@ fn run( } else if options.no_collect { Value::string("stream", head) } else { - let value = input.into_value(head); + let value = stream.into_value(); let base_description = value.get_type().to_string(); Value::string(format!("{} (stream)", base_description), head) } } - _ => { - let value = input.into_value(head); + PipelineData::Value(value, ..) => { if !options.detailed { Value::string(value.get_type().to_string(), head) } else { describe_value(value, head, engine_state) } } + PipelineData::Empty => Value::string(Type::Nothing.to_string(), head), }; Ok(description.into_pipeline_data()) diff --git a/crates/nu-cmd-lang/src/core_commands/do_.rs b/crates/nu-cmd-lang/src/core_commands/do_.rs index b057880cf3..5f14e88c07 100644 --- a/crates/nu-cmd-lang/src/core_commands/do_.rs +++ b/crates/nu-cmd-lang/src/core_commands/do_.rs @@ -1,6 +1,13 @@ use nu_engine::{command_prelude::*, get_eval_block_with_early_return, redirect_env}; -use nu_protocol::{engine::Closure, ListStream, OutDest, RawStream}; -use std::thread; +use nu_protocol::{ + engine::Closure, + process::{ChildPipe, ChildProcess, ExitStatus}, + ByteStream, ByteStreamSource, OutDest, +}; +use std::{ + io::{Cursor, Read}, + thread, +}; #[derive(Clone)] pub struct Do; @@ -86,115 +93,91 @@ impl Command for Do { } match result { - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - }) if capture_errors => { - // Use a thread to receive stdout message. - // Or we may get a deadlock if child process sends out too much bytes to stderr. - // - // For example: in normal linux system, stderr pipe's limit is 65535 bytes. - // if child process sends out 65536 bytes, the process will be hanged because no consumer - // consumes the first 65535 bytes - // So we need a thread to receive stdout message, then the current thread can continue to consume - // stderr messages. - let stdout_handler = stdout - .map(|stdout_stream| { - thread::Builder::new() - .name("stderr redirector".to_string()) - .spawn(move || { - let ctrlc = stdout_stream.ctrlc.clone(); - let span = stdout_stream.span; - RawStream::new( - Box::new(std::iter::once( - stdout_stream.into_bytes().map(|s| s.item), - )), - ctrlc, - span, - None, - ) + Ok(PipelineData::ByteStream(stream, metadata)) if capture_errors => { + let span = stream.span(); + match stream.into_child() { + Ok(mut child) => { + // Use a thread to receive stdout message. + // Or we may get a deadlock if child process sends out too much bytes to stderr. + // + // For example: in normal linux system, stderr pipe's limit is 65535 bytes. + // if child process sends out 65536 bytes, the process will be hanged because no consumer + // consumes the first 65535 bytes + // So we need a thread to receive stdout message, then the current thread can continue to consume + // stderr messages. + let stdout_handler = child + .stdout + .take() + .map(|mut stdout| { + thread::Builder::new() + .name("stdout consumer".to_string()) + .spawn(move || { + let mut buf = Vec::new(); + stdout.read_to_end(&mut buf)?; + Ok::<_, ShellError>(buf) + }) + .err_span(head) }) - .err_span(head) - }) - .transpose()?; + .transpose()?; - // Intercept stderr so we can return it in the error if the exit code is non-zero. - // The threading issues mentioned above dictate why we also need to intercept stdout. - let mut stderr_ctrlc = None; - let stderr_msg = match stderr { - None => "".to_string(), - Some(stderr_stream) => { - stderr_ctrlc.clone_from(&stderr_stream.ctrlc); - stderr_stream.into_string().map(|s| s.item)? - } - }; + // Intercept stderr so we can return it in the error if the exit code is non-zero. + // The threading issues mentioned above dictate why we also need to intercept stdout. + let stderr_msg = match child.stderr.take() { + None => String::new(), + Some(mut stderr) => { + let mut buf = String::new(); + stderr.read_to_string(&mut buf).err_span(span)?; + buf + } + }; - let stdout = if let Some(handle) = stdout_handler { - match handle.join() { - Err(err) => { + let stdout = if let Some(handle) = stdout_handler { + match handle.join() { + Err(err) => { + return Err(ShellError::ExternalCommand { + label: "Fail to receive external commands stdout message" + .to_string(), + help: format!("{err:?}"), + span, + }); + } + Ok(res) => Some(res?), + } + } else { + None + }; + + if child.wait()? != ExitStatus::Exited(0) { return Err(ShellError::ExternalCommand { - label: "Fail to receive external commands stdout message" - .to_string(), - help: format!("{err:?}"), + label: "External command failed".to_string(), + help: stderr_msg, span, }); } - Ok(res) => Some(res), - } - } else { - None - }; - let exit_code: Vec = match exit_code { - None => vec![], - Some(exit_code_stream) => exit_code_stream.into_iter().collect(), - }; - if let Some(Value::Int { val: code, .. }) = exit_code.last() { - if *code != 0 { - return Err(ShellError::ExternalCommand { - label: "External command failed".to_string(), - help: stderr_msg, - span, - }); + let mut child = ChildProcess::from_raw(None, None, None, span); + if let Some(stdout) = stdout { + child.stdout = Some(ChildPipe::Tee(Box::new(Cursor::new(stdout)))); + } + if !stderr_msg.is_empty() { + child.stderr = Some(ChildPipe::Tee(Box::new(Cursor::new(stderr_msg)))); + } + Ok(PipelineData::ByteStream( + ByteStream::child(child, span), + metadata, + )) } + Err(stream) => Ok(PipelineData::ByteStream(stream, metadata)), } - - Ok(PipelineData::ExternalStream { - stdout, - stderr: Some(RawStream::new( - Box::new(std::iter::once(Ok(stderr_msg.into_bytes()))), - stderr_ctrlc, - span, - None, - )), - exit_code: Some(ListStream::new(exit_code.into_iter(), span, None)), - span, - metadata, - trim_end_newline, - }) } - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code: _, - span, - metadata, - trim_end_newline, - }) if ignore_program_errors - && !matches!(caller_stack.stdout(), OutDest::Pipe | OutDest::Capture) => + Ok(PipelineData::ByteStream(mut stream, metadata)) + if ignore_program_errors + && !matches!(caller_stack.stdout(), OutDest::Pipe | OutDest::Capture) => { - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code: None, - span, - metadata, - trim_end_newline, - }) + if let ByteStreamSource::Child(child) = stream.source_mut() { + child.set_exit_code(0) + } + Ok(PipelineData::ByteStream(stream, metadata)) } Ok(PipelineData::Value(Value::Error { .. }, ..)) | Err(_) if ignore_shell_errors => { Ok(PipelineData::empty()) diff --git a/crates/nu-cmd-lang/src/core_commands/for_.rs b/crates/nu-cmd-lang/src/core_commands/for_.rs index 64e6c0a6ba..6f9391614e 100644 --- a/crates/nu-cmd-lang/src/core_commands/for_.rs +++ b/crates/nu-cmd-lang/src/core_commands/for_.rs @@ -121,12 +121,14 @@ impl Command for For { Err(err) => { return Err(err); } - Ok(pipeline) => { - let exit_code = pipeline.drain_with_exit_code()?; - if exit_code != 0 { - return Ok(PipelineData::new_external_stream_with_only_exit_code( - exit_code, - )); + Ok(data) => { + if let Some(status) = data.drain()? { + let code = status.code(); + if code != 0 { + return Ok( + PipelineData::new_external_stream_with_only_exit_code(code), + ); + } } } } @@ -159,12 +161,14 @@ impl Command for For { Err(err) => { return Err(err); } - Ok(pipeline) => { - let exit_code = pipeline.drain_with_exit_code()?; - if exit_code != 0 { - return Ok(PipelineData::new_external_stream_with_only_exit_code( - exit_code, - )); + Ok(data) => { + if let Some(status) = data.drain()? { + let code = status.code(); + if code != 0 { + return Ok( + PipelineData::new_external_stream_with_only_exit_code(code), + ); + } } } } @@ -173,7 +177,7 @@ impl Command for For { x => { stack.add_var(var_id, x); - eval_block(&engine_state, stack, block, PipelineData::empty())?.into_value(head); + eval_block(&engine_state, stack, block, PipelineData::empty())?.into_value(head)?; } } Ok(PipelineData::empty()) diff --git a/crates/nu-cmd-lang/src/core_commands/let_.rs b/crates/nu-cmd-lang/src/core_commands/let_.rs index c780954bc6..cc5504d8d6 100644 --- a/crates/nu-cmd-lang/src/core_commands/let_.rs +++ b/crates/nu-cmd-lang/src/core_commands/let_.rs @@ -61,7 +61,7 @@ impl Command for Let { let eval_block = get_eval_block(engine_state); let stack = &mut stack.start_capture(); let pipeline_data = eval_block(engine_state, stack, block, input)?; - let value = pipeline_data.into_value(call.head); + let value = pipeline_data.into_value(call.head)?; // if given variable type is Glob, and our result is string // then nushell need to convert from Value::String to Value::Glob diff --git a/crates/nu-cmd-lang/src/core_commands/loop_.rs b/crates/nu-cmd-lang/src/core_commands/loop_.rs index 29f22649eb..9b1e36a057 100644 --- a/crates/nu-cmd-lang/src/core_commands/loop_.rs +++ b/crates/nu-cmd-lang/src/core_commands/loop_.rs @@ -53,12 +53,12 @@ impl Command for Loop { Err(err) => { return Err(err); } - Ok(pipeline) => { - let exit_code = pipeline.drain_with_exit_code()?; - if exit_code != 0 { - return Ok(PipelineData::new_external_stream_with_only_exit_code( - exit_code, - )); + Ok(data) => { + if let Some(status) = data.drain()? { + let code = status.code(); + if code != 0 { + return Ok(PipelineData::new_external_stream_with_only_exit_code(code)); + } } } } diff --git a/crates/nu-cmd-lang/src/core_commands/mut_.rs b/crates/nu-cmd-lang/src/core_commands/mut_.rs index be2d66aff4..60c4c146db 100644 --- a/crates/nu-cmd-lang/src/core_commands/mut_.rs +++ b/crates/nu-cmd-lang/src/core_commands/mut_.rs @@ -61,7 +61,7 @@ impl Command for Mut { let eval_block = get_eval_block(engine_state); let stack = &mut stack.start_capture(); let pipeline_data = eval_block(engine_state, stack, block, input)?; - let value = pipeline_data.into_value(call.head); + let value = pipeline_data.into_value(call.head)?; // if given variable type is Glob, and our result is string // then nushell need to convert from Value::String to Value::Glob diff --git a/crates/nu-cmd-lang/src/core_commands/try_.rs b/crates/nu-cmd-lang/src/core_commands/try_.rs index bc96f3c28a..0b399e368a 100644 --- a/crates/nu-cmd-lang/src/core_commands/try_.rs +++ b/crates/nu-cmd-lang/src/core_commands/try_.rs @@ -62,10 +62,11 @@ impl Command for Try { } // external command may fail to run Ok(pipeline) => { - let (pipeline, external_failed) = pipeline.check_external_failed(); + let (pipeline, external_failed) = pipeline.check_external_failed()?; if external_failed { - let exit_code = pipeline.drain_with_exit_code()?; - stack.add_env_var("LAST_EXIT_CODE".into(), Value::int(exit_code, call.head)); + let status = pipeline.drain()?; + let code = status.map(|status| status.code()).unwrap_or(0); + stack.add_env_var("LAST_EXIT_CODE".into(), Value::int(code.into(), call.head)); let err_value = Value::nothing(call.head); handle_catch(err_value, catch_block, engine_state, stack, eval_block) } else { diff --git a/crates/nu-cmd-lang/src/core_commands/while_.rs b/crates/nu-cmd-lang/src/core_commands/while_.rs index e42e4ab6d1..bf9076aa0c 100644 --- a/crates/nu-cmd-lang/src/core_commands/while_.rs +++ b/crates/nu-cmd-lang/src/core_commands/while_.rs @@ -70,14 +70,16 @@ impl Command for While { Err(err) => { return Err(err); } - Ok(pipeline) => { - let exit_code = pipeline.drain_with_exit_code()?; - if exit_code != 0 { - return Ok( - PipelineData::new_external_stream_with_only_exit_code( - exit_code, - ), - ); + Ok(data) => { + if let Some(status) = data.drain()? { + let code = status.code(); + if code != 0 { + return Ok( + PipelineData::new_external_stream_with_only_exit_code( + code, + ), + ); + } } } } diff --git a/crates/nu-cmd-lang/src/example_support.rs b/crates/nu-cmd-lang/src/example_support.rs index 3e50703b70..e3bec47fe6 100644 --- a/crates/nu-cmd-lang/src/example_support.rs +++ b/crates/nu-cmd-lang/src/example_support.rs @@ -122,10 +122,9 @@ pub fn eval_block( stack.add_env_var("PWD".to_string(), Value::test_string(cwd.to_string_lossy())); - match nu_engine::eval_block::(engine_state, &mut stack, &block, input) { - Err(err) => panic!("test eval error in `{}`: {:?}", "TODO", err), - Ok(result) => result.into_value(Span::test_data()), - } + nu_engine::eval_block::(engine_state, &mut stack, &block, input) + .and_then(|data| data.into_value(Span::test_data())) + .unwrap_or_else(|err| panic!("test eval error in `{}`: {:?}", "TODO", err)) } pub fn check_example_evaluates_to_expected_output( @@ -223,7 +222,7 @@ impl<'a> std::fmt::Debug for DebuggableValue<'a> { Value::Date { val, .. } => { write!(f, "Date({:?})", val) } - Value::Range { val, .. } => match val { + Value::Range { val, .. } => match **val { Range::IntRange(range) => match range.end() { Bound::Included(end) => write!( f, diff --git a/crates/nu-cmd-plugin/src/commands/plugin/add.rs b/crates/nu-cmd-plugin/src/commands/plugin/add.rs index e2c1c31151..225941db01 100644 --- a/crates/nu-cmd-plugin/src/commands/plugin/add.rs +++ b/crates/nu-cmd-plugin/src/commands/plugin/add.rs @@ -43,7 +43,8 @@ impl Command for PluginAdd { fn extra_usage(&self) -> &str { r#" -This does not load the plugin commands into the scope - see `register` for that. +This does not load the plugin commands into the scope - see `plugin use` for +that. Instead, it runs the plugin to get its command signatures, and then edits the plugin registry file (by default, `$nu.plugin-path`). The changes will be diff --git a/crates/nu-color-config/src/style_computer.rs b/crates/nu-color-config/src/style_computer.rs index cd2454f011..91907c1428 100644 --- a/crates/nu-color-config/src/style_computer.rs +++ b/crates/nu-color-config/src/style_computer.rs @@ -58,11 +58,11 @@ impl<'a> StyleComputer<'a> { Some(ComputableStyle::Closure(closure, span)) => { let result = ClosureEvalOnce::new(self.engine_state, self.stack, closure.clone()) .debug(false) - .run_with_value(value.clone()); + .run_with_value(value.clone()) + .and_then(|data| data.into_value(*span)); match result { - Ok(v) => { - let value = v.into_value(*span); + Ok(value) => { // These should be the same color data forms supported by color_config. match value { Value::Record { .. } => color_record_to_nustyle(&value), @@ -146,7 +146,10 @@ impl<'a> StyleComputer<'a> { let span = value.span(); match value { Value::Closure { val, .. } => { - map.insert(key.to_string(), ComputableStyle::Closure(val.clone(), span)); + map.insert( + key.to_string(), + ComputableStyle::Closure(*val.clone(), span), + ); } Value::Record { .. } => { map.insert( diff --git a/crates/nu-command/src/bytes/starts_with.rs b/crates/nu-command/src/bytes/starts_with.rs index 69187894b4..2d7ca3e26a 100644 --- a/crates/nu-command/src/bytes/starts_with.rs +++ b/crates/nu-command/src/bytes/starts_with.rs @@ -60,63 +60,13 @@ impl Command for BytesStartsWith { pattern, cell_paths, }; - - match input { - PipelineData::ExternalStream { - stdout: Some(stream), - span, - .. - } => { - let mut i = 0; - - for item in stream { - let byte_slice = match &item { - // String and binary data are valid byte patterns - Ok(Value::String { val, .. }) => val.as_bytes(), - Ok(Value::Binary { val, .. }) => val, - // If any Error value is output, echo it back - Ok(v @ Value::Error { .. }) => return Ok(v.clone().into_pipeline_data()), - // Unsupported data - Ok(other) => { - return Ok(Value::error( - ShellError::OnlySupportsThisInputType { - exp_input_type: "string and binary".into(), - wrong_type: other.get_type().to_string(), - dst_span: span, - src_span: other.span(), - }, - span, - ) - .into_pipeline_data()); - } - Err(err) => return Err(err.to_owned()), - }; - - let max = byte_slice.len().min(arg.pattern.len() - i); - - if byte_slice[..max] == arg.pattern[i..i + max] { - i += max; - - if i >= arg.pattern.len() { - return Ok(Value::bool(true, span).into_pipeline_data()); - } - } else { - return Ok(Value::bool(false, span).into_pipeline_data()); - } - } - - // We reached the end of the stream and never returned, - // the pattern wasn't exhausted so it probably doesn't match - Ok(Value::bool(false, span).into_pipeline_data()) - } - _ => operate( - starts_with, - arg, - input, - call.head, - engine_state.ctrlc.clone(), - ), - } + operate( + starts_with, + arg, + input, + call.head, + engine_state.ctrlc.clone(), + ) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/charting/histogram.rs b/crates/nu-command/src/charting/histogram.rs index 35a9d82a3d..52964b087d 100755 --- a/crates/nu-command/src/charting/histogram.rs +++ b/crates/nu-command/src/charting/histogram.rs @@ -121,7 +121,7 @@ impl Command for Histogram { }; let span = call.head; - let data_as_value = input.into_value(span); + let data_as_value = input.into_value(span)?; let value_span = data_as_value.span(); // `input` is not a list, here we can return an error. run_histogram( diff --git a/crates/nu-command/src/conversions/fill.rs b/crates/nu-command/src/conversions/fill.rs index 76ec1e81e5..6507e4a368 100644 --- a/crates/nu-command/src/conversions/fill.rs +++ b/crates/nu-command/src/conversions/fill.rs @@ -73,7 +73,7 @@ impl Command for Fill { } fn search_terms(&self) -> Vec<&str> { - vec!["display", "render", "format", "pad", "align"] + vec!["display", "render", "format", "pad", "align", "repeat"] } fn examples(&self) -> Vec { @@ -91,9 +91,9 @@ impl Command for Fill { result: Some(Value::string("────────nushell", Span::test_data())), }, Example { - description: "Fill a string on both sides to a width of 15 with the character '─'", - example: "'nushell' | fill --alignment m --character '─' --width 15", - result: Some(Value::string("────nushell────", Span::test_data())), + description: "Fill an empty string with 10 '─' characters", + example: "'' | fill --character '─' --width 10", + result: Some(Value::string("──────────", Span::test_data())), }, Example { description: diff --git a/crates/nu-command/src/conversions/into/binary.rs b/crates/nu-command/src/conversions/into/binary.rs index 6fb997a590..479b0fc7d7 100644 --- a/crates/nu-command/src/conversions/into/binary.rs +++ b/crates/nu-command/src/conversions/into/binary.rs @@ -127,25 +127,15 @@ fn into_binary( let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - match input { - PipelineData::ExternalStream { stdout: None, .. } => { - Ok(Value::binary(vec![], head).into_pipeline_data()) - } - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - // TODO: in the future, we may want this to stream out, converting each to bytes - let output = stream.into_bytes()?; - Ok(Value::binary(output.item, head).into_pipeline_data()) - } - _ => { - let args = Arguments { - cell_paths, - compact: call.has_flag(engine_state, stack, "compact")?, - }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) - } + if let PipelineData::ByteStream(stream, ..) = input { + // TODO: in the future, we may want this to stream out, converting each to bytes + Ok(Value::binary(stream.into_bytes()?, head).into_pipeline_data()) + } else { + let args = Arguments { + cell_paths, + compact: call.has_flag(engine_state, stack, "compact")?, + }; + operate(action, args, input, call.head, engine_state.ctrlc.clone()) } } diff --git a/crates/nu-command/src/conversions/into/cell_path.rs b/crates/nu-command/src/conversions/into/cell_path.rs index 4faa6e83d6..6da317abd3 100644 --- a/crates/nu-command/src/conversions/into/cell_path.rs +++ b/crates/nu-command/src/conversions/into/cell_path.rs @@ -101,11 +101,11 @@ fn into_cell_path(call: &Call, input: PipelineData) -> Result = stream.into_iter().collect(); Ok(list_to_cell_path(&list, head)?.into_pipeline_data()) } - PipelineData::ExternalStream { span, .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, int".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: head, - src_span: span, + src_span: stream.span(), }), PipelineData::Empty => Err(ShellError::PipelineEmpty { dst_span: head }), } diff --git a/crates/nu-command/src/conversions/into/glob.rs b/crates/nu-command/src/conversions/into/glob.rs index 8c167b0dc0..e5d03093f4 100644 --- a/crates/nu-command/src/conversions/into/glob.rs +++ b/crates/nu-command/src/conversions/into/glob.rs @@ -82,20 +82,12 @@ fn glob_helper( let head = call.head; let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - let args = Arguments { cell_paths }; - match input { - PipelineData::ExternalStream { stdout: None, .. } => { - Ok(Value::glob(String::new(), false, head).into_pipeline_data()) - } - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - // TODO: in the future, we may want this to stream out, converting each to bytes - let output = stream.into_string()?; - Ok(Value::glob(output.item, false, head).into_pipeline_data()) - } - _ => operate(action, args, input, head, engine_state.ctrlc.clone()), + if let PipelineData::ByteStream(stream, ..) = input { + // TODO: in the future, we may want this to stream out, converting each to bytes + Ok(Value::glob(stream.into_string()?, false, head).into_pipeline_data()) + } else { + let args = Arguments { cell_paths }; + operate(action, args, input, head, engine_state.ctrlc.clone()) } } diff --git a/crates/nu-command/src/conversions/into/record.rs b/crates/nu-command/src/conversions/into/record.rs index c9342e8e39..e867f06e15 100644 --- a/crates/nu-command/src/conversions/into/record.rs +++ b/crates/nu-command/src/conversions/into/record.rs @@ -108,7 +108,7 @@ fn into_record( call: &Call, input: PipelineData, ) -> Result { - let input = input.into_value(call.head); + let input = input.into_value(call.head)?; let input_type = input.get_type(); let span = input.span(); let res = match input { diff --git a/crates/nu-command/src/conversions/into/string.rs b/crates/nu-command/src/conversions/into/string.rs index bc791a37b2..eda4f7e5a5 100644 --- a/crates/nu-command/src/conversions/into/string.rs +++ b/crates/nu-command/src/conversions/into/string.rs @@ -155,26 +155,18 @@ fn string_helper( } let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - let config = engine_state.get_config().clone(); - let args = Arguments { - decimals_value, - cell_paths, - config, - }; - match input { - PipelineData::ExternalStream { stdout: None, .. } => { - Ok(Value::string(String::new(), head).into_pipeline_data()) - } - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - // TODO: in the future, we may want this to stream out, converting each to bytes - let output = stream.into_string()?; - Ok(Value::string(output.item, head).into_pipeline_data()) - } - _ => operate(action, args, input, head, engine_state.ctrlc.clone()), + if let PipelineData::ByteStream(stream, ..) = input { + // TODO: in the future, we may want this to stream out, converting each to bytes + Ok(Value::string(stream.into_string()?, head).into_pipeline_data()) + } else { + let config = engine_state.get_config().clone(); + let args = Arguments { + decimals_value, + cell_paths, + config, + }; + operate(action, args, input, head, engine_state.ctrlc.clone()) } } diff --git a/crates/nu-command/src/database/mod.rs b/crates/nu-command/src/database/mod.rs index 4fddf6638e..22b92a81d3 100644 --- a/crates/nu-command/src/database/mod.rs +++ b/crates/nu-command/src/database/mod.rs @@ -5,7 +5,7 @@ use commands::add_commands_decls; pub use values::{ convert_sqlite_row_to_nu_value, convert_sqlite_value_to_nu_value, open_connection_in_memory, - open_connection_in_memory_custom, SQLiteDatabase, MEMORY_DB, + open_connection_in_memory_custom, values_to_sql, SQLiteDatabase, MEMORY_DB, }; use nu_protocol::engine::StateWorkingSet; diff --git a/crates/nu-command/src/database/values/mod.rs b/crates/nu-command/src/database/values/mod.rs index 4442ec0783..b9bd3d5d4c 100644 --- a/crates/nu-command/src/database/values/mod.rs +++ b/crates/nu-command/src/database/values/mod.rs @@ -3,5 +3,5 @@ pub mod sqlite; pub use sqlite::{ convert_sqlite_row_to_nu_value, convert_sqlite_value_to_nu_value, open_connection_in_memory, - open_connection_in_memory_custom, SQLiteDatabase, MEMORY_DB, + open_connection_in_memory_custom, values_to_sql, SQLiteDatabase, MEMORY_DB, }; diff --git a/crates/nu-command/src/database/values/sqlite.rs b/crates/nu-command/src/database/values/sqlite.rs index a581d7dfc3..a5761682ad 100644 --- a/crates/nu-command/src/database/values/sqlite.rs +++ b/crates/nu-command/src/database/values/sqlite.rs @@ -91,7 +91,7 @@ impl SQLiteDatabase { } pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(value) } diff --git a/crates/nu-command/src/debug/inspect.rs b/crates/nu-command/src/debug/inspect.rs index 681d2ef6c6..ad6163c7b6 100644 --- a/crates/nu-command/src/debug/inspect.rs +++ b/crates/nu-command/src/debug/inspect.rs @@ -29,7 +29,7 @@ impl Command for Inspect { input: PipelineData, ) -> Result { let input_metadata = input.metadata(); - let input_val = input.into_value(call.head); + let input_val = input.into_value(call.head)?; if input_val.is_nothing() { return Err(ShellError::PipelineEmpty { dst_span: call.head, diff --git a/crates/nu-command/src/debug/timeit.rs b/crates/nu-command/src/debug/timeit.rs index 92f8fe18cd..a445679b81 100644 --- a/crates/nu-command/src/debug/timeit.rs +++ b/crates/nu-command/src/debug/timeit.rs @@ -53,13 +53,12 @@ impl Command for TimeIt { eval_block(engine_state, stack, block, input)? } else { let eval_expression_with_input = get_eval_expression_with_input(engine_state); - eval_expression_with_input(engine_state, stack, command_to_run, input) - .map(|res| res.0)? + eval_expression_with_input(engine_state, stack, command_to_run, input)?.0 } } else { PipelineData::empty() } - .into_value(call.head); + .into_value(call.head)?; let end_time = Instant::now(); diff --git a/crates/nu-command/src/env/with_env.rs b/crates/nu-command/src/env/with_env.rs index bfd7c89130..3671a2cdc5 100644 --- a/crates/nu-command/src/env/with_env.rs +++ b/crates/nu-command/src/env/with_env.rs @@ -90,10 +90,7 @@ fn with_env( return Err(ShellError::CantConvert { to_type: "record".into(), from_type: x.get_type().to_string(), - span: call - .positional_nth(1) - .expect("already checked through .req") - .span, + span: x.span(), help: None, }); } @@ -124,10 +121,7 @@ fn with_env( return Err(ShellError::CantConvert { to_type: "record".into(), from_type: x.get_type().to_string(), - span: call - .positional_nth(1) - .expect("already checked through .req") - .span, + span: x.span(), help: None, }); } diff --git a/crates/nu-command/src/filesystem/cd.rs b/crates/nu-command/src/filesystem/cd.rs index f90e9bfd2a..57fe1c17e3 100644 --- a/crates/nu-command/src/filesystem/cd.rs +++ b/crates/nu-command/src/filesystem/cd.rs @@ -1,3 +1,4 @@ +use nu_cmd_base::util::get_init_cwd; use nu_engine::command_prelude::*; use nu_utils::filesystem::{have_permission, PermissionResult}; @@ -39,7 +40,10 @@ impl Command for Cd { ) -> Result { let physical = call.has_flag(engine_state, stack, "physical")?; let path_val: Option> = call.opt(engine_state, stack, 0)?; - let cwd = engine_state.cwd(Some(stack))?; + + // If getting PWD failed, default to the initial directory. This way, the + // user can use `cd` to recover PWD to a good state. + let cwd = engine_state.cwd(Some(stack)).unwrap_or(get_init_cwd()); let path_val = { if let Some(path) = path_val { @@ -52,13 +56,13 @@ impl Command for Cd { } }; - let (path, span) = match path_val { + let path = match path_val { Some(v) => { if v.item == "-" { if let Some(oldpwd) = stack.get_env_var(engine_state, "OLDPWD") { - (oldpwd.to_path()?, v.span) + oldpwd.to_path()? } else { - (cwd, v.span) + cwd } } else { // Trim whitespace from the end of path. @@ -66,7 +70,7 @@ impl Command for Cd { &v.item.trim_end_matches(|x| matches!(x, '\x09'..='\x0d')); // If `--physical` is specified, canonicalize the path; otherwise expand the path. - let path = if physical { + if physical { if let Ok(path) = nu_path::canonicalize_with(path_no_whitespace, &cwd) { if !path.is_dir() { return Err(ShellError::NotADirectory { span: v.span }); @@ -90,19 +94,12 @@ impl Command for Cd { return Err(ShellError::NotADirectory { span: v.span }); }; path - }; - (path, v.span) + } } } - None => { - let path = nu_path::expand_tilde("~"); - (path, call.head) - } + None => nu_path::expand_tilde("~"), }; - // Strip the trailing slash from the new path. This is required for PWD. - let path = nu_path::strip_trailing_slash(&path); - // Set OLDPWD. // We're using `Stack::get_env_var()` instead of `EngineState::cwd()` to avoid a conversion roundtrip. if let Some(oldpwd) = stack.get_env_var(engine_state, "PWD") { @@ -113,7 +110,7 @@ impl Command for Cd { //FIXME: this only changes the current scope, but instead this environment variable //should probably be a block that loads the information from the state in the overlay PermissionResult::PermissionOk => { - stack.add_env_var("PWD".into(), Value::string(path.to_string_lossy(), span)); + stack.set_cwd(path)?; Ok(PipelineData::empty()) } PermissionResult::PermissionDenied(reason) => Err(ShellError::IOError { diff --git a/crates/nu-command/src/filesystem/open.rs b/crates/nu-command/src/filesystem/open.rs index 23664bb576..5fb8527511 100644 --- a/crates/nu-command/src/filesystem/open.rs +++ b/crates/nu-command/src/filesystem/open.rs @@ -1,8 +1,8 @@ use super::util::get_rest_for_glob_pattern; #[allow(deprecated)] use nu_engine::{command_prelude::*, current_dir, get_eval_block}; -use nu_protocol::{BufferedReader, DataSource, NuGlob, PipelineMetadata, RawStream}; -use std::{io::BufReader, path::Path}; +use nu_protocol::{ByteStream, DataSource, NuGlob, PipelineMetadata}; +use std::path::Path; #[cfg(feature = "sqlite")] use crate::database::SQLiteDatabase; @@ -143,23 +143,13 @@ impl Command for Open { } }; - let buf_reader = BufReader::new(file); - - let file_contents = PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(BufferedReader::new(buf_reader)), - ctrlc.clone(), - call_span, - None, - )), - stderr: None, - exit_code: None, - span: call_span, - metadata: Some(PipelineMetadata { + let stream = PipelineData::ByteStream( + ByteStream::file(file, call_span, ctrlc.clone()), + Some(PipelineMetadata { data_source: DataSource::FilePath(path.to_path_buf()), }), - trim_end_newline: false, - }; + ); + let exts_opt: Option> = if raw { None } else { @@ -184,9 +174,9 @@ impl Command for Open { let decl = engine_state.get_decl(converter_id); let command_output = if let Some(block_id) = decl.get_block_id() { let block = engine_state.get_block(block_id); - eval_block(engine_state, stack, block, file_contents) + eval_block(engine_state, stack, block, stream) } else { - decl.run(engine_state, stack, &Call::new(call_span), file_contents) + decl.run(engine_state, stack, &Call::new(call_span), stream) }; output.push(command_output.map_err(|inner| { ShellError::GenericError{ @@ -198,7 +188,7 @@ impl Command for Open { } })?); } - None => output.push(file_contents), + None => output.push(stream), } } } diff --git a/crates/nu-command/src/filesystem/rm.rs b/crates/nu-command/src/filesystem/rm.rs index 9b9e88b5ff..9696ae0c2f 100644 --- a/crates/nu-command/src/filesystem/rm.rs +++ b/crates/nu-command/src/filesystem/rm.rs @@ -3,7 +3,7 @@ use super::util::{get_rest_for_glob_pattern, try_interaction}; use nu_engine::{command_prelude::*, env::current_dir}; use nu_glob::MatchOptions; use nu_path::expand_path_with; -use nu_protocol::NuGlob; +use nu_protocol::{report_error_new, NuGlob}; #[cfg(unix)] use std::os::unix::prelude::FileTypeExt; use std::{ @@ -118,8 +118,6 @@ fn rm( let interactive = call.has_flag(engine_state, stack, "interactive")?; let interactive_once = call.has_flag(engine_state, stack, "interactive-once")? && !interactive; - let ctrlc = engine_state.ctrlc.clone(); - let mut paths = get_rest_for_glob_pattern(engine_state, stack, call, 0)?; if paths.is_empty() { @@ -341,132 +339,130 @@ fn rm( } } - all_targets - .into_iter() - .map(move |(f, span)| { - let is_empty = || match f.read_dir() { - Ok(mut p) => p.next().is_none(), - Err(_) => false, - }; + let iter = all_targets.into_iter().map(move |(f, span)| { + let is_empty = || match f.read_dir() { + Ok(mut p) => p.next().is_none(), + Err(_) => false, + }; - if let Ok(metadata) = f.symlink_metadata() { - #[cfg(unix)] - let is_socket = metadata.file_type().is_socket(); - #[cfg(unix)] - let is_fifo = metadata.file_type().is_fifo(); + if let Ok(metadata) = f.symlink_metadata() { + #[cfg(unix)] + let is_socket = metadata.file_type().is_socket(); + #[cfg(unix)] + let is_fifo = metadata.file_type().is_fifo(); - #[cfg(not(unix))] - let is_socket = false; - #[cfg(not(unix))] - let is_fifo = false; + #[cfg(not(unix))] + let is_socket = false; + #[cfg(not(unix))] + let is_fifo = false; - if metadata.is_file() - || metadata.file_type().is_symlink() - || recursive - || is_socket - || is_fifo - || is_empty() - { - let (interaction, confirmed) = try_interaction( - interactive, - format!("rm: remove '{}'? ", f.to_string_lossy()), - ); + if metadata.is_file() + || metadata.file_type().is_symlink() + || recursive + || is_socket + || is_fifo + || is_empty() + { + let (interaction, confirmed) = try_interaction( + interactive, + format!("rm: remove '{}'? ", f.to_string_lossy()), + ); - let result = if let Err(e) = interaction { - let e = Error::new(ErrorKind::Other, &*e.to_string()); - Err(e) - } else if interactive && !confirmed { - Ok(()) - } else if TRASH_SUPPORTED && (trash || (rm_always_trash && !permanent)) { - #[cfg(all( - feature = "trash-support", - not(any(target_os = "android", target_os = "ios")) - ))] - { - trash::delete(&f).map_err(|e: trash::Error| { - Error::new( - ErrorKind::Other, - format!("{e:?}\nTry '--permanent' flag"), - ) - }) - } - - // Should not be reachable since we error earlier if - // these options are given on an unsupported platform - #[cfg(any( - not(feature = "trash-support"), - target_os = "android", - target_os = "ios" - ))] - { - unreachable!() - } - } else if metadata.is_symlink() { - // In Windows, symlink pointing to a directory can be removed using - // std::fs::remove_dir instead of std::fs::remove_file. - #[cfg(windows)] - { - f.metadata().and_then(|metadata| { - if metadata.is_dir() { - std::fs::remove_dir(&f) - } else { - std::fs::remove_file(&f) - } - }) - } - - #[cfg(not(windows))] - std::fs::remove_file(&f) - } else if metadata.is_file() || is_socket || is_fifo { - std::fs::remove_file(&f) - } else { - std::fs::remove_dir_all(&f) - }; - - if let Err(e) = result { - let msg = format!("Could not delete {:}: {e:}", f.to_string_lossy()); - Value::error(ShellError::RemoveNotPossible { msg, span }, span) - } else if verbose { - let msg = if interactive && !confirmed { - "not deleted" - } else { - "deleted" - }; - let val = format!("{} {:}", msg, f.to_string_lossy()); - Value::string(val, span) - } else { - Value::nothing(span) + let result = if let Err(e) = interaction { + Err(Error::new(ErrorKind::Other, &*e.to_string())) + } else if interactive && !confirmed { + Ok(()) + } else if TRASH_SUPPORTED && (trash || (rm_always_trash && !permanent)) { + #[cfg(all( + feature = "trash-support", + not(any(target_os = "android", target_os = "ios")) + ))] + { + trash::delete(&f).map_err(|e: trash::Error| { + Error::new(ErrorKind::Other, format!("{e:?}\nTry '--permanent' flag")) + }) } + + // Should not be reachable since we error earlier if + // these options are given on an unsupported platform + #[cfg(any( + not(feature = "trash-support"), + target_os = "android", + target_os = "ios" + ))] + { + unreachable!() + } + } else if metadata.is_symlink() { + // In Windows, symlink pointing to a directory can be removed using + // std::fs::remove_dir instead of std::fs::remove_file. + #[cfg(windows)] + { + f.metadata().and_then(|metadata| { + if metadata.is_dir() { + std::fs::remove_dir(&f) + } else { + std::fs::remove_file(&f) + } + }) + } + + #[cfg(not(windows))] + std::fs::remove_file(&f) + } else if metadata.is_file() || is_socket || is_fifo { + std::fs::remove_file(&f) } else { - let error = format!("Cannot remove {:}. try --recursive", f.to_string_lossy()); - Value::error( - ShellError::GenericError { - error, - msg: "cannot remove non-empty directory".into(), - span: Some(span), - help: None, - inner: vec![], - }, - span, - ) + std::fs::remove_dir_all(&f) + }; + + if let Err(e) = result { + let msg = format!("Could not delete {:}: {e:}", f.to_string_lossy()); + Err(ShellError::RemoveNotPossible { msg, span }) + } else if verbose { + let msg = if interactive && !confirmed { + "not deleted" + } else { + "deleted" + }; + Ok(Some(format!("{} {:}", msg, f.to_string_lossy()))) + } else { + Ok(None) } } else { - let error = format!("no such file or directory: {:}", f.to_string_lossy()); - Value::error( - ShellError::GenericError { - error, - msg: "no such file or directory".into(), - span: Some(span), - help: None, - inner: vec![], - }, - span, - ) + let error = format!("Cannot remove {:}. try --recursive", f.to_string_lossy()); + Err(ShellError::GenericError { + error, + msg: "cannot remove non-empty directory".into(), + span: Some(span), + help: None, + inner: vec![], + }) } - }) - .filter(|x| !matches!(x.get_type(), Type::Nothing)) - .into_pipeline_data(span, ctrlc) - .print_not_formatted(engine_state, false, true)?; + } else { + let error = format!("no such file or directory: {:}", f.to_string_lossy()); + Err(ShellError::GenericError { + error, + msg: "no such file or directory".into(), + span: Some(span), + help: None, + inner: vec![], + }) + } + }); + + for result in iter { + if nu_utils::ctrl_c::was_pressed(&engine_state.ctrlc) { + return Err(ShellError::InterruptedByUser { + span: Some(call.head), + }); + } + + match result { + Ok(None) => {} + Ok(Some(msg)) => eprintln!("{msg}"), + Err(err) => report_error_new(engine_state, &err), + } + } Ok(PipelineData::empty()) } diff --git a/crates/nu-command/src/filesystem/save.rs b/crates/nu-command/src/filesystem/save.rs index 3b92416ab5..ca9943eafb 100644 --- a/crates/nu-command/src/filesystem/save.rs +++ b/crates/nu-command/src/filesystem/save.rs @@ -1,15 +1,19 @@ use crate::progress_bar; +use nu_engine::get_eval_block; #[allow(deprecated)] use nu_engine::{command_prelude::*, current_dir}; use nu_path::expand_path_with; use nu_protocol::{ ast::{Expr, Expression}, - DataSource, OutDest, PipelineMetadata, RawStream, + byte_stream::copy_with_interrupt, + process::ChildPipe, + ByteStreamSource, DataSource, OutDest, PipelineMetadata, }; use std::{ fs::File, - io::Write, + io::{self, BufRead, BufReader, Read, Write}, path::{Path, PathBuf}, + sync::{atomic::AtomicBool, Arc}, thread, }; @@ -103,12 +107,7 @@ impl Command for Save { }); match input { - PipelineData::ExternalStream { - stdout, - stderr, - metadata, - .. - } => { + PipelineData::ByteStream(stream, metadata) => { check_saving_to_source_file(metadata.as_ref(), &path, stderr_path.as_ref())?; let (file, stderr_file) = get_files( @@ -120,40 +119,97 @@ impl Command for Save { force, )?; - match (stdout, stderr) { - (Some(stdout), stderr) => { - // delegate a thread to redirect stderr to result. - let handler = stderr - .map(|stderr| match stderr_file { - Some(stderr_file) => thread::Builder::new() - .name("stderr redirector".to_string()) - .spawn(move || { - stream_to_file(stderr, stderr_file, span, progress) - }), - None => thread::Builder::new() - .name("stderr redirector".to_string()) - .spawn(move || stderr.drain()), - }) - .transpose() - .err_span(span)?; + let size = stream.known_size(); + let ctrlc = engine_state.ctrlc.clone(); - let res = stream_to_file(stdout, file, span, progress); - if let Some(h) = handler { - h.join().map_err(|err| ShellError::ExternalCommand { - label: "Fail to receive external commands stderr message" - .to_string(), - help: format!("{err:?}"), - span, - })??; - } - res?; + match stream.into_source() { + ByteStreamSource::Read(read) => { + stream_to_file(read, size, ctrlc, file, span, progress)?; } - (None, Some(stderr)) => match stderr_file { - Some(stderr_file) => stream_to_file(stderr, stderr_file, span, progress)?, - None => stderr.drain()?, - }, - (None, None) => {} - }; + ByteStreamSource::File(source) => { + stream_to_file(source, size, ctrlc, file, span, progress)?; + } + ByteStreamSource::Child(mut child) => { + fn write_or_consume_stderr( + stderr: ChildPipe, + file: Option, + span: Span, + ctrlc: Option>, + progress: bool, + ) -> Result<(), ShellError> { + if let Some(file) = file { + match stderr { + ChildPipe::Pipe(pipe) => { + stream_to_file(pipe, None, ctrlc, file, span, progress) + } + ChildPipe::Tee(tee) => { + stream_to_file(tee, None, ctrlc, file, span, progress) + } + }? + } else { + match stderr { + ChildPipe::Pipe(mut pipe) => { + io::copy(&mut pipe, &mut io::sink()) + } + ChildPipe::Tee(mut tee) => io::copy(&mut tee, &mut io::sink()), + } + .err_span(span)?; + } + Ok(()) + } + + match (child.stdout.take(), child.stderr.take()) { + (Some(stdout), stderr) => { + // delegate a thread to redirect stderr to result. + let handler = stderr + .map(|stderr| { + let ctrlc = ctrlc.clone(); + thread::Builder::new().name("stderr saver".into()).spawn( + move || { + write_or_consume_stderr( + stderr, + stderr_file, + span, + ctrlc, + progress, + ) + }, + ) + }) + .transpose() + .err_span(span)?; + + let res = match stdout { + ChildPipe::Pipe(pipe) => { + stream_to_file(pipe, None, ctrlc, file, span, progress) + } + ChildPipe::Tee(tee) => { + stream_to_file(tee, None, ctrlc, file, span, progress) + } + }; + if let Some(h) = handler { + h.join().map_err(|err| ShellError::ExternalCommand { + label: "Fail to receive external commands stderr message" + .to_string(), + help: format!("{err:?}"), + span, + })??; + } + res?; + } + (None, Some(stderr)) => { + write_or_consume_stderr( + stderr, + stderr_file, + span, + ctrlc, + progress, + )?; + } + (None, None) => {} + }; + } + } Ok(PipelineData::Empty) } @@ -301,8 +357,7 @@ fn input_to_bytes( ) -> Result, ShellError> { let ext = if raw { None - // if is extern stream , in other words , not value - } else if let PipelineData::ExternalStream { .. } = input { + } else if let PipelineData::ByteStream(..) = input { None } else if let PipelineData::Value(Value::String { .. }, ..) = input { None @@ -311,12 +366,13 @@ fn input_to_bytes( .map(|name| name.to_string_lossy().to_string()) }; - if let Some(ext) = ext { - convert_to_extension(engine_state, &ext, stack, input, span) + let input = if let Some(ext) = ext { + convert_to_extension(engine_state, &ext, stack, input, span)? } else { - let value = input.into_value(span); - value_to_bytes(value) - } + input + }; + + value_to_bytes(input.into_value(span)?) } /// Convert given data into content of file of specified extension if @@ -328,24 +384,19 @@ fn convert_to_extension( stack: &mut Stack, input: PipelineData, span: Span, -) -> Result, ShellError> { - let converter = engine_state.find_decl(format!("to {extension}").as_bytes(), &[]); - - let output = match converter { - Some(converter_id) => { - let output = engine_state.get_decl(converter_id).run( - engine_state, - stack, - &Call::new(span), - input, - )?; - - output.into_value(span) +) -> Result { + if let Some(decl_id) = engine_state.find_decl(format!("to {extension}").as_bytes(), &[]) { + let decl = engine_state.get_decl(decl_id); + if let Some(block_id) = decl.get_block_id() { + let block = engine_state.get_block(block_id); + let eval_block = get_eval_block(engine_state); + eval_block(engine_state, stack, block, input) + } else { + decl.run(engine_state, stack, &Call::new(span), input) } - None => input.into_value(span), - }; - - value_to_bytes(output) + } else { + Ok(input) + } } /// Convert [`Value::String`] [`Value::Binary`] or [`Value::List`] into [`Vec`] of bytes @@ -451,84 +502,54 @@ fn get_files( } fn stream_to_file( - mut stream: RawStream, + mut source: impl Read, + known_size: Option, + ctrlc: Option>, mut file: File, span: Span, progress: bool, ) -> Result<(), ShellError> { - // https://github.com/nushell/nushell/pull/9377 contains the reason - // for not using BufWriter - let writer = &mut file; + // https://github.com/nushell/nushell/pull/9377 contains the reason for not using `BufWriter` + if progress { + let mut bytes_processed = 0; - let mut bytes_processed: u64 = 0; - let bytes_processed_p = &mut bytes_processed; - let file_total_size = stream.known_size; - let mut process_failed = false; - let process_failed_p = &mut process_failed; + let mut bar = progress_bar::NuProgressBar::new(known_size); - // Create the progress bar - // It looks a bit messy but I am doing it this way to avoid - // creating the bar when is not needed - let (mut bar_opt, bar_opt_clone) = if progress { - let tmp_bar = progress_bar::NuProgressBar::new(file_total_size); - let tmp_bar_clone = tmp_bar.clone(); + // TODO: reduce the number of progress bar updates? - (Some(tmp_bar), Some(tmp_bar_clone)) - } else { - (None, None) - }; + let mut reader = BufReader::new(source); - stream.try_for_each(move |result| { - let buf = match result { - Ok(v) => match v { - Value::String { val, .. } => val.into_bytes(), - Value::Binary { val, .. } => val, - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => return Err(*error), - other => { - return Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "string or binary".into(), - wrong_type: other.get_type().to_string(), - dst_span: span, - src_span: other.span(), - }); + let res = loop { + if nu_utils::ctrl_c::was_pressed(&ctrlc) { + bar.abandoned_msg("# Cancelled #".to_owned()); + return Ok(()); + } + + match reader.fill_buf() { + Ok(&[]) => break Ok(()), + Ok(buf) => { + file.write_all(buf).err_span(span)?; + let len = buf.len(); + reader.consume(len); + bytes_processed += len as u64; + bar.update_bar(bytes_processed); } - }, - Err(err) => { - *process_failed_p = true; - return Err(err); + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => break Err(e), } }; - // If the `progress` flag is set then - if progress { - // Update the total amount of bytes that has been saved and then print the progress bar - *bytes_processed_p += buf.len() as u64; - if let Some(bar) = &mut bar_opt { - bar.update_bar(*bytes_processed_p); - } - } - - if let Err(err) = writer.write_all(&buf) { - *process_failed_p = true; - return Err(ShellError::IOError { - msg: err.to_string(), - }); - } - Ok(()) - })?; - - // If the `progress` flag is set then - if progress { // If the process failed, stop the progress bar with an error message. - if process_failed { - if let Some(bar) = bar_opt_clone { - bar.abandoned_msg("# Error while saving #".to_owned()); - } + if let Err(err) = res { + let _ = file.flush(); + bar.abandoned_msg("# Error while saving #".to_owned()); + Err(err.into_spanned(span).into()) + } else { + file.flush().err_span(span)?; + Ok(()) } + } else { + copy_with_interrupt(&mut source, &mut file, span, ctrlc.as_deref())?; + Ok(()) } - - file.flush()?; - - Ok(()) } diff --git a/crates/nu-command/src/filesystem/touch.rs b/crates/nu-command/src/filesystem/touch.rs index ce8ed0dad3..08843b11f0 100644 --- a/crates/nu-command/src/filesystem/touch.rs +++ b/crates/nu-command/src/filesystem/touch.rs @@ -117,7 +117,7 @@ impl Command for Touch { #[allow(deprecated)] let cwd = current_dir(engine_state, stack)?; - for (index, glob) in files.into_iter().enumerate() { + for glob in files { let path = expand_path_with(glob.item.as_ref(), &cwd, glob.item.is_expand()); // If --no-create is passed and the file/dir does not exist there's nothing to do @@ -135,10 +135,7 @@ impl Command for Touch { { return Err(ShellError::CreateNotPossible { msg: format!("Failed to create file: {err}"), - span: call - .positional_nth(index) - .expect("already checked positional") - .span, + span: glob.span, }); }; } @@ -148,10 +145,7 @@ impl Command for Touch { { return Err(ShellError::ChangeModifiedTimeNotPossible { msg: format!("Failed to change the modified time: {err}"), - span: call - .positional_nth(index) - .expect("already checked positional") - .span, + span: glob.span, }); }; } @@ -161,10 +155,7 @@ impl Command for Touch { { return Err(ShellError::ChangeAccessTimeNotPossible { msg: format!("Failed to change the access time: {err}"), - span: call - .positional_nth(index) - .expect("already checked positional") - .span, + span: glob.span, }); }; } diff --git a/crates/nu-command/src/filters/columns.rs b/crates/nu-command/src/filters/columns.rs index 67c5fe98bd..96f98bf4f1 100644 --- a/crates/nu-command/src/filters/columns.rs +++ b/crates/nu-command/src/filters/columns.rs @@ -124,13 +124,11 @@ fn getcol(head: Span, input: PipelineData) -> Result { .into_pipeline_data() .set_metadata(metadata)) } - PipelineData::ExternalStream { .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "record or table".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: head, - src_span: input - .span() - .expect("PipelineData::ExternalStream had no span"), + src_span: stream.span(), }), } } diff --git a/crates/nu-command/src/filters/compact.rs b/crates/nu-command/src/filters/compact.rs index d8872448be..9a4e968e9b 100644 --- a/crates/nu-command/src/filters/compact.rs +++ b/crates/nu-command/src/filters/compact.rs @@ -31,6 +31,10 @@ impl Command for Compact { "Creates a table with non-empty rows." } + fn search_terms(&self) -> Vec<&str> { + vec!["empty", "remove"] + } + fn run( &self, engine_state: &EngineState, diff --git a/crates/nu-command/src/filters/drop/column.rs b/crates/nu-command/src/filters/drop/column.rs index 689a335923..ea595d4cb2 100644 --- a/crates/nu-command/src/filters/drop/column.rs +++ b/crates/nu-command/src/filters/drop/column.rs @@ -133,11 +133,11 @@ fn drop_cols( } } PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::ExternalStream { span, .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "table or record".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: head, - src_span: span, + src_span: stream.span(), }), } } diff --git a/crates/nu-command/src/filters/each.rs b/crates/nu-command/src/filters/each.rs index 65d61fd3a8..a074f63abb 100644 --- a/crates/nu-command/src/filters/each.rs +++ b/crates/nu-command/src/filters/each.rs @@ -129,7 +129,9 @@ with 'transpose' first."# } Some(Value::list(vals, span)) } - Ok(data) => Some(data.into_value(head)), + Ok(data) => Some(data.into_value(head).unwrap_or_else(|err| { + Value::error(chain_error_with_input(err, is_error, span), span) + })), Err(ShellError::Continue { span }) => Some(Value::nothing(span)), Err(ShellError::Break { .. }) => None, Err(error) => { @@ -140,37 +142,39 @@ with 'transpose' first."# }) .into_pipeline_data(head, engine_state.ctrlc.clone())) } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - let mut closure = ClosureEval::new(engine_state, stack, closure); - Ok(stream - .into_iter() - .map_while(move |value| { - let value = match value { - Ok(value) => value, - Err(ShellError::Continue { span }) => { - return Some(Value::nothing(span)) - } - Err(ShellError::Break { .. }) => return None, - Err(err) => return Some(Value::error(err, head)), - }; + PipelineData::ByteStream(stream, ..) => { + if let Some(chunks) = stream.chunks() { + let mut closure = ClosureEval::new(engine_state, stack, closure); + Ok(chunks + .map_while(move |value| { + let value = match value { + Ok(value) => value, + Err(ShellError::Continue { span }) => { + return Some(Value::nothing(span)) + } + Err(ShellError::Break { .. }) => return None, + Err(err) => return Some(Value::error(err, head)), + }; - let span = value.span(); - let is_error = value.is_error(); - match closure.run_with_value(value) { - Ok(data) => Some(data.into_value(head)), - Err(ShellError::Continue { span }) => Some(Value::nothing(span)), - Err(ShellError::Break { .. }) => None, - Err(error) => { - let error = chain_error_with_input(error, is_error, span); - Some(Value::error(error, span)) + let span = value.span(); + let is_error = value.is_error(); + match closure + .run_with_value(value) + .and_then(|data| data.into_value(head)) + { + Ok(value) => Some(value), + Err(ShellError::Continue { span }) => Some(Value::nothing(span)), + Err(ShellError::Break { .. }) => None, + Err(error) => { + let error = chain_error_with_input(error, is_error, span); + Some(Value::error(error, span)) + } } - } - }) - .into_pipeline_data(head, engine_state.ctrlc.clone())) + }) + .into_pipeline_data(head, engine_state.ctrlc.clone())) + } else { + Ok(PipelineData::Empty) + } } // This match allows non-iterables to be accepted, // which is currently considered undesirable (Nov 2022). diff --git a/crates/nu-command/src/filters/empty.rs b/crates/nu-command/src/filters/empty.rs index fd55921414..f4dd428b77 100644 --- a/crates/nu-command/src/filters/empty.rs +++ b/crates/nu-command/src/filters/empty.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use std::io::Read; pub fn empty( engine_state: &EngineState, @@ -36,29 +37,26 @@ pub fn empty( } else { match input { PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::ExternalStream { stdout, .. } => match stdout { - Some(s) => { - let bytes = s.into_bytes(); - - match bytes { - Ok(s) => { - if negate { - Ok(Value::bool(!s.item.is_empty(), head).into_pipeline_data()) - } else { - Ok(Value::bool(s.item.is_empty(), head).into_pipeline_data()) - } + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + match stream.reader() { + Some(reader) => { + let is_empty = reader.bytes().next().transpose().err_span(span)?.is_none(); + if negate { + Ok(Value::bool(!is_empty, head).into_pipeline_data()) + } else { + Ok(Value::bool(is_empty, head).into_pipeline_data()) + } + } + None => { + if negate { + Ok(Value::bool(false, head).into_pipeline_data()) + } else { + Ok(Value::bool(true, head).into_pipeline_data()) } - Err(err) => Err(err), } } - None => { - if negate { - Ok(Value::bool(false, head).into_pipeline_data()) - } else { - Ok(Value::bool(true, head).into_pipeline_data()) - } - } - }, + } PipelineData::ListStream(s, ..) => { let empty = s.into_iter().next().is_none(); if negate { diff --git a/crates/nu-command/src/filters/filter.rs b/crates/nu-command/src/filters/filter.rs index b158dd3be3..1ba1508839 100644 --- a/crates/nu-command/src/filters/filter.rs +++ b/crates/nu-command/src/filters/filter.rs @@ -58,33 +58,13 @@ a variable. On the other hand, the "row condition" syntax is not supported."# | PipelineData::ListStream(..) => { let mut closure = ClosureEval::new(engine_state, stack, closure); Ok(input - .into_iter() - .filter_map(move |value| match closure.run_with_value(value.clone()) { - Ok(pred) => pred.into_value(head).is_true().then_some(value), - Err(err) => { - let span = value.span(); - let err = chain_error_with_input(err, value.is_error(), span); - Some(Value::error(err, span)) - } - }) - .into_pipeline_data(head, engine_state.ctrlc.clone())) - } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - let mut closure = ClosureEval::new(engine_state, stack, closure); - Ok(stream .into_iter() .filter_map(move |value| { - let value = match value { - Ok(value) => value, - Err(err) => return Some(Value::error(err, head)), - }; - - match closure.run_with_value(value.clone()) { - Ok(pred) => pred.into_value(head).is_true().then_some(value), + match closure + .run_with_value(value.clone()) + .and_then(|data| data.into_value(head)) + { + Ok(cond) => cond.is_true().then_some(value), Err(err) => { let span = value.span(); let err = chain_error_with_input(err, value.is_error(), span); @@ -94,14 +74,43 @@ a variable. On the other hand, the "row condition" syntax is not supported."# }) .into_pipeline_data(head, engine_state.ctrlc.clone())) } + PipelineData::ByteStream(stream, ..) => { + if let Some(chunks) = stream.chunks() { + let mut closure = ClosureEval::new(engine_state, stack, closure); + Ok(chunks + .into_iter() + .filter_map(move |value| { + let value = match value { + Ok(value) => value, + Err(err) => return Some(Value::error(err, head)), + }; + + match closure + .run_with_value(value.clone()) + .and_then(|data| data.into_value(head)) + { + Ok(cond) => cond.is_true().then_some(value), + Err(err) => { + let span = value.span(); + let err = chain_error_with_input(err, value.is_error(), span); + Some(Value::error(err, span)) + } + } + }) + .into_pipeline_data(head, engine_state.ctrlc.clone())) + } else { + Ok(PipelineData::Empty) + } + } // This match allows non-iterables to be accepted, // which is currently considered undesirable (Nov 2022). PipelineData::Value(value, ..) => { let result = ClosureEvalOnce::new(engine_state, stack, closure) - .run_with_value(value.clone()); + .run_with_value(value.clone()) + .and_then(|data| data.into_value(head)); Ok(match result { - Ok(pred) => pred.into_value(head).is_true().then_some(value), + Ok(cond) => cond.is_true().then_some(value), Err(err) => { let span = value.span(); let err = chain_error_with_input(err, value.is_error(), span); diff --git a/crates/nu-command/src/filters/find.rs b/crates/nu-command/src/filters/find.rs index b45fe8d810..dfdef66969 100644 --- a/crates/nu-command/src/filters/find.rs +++ b/crates/nu-command/src/filters/find.rs @@ -447,57 +447,35 @@ fn find_with_rest_and_highlight( Ok(PipelineData::ListStream(stream, metadata)) } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - let mut output: Vec = vec![]; - for filter_val in stream { - match filter_val { - Ok(value) => { - let span = value.span(); - match value { - Value::String { val, .. } => { - let split_char = if val.contains("\r\n") { "\r\n" } else { "\n" }; + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + if let Some(lines) = stream.lines() { + let terms = lower_terms + .into_iter() + .map(|term| term.to_expanded_string("", &filter_config).to_lowercase()) + .collect::>(); - for line in val.split(split_char) { - for term in lower_terms.iter() { - let term_str = term.to_expanded_string("", &filter_config); - let lower_val = line.to_lowercase(); - if lower_val.contains( - &term.to_expanded_string("", &config).to_lowercase(), - ) { - output.push(Value::string( - highlight_search_string( - line, - &term_str, - &string_style, - &highlight_style, - )?, - span, - )) - } - } - } - } - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => return Err(*error), - other => { - return Err(ShellError::UnsupportedInput { - msg: "unsupported type from raw stream".into(), - input: format!("input: {:?}", other.get_type()), - msg_span: span, - input_span: other.span(), - }); - } + let mut output: Vec = vec![]; + for line in lines { + let line = line?.to_lowercase(); + for term in &terms { + if line.contains(term) { + output.push(Value::string( + highlight_search_string( + &line, + term, + &string_style, + &highlight_style, + )?, + span, + )) } } - // Propagate any errors that were in the stream - Err(e) => return Err(e), - }; + } + Ok(Value::list(output, span).into_pipeline_data()) + } else { + Ok(PipelineData::Empty) } - Ok(output.into_pipeline_data(span, ctrlc)) } } } diff --git a/crates/nu-command/src/filters/first.rs b/crates/nu-command/src/filters/first.rs index 1bc51f2562..e581c3e84d 100644 --- a/crates/nu-command/src/filters/first.rs +++ b/crates/nu-command/src/filters/first.rs @@ -170,11 +170,11 @@ fn first_helper( )) } } - PipelineData::ExternalStream { span, .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: head, - src_span: span, + src_span: stream.span(), }), PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), diff --git a/crates/nu-command/src/filters/get.rs b/crates/nu-command/src/filters/get.rs index c30ad1f3a4..07f0ea9440 100644 --- a/crates/nu-command/src/filters/get.rs +++ b/crates/nu-command/src/filters/get.rs @@ -81,7 +81,7 @@ If multiple cell paths are given, this will produce a list of values."# let paths = std::iter::once(cell_path).chain(rest); - let input = input.into_value(span); + let input = input.into_value(span)?; for path in paths { let val = input.clone().follow_cell_path(&path.members, !sensitive); @@ -115,7 +115,7 @@ If multiple cell paths are given, this will produce a list of values."# }, Example { description: - "Extract the name of the 3rd record in a list (same as `ls | $in.name`)", + "Extract the name of the 3rd record in a list (same as `ls | $in.name.2`)", example: "ls | get name.2", result: None, }, diff --git a/crates/nu-command/src/filters/group_by.rs b/crates/nu-command/src/filters/group_by.rs index acd5ae5b1a..c1d76ebe08 100644 --- a/crates/nu-command/src/filters/group_by.rs +++ b/crates/nu-command/src/filters/group_by.rs @@ -139,7 +139,7 @@ pub fn group_by( match grouper { Value::CellPath { val, .. } => group_cell_path(val, values)?, Value::Closure { val, .. } => { - group_closure(values, span, val, engine_state, stack)? + group_closure(values, span, *val, engine_state, stack)? } _ => { return Err(ShellError::TypeMismatch { @@ -207,7 +207,7 @@ fn group_closure( for value in values { let key = closure .run_with_value(value.clone())? - .into_value(span) + .into_value(span)? .coerce_into_string()?; groups.entry(key).or_default().push(value); diff --git a/crates/nu-command/src/filters/headers.rs b/crates/nu-command/src/filters/headers.rs index cacf88daa2..da316a52f3 100644 --- a/crates/nu-command/src/filters/headers.rs +++ b/crates/nu-command/src/filters/headers.rs @@ -66,7 +66,7 @@ impl Command for Headers { let config = engine_state.get_config(); let metadata = input.metadata(); let span = input.span().unwrap_or(call.head); - let value = input.into_value(span); + let value = input.into_value(span)?; let Value::List { vals: table, .. } = value else { return Err(ShellError::TypeMismatch { err_message: "not a table".to_string(), diff --git a/crates/nu-command/src/filters/insert.rs b/crates/nu-command/src/filters/insert.rs index c87a2a78b9..e8794304c8 100644 --- a/crates/nu-command/src/filters/insert.rs +++ b/crates/nu-command/src/filters/insert.rs @@ -133,7 +133,7 @@ fn insert( if let Value::Closure { val, .. } = replacement { match (cell_path.members.first(), &mut value) { (Some(PathMember::String { .. }), Value::List { vals, .. }) => { - let mut closure = ClosureEval::new(engine_state, stack, val); + let mut closure = ClosureEval::new(engine_state, stack, *val); for val in vals { insert_value_by_closure( val, @@ -147,7 +147,7 @@ fn insert( (first, _) => { insert_single_value_by_closure( &mut value, - ClosureEvalOnce::new(engine_state, stack, val), + ClosureEvalOnce::new(engine_state, stack, *val), head, &cell_path.members, matches!(first, Some(PathMember::Int { .. })), @@ -188,9 +188,9 @@ fn insert( let value = stream.next(); let end_of_stream = value.is_none(); let value = value.unwrap_or(Value::nothing(head)); - let new_value = ClosureEvalOnce::new(engine_state, stack, val) + let new_value = ClosureEvalOnce::new(engine_state, stack, *val) .run_with_value(value.clone())? - .into_value(head); + .into_value(head)?; pre_elems.push(new_value); if !end_of_stream { @@ -203,7 +203,7 @@ fn insert( if let Value::Closure { val, .. } = replacement { insert_single_value_by_closure( &mut value, - ClosureEvalOnce::new(engine_state, stack, val), + ClosureEvalOnce::new(engine_state, stack, *val), head, path, true, @@ -224,7 +224,7 @@ fn insert( .chain(stream) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) } else if let Value::Closure { val, .. } = replacement { - let mut closure = ClosureEval::new(engine_state, stack, val); + let mut closure = ClosureEval::new(engine_state, stack, *val); let stream = stream.map(move |mut value| { let err = insert_value_by_closure( &mut value, @@ -261,8 +261,8 @@ fn insert( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ExternalStream { .. } => Err(ShellError::IncompatiblePathAccess { - type_name: "external stream".to_string(), + PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { + type_name: "byte stream".to_string(), span: head, }), } @@ -284,7 +284,7 @@ fn insert_value_by_closure( value.clone() }; - let new_value = closure.run_with_value(value_at_path)?.into_value(span); + let new_value = closure.run_with_value(value_at_path)?.into_value(span)?; value.insert_data_at_cell_path(cell_path, new_value, span) } @@ -304,7 +304,7 @@ fn insert_single_value_by_closure( value.clone() }; - let new_value = closure.run_with_value(value_at_path)?.into_value(span); + let new_value = closure.run_with_value(value_at_path)?.into_value(span)?; value.insert_data_at_cell_path(cell_path, new_value, span) } diff --git a/crates/nu-command/src/filters/items.rs b/crates/nu-command/src/filters/items.rs index 490477d965..a75e5e6cbf 100644 --- a/crates/nu-command/src/filters/items.rs +++ b/crates/nu-command/src/filters/items.rs @@ -54,10 +54,11 @@ impl Command for Items { let result = closure .add_arg(Value::string(col, span)) .add_arg(val) - .run_with_input(PipelineData::Empty); + .run_with_input(PipelineData::Empty) + .and_then(|data| data.into_value(head)); match result { - Ok(data) => Some(data.into_value(head)), + Ok(value) => Some(value), Err(ShellError::Break { .. }) => None, Err(err) => { let err = chain_error_with_input(err, false, span); @@ -76,20 +77,18 @@ impl Command for Items { }), } } - PipelineData::ListStream(..) => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ListStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "record".into(), wrong_type: "stream".into(), - dst_span: head, - src_span: head, + dst_span: call.head, + src_span: stream.span(), + }), + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "record".into(), + wrong_type: "byte stream".into(), + dst_span: call.head, + src_span: stream.span(), }), - PipelineData::ExternalStream { span, .. } => { - Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "record".into(), - wrong_type: "raw data".into(), - dst_span: head, - src_span: span, - }) - } } .map(|data| data.set_metadata(metadata)) } diff --git a/crates/nu-command/src/filters/join.rs b/crates/nu-command/src/filters/join.rs index 343cc0eb19..f5e6d63deb 100644 --- a/crates/nu-command/src/filters/join.rs +++ b/crates/nu-command/src/filters/join.rs @@ -75,7 +75,7 @@ impl Command for Join { let join_type = join_type(engine_state, stack, call)?; // FIXME: we should handle ListStreams properly instead of collecting - let collected_input = input.into_value(span); + let collected_input = input.into_value(span)?; match (&collected_input, &table_2, &l_on, &r_on) { ( diff --git a/crates/nu-command/src/filters/last.rs b/crates/nu-command/src/filters/last.rs index f41b7c7e4d..7530126c26 100644 --- a/crates/nu-command/src/filters/last.rs +++ b/crates/nu-command/src/filters/last.rs @@ -160,14 +160,12 @@ impl Command for Last { }), } } - PipelineData::ExternalStream { span, .. } => { - Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "raw data".into(), - dst_span: head, - src_span: span, - }) - } + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: "byte stream".into(), + dst_span: head, + src_span: stream.span(), + }), PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), wrong_type: "null".into(), diff --git a/crates/nu-command/src/filters/lines.rs b/crates/nu-command/src/filters/lines.rs index e3e0f5d9fd..0b037dcaac 100644 --- a/crates/nu-command/src/filters/lines.rs +++ b/crates/nu-command/src/filters/lines.rs @@ -1,6 +1,4 @@ use nu_engine::command_prelude::*; -use nu_protocol::RawStream; -use std::collections::VecDeque; #[derive(Clone)] pub struct Lines; @@ -33,23 +31,33 @@ impl Command for Lines { let span = input.span().unwrap_or(call.head); match input { - PipelineData::Value(Value::String { val, .. }, ..) => { - let lines = if skip_empty { - val.lines() - .filter_map(|s| { - if s.trim().is_empty() { - None - } else { - Some(Value::string(s, span)) - } - }) - .collect() - } else { - val.lines().map(|s| Value::string(s, span)).collect() - }; + PipelineData::Value(value, ..) => match value { + Value::String { val, .. } => { + let lines = if skip_empty { + val.lines() + .filter_map(|s| { + if s.trim().is_empty() { + None + } else { + Some(Value::string(s, span)) + } + }) + .collect() + } else { + val.lines().map(|s| Value::string(s, span)).collect() + }; - Ok(Value::list(lines, span).into_pipeline_data()) - } + Ok(Value::list(lines, span).into_pipeline_data()) + } + // Propagate existing errors + Value::Error { error, .. } => Err(*error), + value => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "string or byte stream".into(), + wrong_type: value.get_type().to_string(), + dst_span: head, + src_span: value.span(), + }), + }, PipelineData::Empty => Ok(PipelineData::Empty), PipelineData::ListStream(stream, metadata) => { let stream = stream.modify(|iter| { @@ -76,27 +84,18 @@ impl Command for Lines { Ok(PipelineData::ListStream(stream, metadata)) } - PipelineData::Value(val, ..) => { - match val { - // Propagate existing errors - Value::Error { error, .. } => Err(*error), - _ => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "string or raw data".into(), - wrong_type: val.get_type().to_string(), - dst_span: head, - src_span: val.span(), - }), + PipelineData::ByteStream(stream, ..) => { + if let Some(lines) = stream.lines() { + Ok(lines + .map(move |line| match line { + Ok(line) => Value::string(line, head), + Err(err) => Value::error(err, head), + }) + .into_pipeline_data(head, ctrlc)) + } else { + Ok(PipelineData::empty()) } } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - metadata, - .. - } => Ok(RawStreamLinesAdapter::new(stream, head, skip_empty) - .map(move |x| x.unwrap_or_else(|err| Value::error(err, head))) - .into_pipeline_data(head, ctrlc) - .set_metadata(metadata)), } } @@ -112,108 +111,6 @@ impl Command for Lines { } } -#[derive(Debug)] -struct RawStreamLinesAdapter { - inner: RawStream, - inner_complete: bool, - skip_empty: bool, - span: Span, - incomplete_line: String, - queue: VecDeque, -} - -impl Iterator for RawStreamLinesAdapter { - type Item = Result; - - fn next(&mut self) -> Option { - loop { - if let Some(s) = self.queue.pop_front() { - if self.skip_empty && s.trim().is_empty() { - continue; - } - return Some(Ok(Value::string(s, self.span))); - } else { - // inner is complete, feed out remaining state - if self.inner_complete { - return if self.incomplete_line.is_empty() { - None - } else { - Some(Ok(Value::string( - std::mem::take(&mut self.incomplete_line), - self.span, - ))) - }; - } - - // pull more data from inner - if let Some(result) = self.inner.next() { - match result { - Ok(v) => { - let span = v.span(); - match v { - // TODO: Value::Binary support required? - Value::String { val, .. } => { - self.span = span; - - let mut lines = val.lines(); - - // handle incomplete line from previous - if !self.incomplete_line.is_empty() { - if let Some(first) = lines.next() { - self.incomplete_line.push_str(first); - self.queue.push_back(std::mem::take( - &mut self.incomplete_line, - )); - } - } - - // save completed lines - self.queue.extend(lines.map(String::from)); - - if !val.ends_with('\n') { - // incomplete line, save for next time - // if `val` and `incomplete_line` were empty, - // then pop will return none - if let Some(s) = self.queue.pop_back() { - self.incomplete_line = s; - } - } - } - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => return Some(Err(*error)), - other => { - return Some(Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "string".into(), - wrong_type: other.get_type().to_string(), - dst_span: self.span, - src_span: other.span(), - })); - } - } - } - Err(err) => return Some(Err(err)), - } - } else { - self.inner_complete = true; - } - } - } - } -} - -impl RawStreamLinesAdapter { - pub fn new(inner: RawStream, span: Span, skip_empty: bool) -> Self { - Self { - inner, - span, - skip_empty, - incomplete_line: String::new(), - queue: VecDeque::new(), - inner_complete: false, - } - } -} - #[cfg(test)] mod test { use super::*; diff --git a/crates/nu-command/src/filters/par_each.rs b/crates/nu-command/src/filters/par_each.rs index 52c3024270..af72895df6 100644 --- a/crates/nu-command/src/filters/par_each.rs +++ b/crates/nu-command/src/filters/par_each.rs @@ -143,17 +143,16 @@ impl Command for ParEach { .map(move |(index, value)| { let span = value.span(); let is_error = value.is_error(); - let result = + let value = ClosureEvalOnce::new(engine_state, stack, closure.clone()) - .run_with_value(value); - - let value = match result { - Ok(data) => data.into_value(span), - Err(err) => Value::error( - chain_error_with_input(err, is_error, span), - span, - ), - }; + .run_with_value(value) + .and_then(|data| data.into_value(span)) + .unwrap_or_else(|err| { + Value::error( + chain_error_with_input(err, is_error, span), + span, + ) + }); (index, value) }) @@ -170,17 +169,16 @@ impl Command for ParEach { .map(move |(index, value)| { let span = value.span(); let is_error = value.is_error(); - let result = + let value = ClosureEvalOnce::new(engine_state, stack, closure.clone()) - .run_with_value(value); - - let value = match result { - Ok(data) => data.into_value(span), - Err(err) => Value::error( - chain_error_with_input(err, is_error, span), - span, - ), - }; + .run_with_value(value) + .and_then(|data| data.into_value(span)) + .unwrap_or_else(|err| { + Value::error( + chain_error_with_input(err, is_error, span), + span, + ) + }); (index, value) }) @@ -203,40 +201,12 @@ impl Command for ParEach { .map(move |(index, value)| { let span = value.span(); let is_error = value.is_error(); - let result = ClosureEvalOnce::new(engine_state, stack, closure.clone()) - .run_with_value(value); - - let value = match result { - Ok(data) => data.into_value(head), - Err(err) => { - Value::error(chain_error_with_input(err, is_error, span), span) - } - }; - - (index, value) - }) - .collect::>(); - - apply_order(vec).into_pipeline_data(head, engine_state.ctrlc.clone()) - })), - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => Ok(create_pool(max_threads)?.install(|| { - let vec = stream - .enumerate() - .par_bridge() - .map(move |(index, value)| { - let value = match value { - Ok(value) => value, - Err(err) => return (index, Value::error(err, head)), - }; - let value = ClosureEvalOnce::new(engine_state, stack, closure.clone()) .run_with_value(value) - .map(|data| data.into_value(head)) - .unwrap_or_else(|err| Value::error(err, head)); + .and_then(|data| data.into_value(head)) + .unwrap_or_else(|err| { + Value::error(chain_error_with_input(err, is_error, span), span) + }); (index, value) }) @@ -244,6 +214,34 @@ impl Command for ParEach { apply_order(vec).into_pipeline_data(head, engine_state.ctrlc.clone()) })), + PipelineData::ByteStream(stream, ..) => { + if let Some(chunks) = stream.chunks() { + Ok(create_pool(max_threads)?.install(|| { + let vec = chunks + .enumerate() + .par_bridge() + .map(move |(index, value)| { + let value = match value { + Ok(value) => value, + Err(err) => return (index, Value::error(err, head)), + }; + + let value = + ClosureEvalOnce::new(engine_state, stack, closure.clone()) + .run_with_value(value) + .and_then(|data| data.into_value(head)) + .unwrap_or_else(|err| Value::error(err, head)); + + (index, value) + }) + .collect::>(); + + apply_order(vec).into_pipeline_data(head, engine_state.ctrlc.clone()) + })) + } else { + Ok(PipelineData::empty()) + } + } } .and_then(|x| x.filter(|v| !v.is_nothing(), engine_state.ctrlc.clone())) .map(|data| data.set_metadata(metadata)) diff --git a/crates/nu-command/src/filters/reduce.rs b/crates/nu-command/src/filters/reduce.rs index 756fe051a9..fc808ca9af 100644 --- a/crates/nu-command/src/filters/reduce.rs +++ b/crates/nu-command/src/filters/reduce.rs @@ -115,7 +115,7 @@ impl Command for Reduce { .add_arg(value) .add_arg(acc) .run_with_input(PipelineData::Empty)? - .into_value(head); + .into_value(head)?; } Ok(acc.with_span(head).into_pipeline_data()) diff --git a/crates/nu-command/src/filters/reject.rs b/crates/nu-command/src/filters/reject.rs index 251e92c905..f8583d3f47 100644 --- a/crates/nu-command/src/filters/reject.rs +++ b/crates/nu-command/src/filters/reject.rs @@ -173,7 +173,7 @@ fn reject( ) -> Result { let mut unique_rows: HashSet = HashSet::new(); let metadata = input.metadata(); - let val = input.into_value(span); + let val = input.into_value(span)?; let mut val = val; let mut new_columns = vec![]; let mut new_rows = vec![]; diff --git a/crates/nu-command/src/filters/skip/skip_.rs b/crates/nu-command/src/filters/skip/skip_.rs index 1919263aa3..9048b34a58 100644 --- a/crates/nu-command/src/filters/skip/skip_.rs +++ b/crates/nu-command/src/filters/skip/skip_.rs @@ -87,15 +87,14 @@ impl Command for Skip { let ctrlc = engine_state.ctrlc.clone(); let input_span = input.span().unwrap_or(call.head); match input { - PipelineData::ExternalStream { .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: call.head, - src_span: input_span, + src_span: stream.span(), }), PipelineData::Value(Value::Binary { val, .. }, metadata) => { let bytes = val.into_iter().skip(n).collect::>(); - Ok(Value::binary(bytes, input_span).into_pipeline_data_with_metadata(metadata)) } _ => Ok(input diff --git a/crates/nu-command/src/filters/skip/skip_until.rs b/crates/nu-command/src/filters/skip/skip_until.rs index 74deeda84d..bb36785e00 100644 --- a/crates/nu-command/src/filters/skip/skip_until.rs +++ b/crates/nu-command/src/filters/skip/skip_until.rs @@ -85,7 +85,8 @@ impl Command for SkipUntil { .skip_while(move |value| { closure .run_with_value(value.clone()) - .map(|data| data.into_value(head).is_false()) + .and_then(|data| data.into_value(head)) + .map(|cond| cond.is_false()) .unwrap_or(false) }) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) diff --git a/crates/nu-command/src/filters/skip/skip_while.rs b/crates/nu-command/src/filters/skip/skip_while.rs index a832d8f7b1..2747ea6f97 100644 --- a/crates/nu-command/src/filters/skip/skip_while.rs +++ b/crates/nu-command/src/filters/skip/skip_while.rs @@ -90,7 +90,8 @@ impl Command for SkipWhile { .skip_while(move |value| { closure .run_with_value(value.clone()) - .map(|data| data.into_value(head).is_true()) + .and_then(|data| data.into_value(head)) + .map(|cond| cond.is_true()) .unwrap_or(false) }) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) diff --git a/crates/nu-command/src/filters/take/take_.rs b/crates/nu-command/src/filters/take/take_.rs index 01700420b8..12840aa8d6 100644 --- a/crates/nu-command/src/filters/take/take_.rs +++ b/crates/nu-command/src/filters/take/take_.rs @@ -78,14 +78,12 @@ impl Command for Take { stream.modify(|iter| iter.take(rows_desired)), metadata, )), - PipelineData::ExternalStream { span, .. } => { - Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "raw data".into(), - dst_span: head, - src_span: span, - }) - } + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: "byte stream".into(), + dst_span: head, + src_span: stream.span(), + }), PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), wrong_type: "null".into(), diff --git a/crates/nu-command/src/filters/take/take_until.rs b/crates/nu-command/src/filters/take/take_until.rs index e3a2a37162..0df2407cb1 100644 --- a/crates/nu-command/src/filters/take/take_until.rs +++ b/crates/nu-command/src/filters/take/take_until.rs @@ -81,7 +81,8 @@ impl Command for TakeUntil { .take_while(move |value| { closure .run_with_value(value.clone()) - .map(|data| data.into_value(head).is_false()) + .and_then(|data| data.into_value(head)) + .map(|cond| cond.is_false()) .unwrap_or(false) }) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) diff --git a/crates/nu-command/src/filters/take/take_while.rs b/crates/nu-command/src/filters/take/take_while.rs index 632c165847..7c282ac38a 100644 --- a/crates/nu-command/src/filters/take/take_while.rs +++ b/crates/nu-command/src/filters/take/take_while.rs @@ -81,7 +81,8 @@ impl Command for TakeWhile { .take_while(move |value| { closure .run_with_value(value.clone()) - .map(|data| data.into_value(head).is_true()) + .and_then(|data| data.into_value(head)) + .map(|cond| cond.is_true()) .unwrap_or(false) }) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) diff --git a/crates/nu-command/src/filters/tee.rs b/crates/nu-command/src/filters/tee.rs index 319f70905c..936dee5c79 100644 --- a/crates/nu-command/src/filters/tee.rs +++ b/crates/nu-command/src/filters/tee.rs @@ -1,6 +1,17 @@ use nu_engine::{command_prelude::*, get_eval_block_with_early_return}; -use nu_protocol::{engine::Closure, OutDest, RawStream}; -use std::{sync::mpsc, thread}; +use nu_protocol::{ + byte_stream::copy_with_interrupt, engine::Closure, process::ChildPipe, ByteStream, + ByteStreamSource, OutDest, +}; +use std::{ + io::{self, Read, Write}, + sync::{ + atomic::AtomicBool, + mpsc::{self, Sender}, + Arc, + }, + thread::{self, JoinHandle}, +}; #[derive(Clone)] pub struct Tee; @@ -67,138 +78,205 @@ use it in your pipeline."# let head = call.head; let use_stderr = call.has_flag(engine_state, stack, "stderr")?; - let Spanned { - item: Closure { block_id, captures }, - span: closure_span, - } = call.req(engine_state, stack, 0)?; + let closure: Spanned = call.req(engine_state, stack, 0)?; + let closure_span = closure.span; + let closure = closure.item; - let closure_engine_state = engine_state.clone(); - let mut closure_stack = stack - .captures_to_stack_preserve_out_dest(captures) - .reset_pipes(); + let mut eval_block = { + let closure_engine_state = engine_state.clone(); + let mut closure_stack = stack + .captures_to_stack_preserve_out_dest(closure.captures) + .reset_pipes(); + let eval_block_with_early_return = get_eval_block_with_early_return(engine_state); - let metadata = input.metadata(); - let metadata_clone = metadata.clone(); + move |input| { + let result = eval_block_with_early_return( + &closure_engine_state, + &mut closure_stack, + closure_engine_state.get_block(closure.block_id), + input, + ); + // Make sure to drain any iterator produced to avoid unexpected behavior + result.and_then(|data| data.drain().map(|_| ())) + } + }; - let eval_block_with_early_return = get_eval_block_with_early_return(engine_state); + if let PipelineData::ByteStream(stream, metadata) = input { + let span = stream.span(); + let ctrlc = engine_state.ctrlc.clone(); + let eval_block = { + let metadata = metadata.clone(); + move |stream| eval_block(PipelineData::ByteStream(stream, metadata)) + }; - match input { - // Handle external streams specially, to make sure they pass through - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - } => { - let known_size = if use_stderr { - stderr.as_ref().and_then(|s| s.known_size) - } else { - stdout.as_ref().and_then(|s| s.known_size) - }; + match stream.into_source() { + ByteStreamSource::Read(read) => { + if use_stderr { + return stderr_misuse(span, head); + } - let with_stream = move |rx: mpsc::Receiver, ShellError>>| { - let iter = rx.into_iter(); - let input_from_channel = PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(iter), - closure_engine_state.ctrlc.clone(), - span, - known_size, - )), - stderr: None, - exit_code: None, - span, - metadata: metadata_clone, - trim_end_newline, + let tee = IoTee::new(read, span, eval_block)?; + + Ok(PipelineData::ByteStream( + ByteStream::read(tee, span, ctrlc), + metadata, + )) + } + ByteStreamSource::File(file) => { + if use_stderr { + return stderr_misuse(span, head); + } + + let tee = IoTee::new(file, span, eval_block)?; + + Ok(PipelineData::ByteStream( + ByteStream::read(tee, span, ctrlc), + metadata, + )) + } + ByteStreamSource::Child(mut child) => { + let stderr_thread = if use_stderr { + let stderr_thread = if let Some(stderr) = child.stderr.take() { + match stack.stderr() { + OutDest::Pipe | OutDest::Capture => { + let tee = IoTee::new(stderr, span, eval_block)?; + child.stderr = Some(ChildPipe::Tee(Box::new(tee))); + None + } + OutDest::Null => Some(tee_pipe_on_thread( + stderr, + io::sink(), + span, + ctrlc.as_ref(), + eval_block, + )?), + OutDest::Inherit => Some(tee_pipe_on_thread( + stderr, + io::stderr(), + span, + ctrlc.as_ref(), + eval_block, + )?), + OutDest::File(file) => Some(tee_pipe_on_thread( + stderr, + file.clone(), + span, + ctrlc.as_ref(), + eval_block, + )?), + } + } else { + None + }; + + if let Some(stdout) = child.stdout.take() { + match stack.stdout() { + OutDest::Pipe | OutDest::Capture => { + child.stdout = Some(stdout); + Ok(()) + } + OutDest::Null => { + copy_pipe(stdout, io::sink(), span, ctrlc.as_deref()) + } + OutDest::Inherit => { + copy_pipe(stdout, io::stdout(), span, ctrlc.as_deref()) + } + OutDest::File(file) => { + copy_pipe(stdout, file.as_ref(), span, ctrlc.as_deref()) + } + }?; + } + + stderr_thread + } else { + let stderr_thread = if let Some(stderr) = child.stderr.take() { + match stack.stderr() { + OutDest::Pipe | OutDest::Capture => { + child.stderr = Some(stderr); + Ok(None) + } + OutDest::Null => { + copy_pipe_on_thread(stderr, io::sink(), span, ctrlc.as_ref()) + .map(Some) + } + OutDest::Inherit => { + copy_pipe_on_thread(stderr, io::stderr(), span, ctrlc.as_ref()) + .map(Some) + } + OutDest::File(file) => { + copy_pipe_on_thread(stderr, file.clone(), span, ctrlc.as_ref()) + .map(Some) + } + }? + } else { + None + }; + + if let Some(stdout) = child.stdout.take() { + match stack.stdout() { + OutDest::Pipe | OutDest::Capture => { + let tee = IoTee::new(stdout, span, eval_block)?; + child.stdout = Some(ChildPipe::Tee(Box::new(tee))); + Ok(()) + } + OutDest::Null => { + tee_pipe(stdout, io::sink(), span, ctrlc.as_deref(), eval_block) + } + OutDest::Inherit => tee_pipe( + stdout, + io::stdout(), + span, + ctrlc.as_deref(), + eval_block, + ), + OutDest::File(file) => tee_pipe( + stdout, + file.as_ref(), + span, + ctrlc.as_deref(), + eval_block, + ), + }?; + } + + stderr_thread }; - let result = eval_block_with_early_return( - &closure_engine_state, - &mut closure_stack, - closure_engine_state.get_block(block_id), - input_from_channel, - ); - // Make sure to drain any iterator produced to avoid unexpected behavior - result.and_then(|data| data.drain()) - }; - if use_stderr { - let stderr = stderr - .map(|stderr| { - let iter = tee(stderr.stream, with_stream).err_span(head)?; - Ok::<_, ShellError>(RawStream::new( - Box::new(iter.map(flatten_result)), - stderr.ctrlc, - stderr.span, - stderr.known_size, - )) - }) - .transpose()?; - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - }) - } else { - let stdout = stdout - .map(|stdout| { - let iter = tee(stdout.stream, with_stream).err_span(head)?; - Ok::<_, ShellError>(RawStream::new( - Box::new(iter.map(flatten_result)), - stdout.ctrlc, - stdout.span, - stdout.known_size, - )) - }) - .transpose()?; - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - }) + if child.stdout.is_some() || child.stderr.is_some() { + Ok(PipelineData::ByteStream( + ByteStream::child(*child, span), + metadata, + )) + } else { + if let Some(thread) = stderr_thread { + thread.join().unwrap_or_else(|_| Err(panic_error()))?; + } + child.wait()?; + Ok(PipelineData::Empty) + } } } - // --stderr is not allowed if the input is not an external stream - _ if use_stderr => Err(ShellError::UnsupportedInput { - msg: "--stderr can only be used on external streams".into(), - input: "the input to `tee` is not an external stream".into(), - msg_span: head, - input_span: input.span().unwrap_or(head), - }), - // Handle others with the plain iterator - _ => { - let teed = tee(input.into_iter(), move |rx| { - let input_from_channel = rx.into_pipeline_data_with_metadata( - head, - closure_engine_state.ctrlc.clone(), - metadata_clone, - ); - let result = eval_block_with_early_return( - &closure_engine_state, - &mut closure_stack, - closure_engine_state.get_block(block_id), - input_from_channel, - ); - // Make sure to drain any iterator produced to avoid unexpected behavior - result.and_then(|data| data.drain()) - }) - .err_span(head)? - .map(move |result| result.unwrap_or_else(|err| Value::error(err, closure_span))) - .into_pipeline_data_with_metadata( - head, - engine_state.ctrlc.clone(), - metadata, - ); - - Ok(teed) + } else { + if use_stderr { + return stderr_misuse(input.span().unwrap_or(head), head); } + + let span = input.span().unwrap_or(head); + let ctrlc = engine_state.ctrlc.clone(); + let metadata = input.metadata(); + let metadata_clone = metadata.clone(); + + Ok(tee(input.into_iter(), move |rx| { + let input = rx.into_pipeline_data_with_metadata(span, ctrlc, metadata_clone); + eval_block(input) + }) + .err_span(call.head)? + .map(move |result| result.unwrap_or_else(|err| Value::error(err, closure_span))) + .into_pipeline_data_with_metadata( + span, + engine_state.ctrlc.clone(), + metadata, + )) } } @@ -213,10 +291,6 @@ fn panic_error() -> ShellError { } } -fn flatten_result(result: Result, E>) -> Result { - result.unwrap_or_else(Err) -} - /// Copies the iterator to a channel on another thread. If an error is produced on that thread, /// it is embedded in the resulting iterator as an `Err` as soon as possible. When the iterator /// finishes, it waits for the other thread to finish, also handling any error produced at that @@ -233,7 +307,7 @@ where let mut thread = Some( thread::Builder::new() - .name("stderr consumer".into()) + .name("tee".into()) .spawn(move || with_cloned_stream(rx))?, ); @@ -273,6 +347,134 @@ where })) } +fn stderr_misuse(span: Span, head: Span) -> Result { + Err(ShellError::UnsupportedInput { + msg: "--stderr can only be used on external commands".into(), + input: "the input to `tee` is not an external commands".into(), + msg_span: head, + input_span: span, + }) +} + +struct IoTee { + reader: R, + sender: Option>>, + thread: Option>>, +} + +impl IoTee { + fn new( + reader: R, + span: Span, + eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, + ) -> Result { + let (sender, receiver) = mpsc::channel(); + + let thread = thread::Builder::new() + .name("tee".into()) + .spawn(move || eval_block(ByteStream::from_iter(receiver, span, None))) + .err_span(span)?; + + Ok(Self { + reader, + sender: Some(sender), + thread: Some(thread), + }) + } +} + +impl Read for IoTee { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if let Some(thread) = self.thread.take() { + if thread.is_finished() { + if let Err(err) = thread.join().unwrap_or_else(|_| Err(panic_error())) { + return Err(io::Error::new(io::ErrorKind::Other, err)); + } + } else { + self.thread = Some(thread) + } + } + let len = self.reader.read(buf)?; + if len == 0 { + self.sender = None; + if let Some(thread) = self.thread.take() { + if let Err(err) = thread.join().unwrap_or_else(|_| Err(panic_error())) { + return Err(io::Error::new(io::ErrorKind::Other, err)); + } + } + } else if let Some(sender) = self.sender.as_mut() { + if sender.send(buf[..len].to_vec()).is_err() { + self.sender = None; + } + } + Ok(len) + } +} + +fn tee_pipe( + pipe: ChildPipe, + mut dest: impl Write, + span: Span, + ctrlc: Option<&AtomicBool>, + eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, +) -> Result<(), ShellError> { + match pipe { + ChildPipe::Pipe(pipe) => { + let mut tee = IoTee::new(pipe, span, eval_block)?; + copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; + } + ChildPipe::Tee(tee) => { + let mut tee = IoTee::new(tee, span, eval_block)?; + copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; + } + } + Ok(()) +} + +fn tee_pipe_on_thread( + pipe: ChildPipe, + dest: impl Write + Send + 'static, + span: Span, + ctrlc: Option<&Arc>, + eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, +) -> Result>, ShellError> { + let ctrlc = ctrlc.cloned(); + thread::Builder::new() + .name("stderr tee".into()) + .spawn(move || tee_pipe(pipe, dest, span, ctrlc.as_deref(), eval_block)) + .map_err(|e| e.into_spanned(span).into()) +} + +fn copy_pipe( + pipe: ChildPipe, + mut dest: impl Write, + span: Span, + ctrlc: Option<&AtomicBool>, +) -> Result<(), ShellError> { + match pipe { + ChildPipe::Pipe(mut pipe) => { + copy_with_interrupt(&mut pipe, &mut dest, span, ctrlc)?; + } + ChildPipe::Tee(mut tee) => { + copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; + } + } + Ok(()) +} + +fn copy_pipe_on_thread( + pipe: ChildPipe, + dest: impl Write + Send + 'static, + span: Span, + ctrlc: Option<&Arc>, +) -> Result>, ShellError> { + let ctrlc = ctrlc.cloned(); + thread::Builder::new() + .name("stderr copier".into()) + .spawn(move || copy_pipe(pipe, dest, span, ctrlc.as_deref())) + .map_err(|e| e.into_spanned(span).into()) +} + #[test] fn tee_copies_values_to_other_thread_and_passes_them_through() { let (tx, rx) = mpsc::channel(); diff --git a/crates/nu-command/src/filters/update.rs b/crates/nu-command/src/filters/update.rs index 2cea7deead..0d914d2d8e 100644 --- a/crates/nu-command/src/filters/update.rs +++ b/crates/nu-command/src/filters/update.rs @@ -117,7 +117,7 @@ fn update( if let Value::Closure { val, .. } = replacement { match (cell_path.members.first(), &mut value) { (Some(PathMember::String { .. }), Value::List { vals, .. }) => { - let mut closure = ClosureEval::new(engine_state, stack, val); + let mut closure = ClosureEval::new(engine_state, stack, *val); for val in vals { update_value_by_closure( val, @@ -131,7 +131,7 @@ fn update( (first, _) => { update_single_value_by_closure( &mut value, - ClosureEvalOnce::new(engine_state, stack, val), + ClosureEvalOnce::new(engine_state, stack, *val), head, &cell_path.members, matches!(first, Some(PathMember::Int { .. })), @@ -175,7 +175,7 @@ fn update( if let Value::Closure { val, .. } = replacement { update_single_value_by_closure( value, - ClosureEvalOnce::new(engine_state, stack, val), + ClosureEvalOnce::new(engine_state, stack, *val), head, path, true, @@ -189,7 +189,7 @@ fn update( .chain(stream) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) } else if let Value::Closure { val, .. } = replacement { - let mut closure = ClosureEval::new(engine_state, stack, val); + let mut closure = ClosureEval::new(engine_state, stack, *val); let stream = stream.map(move |mut value| { let err = update_value_by_closure( &mut value, @@ -225,8 +225,8 @@ fn update( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ExternalStream { .. } => Err(ShellError::IncompatiblePathAccess { - type_name: "external stream".to_string(), + PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { + type_name: "byte stream".to_string(), span: head, }), } @@ -250,7 +250,7 @@ fn update_value_by_closure( let new_value = closure .add_arg(arg.clone()) .run_with_input(value_at_path.into_pipeline_data())? - .into_value(span); + .into_value(span)?; value.update_data_at_cell_path(cell_path, new_value) } @@ -273,7 +273,7 @@ fn update_single_value_by_closure( let new_value = closure .add_arg(arg.clone()) .run_with_input(value_at_path.into_pipeline_data())? - .into_value(span); + .into_value(span)?; value.update_data_at_cell_path(cell_path, new_value) } diff --git a/crates/nu-command/src/filters/upsert.rs b/crates/nu-command/src/filters/upsert.rs index b7b4a782f2..4313addd89 100644 --- a/crates/nu-command/src/filters/upsert.rs +++ b/crates/nu-command/src/filters/upsert.rs @@ -163,7 +163,7 @@ fn upsert( if let Value::Closure { val, .. } = replacement { match (cell_path.members.first(), &mut value) { (Some(PathMember::String { .. }), Value::List { vals, .. }) => { - let mut closure = ClosureEval::new(engine_state, stack, val); + let mut closure = ClosureEval::new(engine_state, stack, *val); for val in vals { upsert_value_by_closure( val, @@ -177,7 +177,7 @@ fn upsert( (first, _) => { upsert_single_value_by_closure( &mut value, - ClosureEvalOnce::new(engine_state, stack, val), + ClosureEvalOnce::new(engine_state, stack, *val), head, &cell_path.members, matches!(first, Some(PathMember::Int { .. })), @@ -216,9 +216,9 @@ fn upsert( let value = if path.is_empty() { let value = stream.next().unwrap_or(Value::nothing(head)); if let Value::Closure { val, .. } = replacement { - ClosureEvalOnce::new(engine_state, stack, val) + ClosureEvalOnce::new(engine_state, stack, *val) .run_with_value(value)? - .into_value(head) + .into_value(head)? } else { replacement } @@ -226,7 +226,7 @@ fn upsert( if let Value::Closure { val, .. } = replacement { upsert_single_value_by_closure( &mut value, - ClosureEvalOnce::new(engine_state, stack, val), + ClosureEvalOnce::new(engine_state, stack, *val), head, path, true, @@ -249,7 +249,7 @@ fn upsert( .chain(stream) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) } else if let Value::Closure { val, .. } = replacement { - let mut closure = ClosureEval::new(engine_state, stack, val); + let mut closure = ClosureEval::new(engine_state, stack, *val); let stream = stream.map(move |mut value| { let err = upsert_value_by_closure( &mut value, @@ -285,8 +285,8 @@ fn upsert( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ExternalStream { .. } => Err(ShellError::IncompatiblePathAccess { - type_name: "external stream".to_string(), + PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { + type_name: "byte stream".to_string(), span: head, }), } @@ -311,7 +311,11 @@ fn upsert_value_by_closure( .map(IntoPipelineData::into_pipeline_data) .unwrap_or(PipelineData::Empty); - let new_value = closure.add_arg(arg).run_with_input(input)?.into_value(span); + let new_value = closure + .add_arg(arg) + .run_with_input(input)? + .into_value(span)?; + value.upsert_data_at_cell_path(cell_path, new_value) } @@ -334,7 +338,11 @@ fn upsert_single_value_by_closure( .map(IntoPipelineData::into_pipeline_data) .unwrap_or(PipelineData::Empty); - let new_value = closure.add_arg(arg).run_with_input(input)?.into_value(span); + let new_value = closure + .add_arg(arg) + .run_with_input(input)? + .into_value(span)?; + value.upsert_data_at_cell_path(cell_path, new_value) } diff --git a/crates/nu-command/src/filters/utils.rs b/crates/nu-command/src/filters/utils.rs index 0ef7d916b7..8d9b1300f6 100644 --- a/crates/nu-command/src/filters/utils.rs +++ b/crates/nu-command/src/filters/utils.rs @@ -36,7 +36,7 @@ pub fn boolean_fold( break; } - let pred = closure.run_with_value(value)?.into_value(head).is_true(); + let pred = closure.run_with_value(value)?.into_value(head)?.is_true(); if pred == accumulator { return Ok(Value::bool(accumulator, head).into_pipeline_data()); diff --git a/crates/nu-command/src/filters/values.rs b/crates/nu-command/src/filters/values.rs index 15aad30f69..9d42f50e31 100644 --- a/crates/nu-command/src/filters/values.rs +++ b/crates/nu-command/src/filters/values.rs @@ -180,13 +180,11 @@ fn values( Err(err) => Err(err), } } - PipelineData::ExternalStream { .. } => Err(ShellError::OnlySupportsThisInputType { + PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "record or table".into(), - wrong_type: "raw data".into(), + wrong_type: "byte stream".into(), dst_span: head, - src_span: input - .span() - .expect("PipelineData::ExternalStream had no span"), + src_span: stream.span(), }), } } diff --git a/crates/nu-command/src/filters/where_.rs b/crates/nu-command/src/filters/where_.rs index 7507a7ede1..fe73de354f 100644 --- a/crates/nu-command/src/filters/where_.rs +++ b/crates/nu-command/src/filters/where_.rs @@ -57,9 +57,14 @@ not supported."# let metadata = input.metadata(); Ok(input .into_iter_strict(head)? - .filter_map(move |value| match closure.run_with_value(value.clone()) { - Ok(data) => data.into_value(head).is_true().then_some(value), - Err(err) => Some(Value::error(err, head)), + .filter_map(move |value| { + match closure + .run_with_value(value.clone()) + .and_then(|data| data.into_value(head)) + { + Ok(cond) => cond.is_true().then_some(value), + Err(err) => Some(Value::error(err, head)), + } }) .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) } diff --git a/crates/nu-command/src/filters/wrap.rs b/crates/nu-command/src/filters/wrap.rs index 24ce8e6821..52a0fb22c3 100644 --- a/crates/nu-command/src/filters/wrap.rs +++ b/crates/nu-command/src/filters/wrap.rs @@ -43,8 +43,8 @@ impl Command for Wrap { .into_iter() .map(move |x| Value::record(record! { name.clone() => x }, span)) .into_pipeline_data_with_metadata(span, engine_state.ctrlc.clone(), metadata)), - PipelineData::ExternalStream { .. } => Ok(Value::record( - record! { name => input.into_value(span) }, + PipelineData::ByteStream(stream, ..) => Ok(Value::record( + record! { name => stream.into_value()? }, span, ) .into_pipeline_data_with_metadata(metadata)), diff --git a/crates/nu-command/src/filters/zip.rs b/crates/nu-command/src/filters/zip.rs index f4ee739f50..9d81451ed4 100644 --- a/crates/nu-command/src/filters/zip.rs +++ b/crates/nu-command/src/filters/zip.rs @@ -103,7 +103,7 @@ impl Command for Zip { let metadata = input.metadata(); let other = if let Value::Closure { val, .. } = other { // If a closure was provided, evaluate it and consume its stream output - ClosureEvalOnce::new(engine_state, stack, val).run_with_input(PipelineData::Empty)? + ClosureEvalOnce::new(engine_state, stack, *val).run_with_input(PipelineData::Empty)? } else { other.into_pipeline_data() }; diff --git a/crates/nu-command/src/formats/from/json.rs b/crates/nu-command/src/formats/from/json.rs index 44f127152f..ea449711c1 100644 --- a/crates/nu-command/src/formats/from/json.rs +++ b/crates/nu-command/src/formats/from/json.rs @@ -59,7 +59,7 @@ impl Command for FromJson { let (string_input, span, metadata) = input.collect_string_strict(span)?; if string_input.is_empty() { - return Ok(PipelineData::new_with_metadata(metadata, span)); + return Ok(Value::nothing(span).into_pipeline_data()); } let strict = call.has_flag(engine_state, stack, "strict")?; diff --git a/crates/nu-command/src/formats/from/msgpack.rs b/crates/nu-command/src/formats/from/msgpack.rs index 75f2be2056..4d8ea5e320 100644 --- a/crates/nu-command/src/formats/from/msgpack.rs +++ b/crates/nu-command/src/formats/from/msgpack.rs @@ -2,9 +2,8 @@ // implementation here is unique. use std::{ - collections::VecDeque, error::Error, - io::{self, Cursor, ErrorKind, Write}, + io::{self, Cursor, ErrorKind}, string::FromUtf8Error, sync::{atomic::AtomicBool, Arc}, }; @@ -12,7 +11,6 @@ use std::{ use byteorder::{BigEndian, ReadBytesExt}; use chrono::{TimeZone, Utc}; use nu_engine::command_prelude::*; -use nu_protocol::RawStream; use rmp::decode::{self as mp, ValueReadError}; /// Max recursion depth @@ -121,12 +119,20 @@ MessagePack: https://msgpack.org/ read_msgpack(Cursor::new(bytes), opts) } // Deserialize from a raw stream directly without having to collect it - PipelineData::ExternalStream { - stdout: Some(raw_stream), - .. - } => read_msgpack(ReadRawStream::new(raw_stream), opts), + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + if let Some(reader) = stream.reader() { + read_msgpack(reader, opts) + } else { + Err(ShellError::PipelineMismatch { + exp_input_type: "binary or byte stream".into(), + dst_span: call.head, + src_span: span, + }) + } + } input => Err(ShellError::PipelineMismatch { - exp_input_type: "binary".into(), + exp_input_type: "binary or byte stream".into(), dst_span: call.head, src_span: input.span().unwrap_or(call.head), }), @@ -483,57 +489,6 @@ where .map_err(|err| ReadError::Io(err, span)) } -/// Adapter to read MessagePack from a `RawStream` -/// -/// TODO: contribute this back to `RawStream` in general, with more polish, if it works -pub(crate) struct ReadRawStream { - pub stream: RawStream, - // Use a `VecDeque` for read efficiency - pub leftover: VecDeque, -} - -impl ReadRawStream { - pub(crate) fn new(mut stream: RawStream) -> ReadRawStream { - ReadRawStream { - leftover: std::mem::take(&mut stream.leftover).into(), - stream, - } - } -} - -impl io::Read for ReadRawStream { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if buf.is_empty() { - Ok(0) - } else if !self.leftover.is_empty() { - // Take as many leftover bytes as possible - self.leftover.read(buf) - } else { - // Try to get data from the RawStream. We have to be careful not to break on a zero-len - // buffer though, since that would mean EOF - loop { - if let Some(result) = self.stream.stream.next() { - let bytes = result.map_err(|err| io::Error::new(ErrorKind::Other, err))?; - if !bytes.is_empty() { - let min_len = bytes.len().min(buf.len()); - let (source, leftover_bytes) = bytes.split_at(min_len); - buf[0..min_len].copy_from_slice(source); - // Keep whatever bytes we couldn't use in the leftover vec - self.leftover.write_all(leftover_bytes)?; - return Ok(min_len); - } else { - // Zero-length buf, continue - continue; - } - } else { - // End of input - return Ok(0); - } - } - } - } -} - /// Return an error if this is not the end of file. /// /// This can help detect if parsing succeeded incorrectly, perhaps due to corruption. diff --git a/crates/nu-command/src/formats/from/msgpackz.rs b/crates/nu-command/src/formats/from/msgpackz.rs index 3200d5d876..7960f3f97a 100644 --- a/crates/nu-command/src/formats/from/msgpackz.rs +++ b/crates/nu-command/src/formats/from/msgpackz.rs @@ -2,7 +2,7 @@ use std::io::Cursor; use nu_engine::command_prelude::*; -use super::msgpack::{read_msgpack, Opts, ReadRawStream}; +use super::msgpack::{read_msgpack, Opts}; const BUFFER_SIZE: usize = 65536; @@ -50,15 +50,21 @@ impl Command for FromMsgpackz { read_msgpack(reader, opts) } // Deserialize from a raw stream directly without having to collect it - PipelineData::ExternalStream { - stdout: Some(raw_stream), - .. - } => { - let reader = brotli::Decompressor::new(ReadRawStream::new(raw_stream), BUFFER_SIZE); - read_msgpack(reader, opts) + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + if let Some(reader) = stream.reader() { + let reader = brotli::Decompressor::new(reader, BUFFER_SIZE); + read_msgpack(reader, opts) + } else { + Err(ShellError::PipelineMismatch { + exp_input_type: "binary or byte stream".into(), + dst_span: call.head, + src_span: span, + }) + } } _ => Err(ShellError::PipelineMismatch { - exp_input_type: "binary".into(), + exp_input_type: "binary or byte stream".into(), dst_span: call.head, src_span: span, }), diff --git a/crates/nu-command/src/formats/from/ods.rs b/crates/nu-command/src/formats/from/ods.rs index fff9e98be6..ff7a76c0ca 100644 --- a/crates/nu-command/src/formats/from/ods.rs +++ b/crates/nu-command/src/formats/from/ods.rs @@ -81,28 +81,32 @@ fn convert_columns(columns: &[Value]) -> Result, ShellError> { } fn collect_binary(input: PipelineData, span: Span) -> Result, ShellError> { - let mut bytes = vec![]; - let mut values = input.into_iter(); + if let PipelineData::ByteStream(stream, ..) = input { + stream.into_bytes() + } else { + let mut bytes = vec![]; + let mut values = input.into_iter(); - loop { - match values.next() { - Some(Value::Binary { val: b, .. }) => { - bytes.extend_from_slice(&b); + loop { + match values.next() { + Some(Value::Binary { val: b, .. }) => { + bytes.extend_from_slice(&b); + } + Some(Value::Error { error, .. }) => return Err(*error), + Some(x) => { + return Err(ShellError::UnsupportedInput { + msg: "Expected binary from pipeline".to_string(), + input: "value originates from here".into(), + msg_span: span, + input_span: x.span(), + }) + } + None => break, } - Some(Value::Error { error, .. }) => return Err(*error), - Some(x) => { - return Err(ShellError::UnsupportedInput { - msg: "Expected binary from pipeline".to_string(), - input: "value originates from here".into(), - msg_span: span, - input_span: x.span(), - }) - } - None => break, } - } - Ok(bytes) + Ok(bytes) + } } fn from_ods( diff --git a/crates/nu-command/src/formats/from/xlsx.rs b/crates/nu-command/src/formats/from/xlsx.rs index b54cffe3aa..21e2567b45 100644 --- a/crates/nu-command/src/formats/from/xlsx.rs +++ b/crates/nu-command/src/formats/from/xlsx.rs @@ -82,27 +82,31 @@ fn convert_columns(columns: &[Value]) -> Result, ShellError> { } fn collect_binary(input: PipelineData, span: Span) -> Result, ShellError> { - let mut bytes = vec![]; - let mut values = input.into_iter(); + if let PipelineData::ByteStream(stream, ..) = input { + stream.into_bytes() + } else { + let mut bytes = vec![]; + let mut values = input.into_iter(); - loop { - match values.next() { - Some(Value::Binary { val: b, .. }) => { - bytes.extend_from_slice(&b); + loop { + match values.next() { + Some(Value::Binary { val: b, .. }) => { + bytes.extend_from_slice(&b); + } + Some(x) => { + return Err(ShellError::UnsupportedInput { + msg: "Expected binary from pipeline".to_string(), + input: "value originates from here".into(), + msg_span: span, + input_span: x.span(), + }) + } + None => break, } - Some(x) => { - return Err(ShellError::UnsupportedInput { - msg: "Expected binary from pipeline".to_string(), - input: "value originates from here".into(), - msg_span: span, - input_span: x.span(), - }) - } - None => break, } - } - Ok(bytes) + Ok(bytes) + } } fn from_xlsx( diff --git a/crates/nu-command/src/formats/to/delimited.rs b/crates/nu-command/src/formats/to/delimited.rs index a10f611f60..490983d67b 100644 --- a/crates/nu-command/src/formats/to/delimited.rs +++ b/crates/nu-command/src/formats/to/delimited.rs @@ -150,7 +150,7 @@ pub fn to_delimited_data( span: Span, config: &Config, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; let output = match from_value_to_delimited_string(&value, sep, config, span) { Ok(mut x) => { if noheaders { diff --git a/crates/nu-command/src/formats/to/json.rs b/crates/nu-command/src/formats/to/json.rs index 1401b69a34..2a1c1e2228 100644 --- a/crates/nu-command/src/formats/to/json.rs +++ b/crates/nu-command/src/formats/to/json.rs @@ -46,7 +46,7 @@ impl Command for ToJson { let span = call.head; // allow ranges to expand and turn into array let input = input.try_expand_range()?; - let value = input.into_value(span); + let value = input.into_value(span)?; let json_value = value_to_json_value(&value)?; let json_result = if raw { diff --git a/crates/nu-command/src/formats/to/msgpack.rs b/crates/nu-command/src/formats/to/msgpack.rs index a4575f37e4..bfeb428e3e 100644 --- a/crates/nu-command/src/formats/to/msgpack.rs +++ b/crates/nu-command/src/formats/to/msgpack.rs @@ -75,7 +75,7 @@ MessagePack: https://msgpack.org/ input: PipelineData, ) -> Result { let value_span = input.span().unwrap_or(call.head); - let value = input.into_value(value_span); + let value = input.into_value(value_span)?; let mut out = vec![]; write_value(&mut out, &value, 0)?; diff --git a/crates/nu-command/src/formats/to/msgpackz.rs b/crates/nu-command/src/formats/to/msgpackz.rs index a07e1206c1..9168d05018 100644 --- a/crates/nu-command/src/formats/to/msgpackz.rs +++ b/crates/nu-command/src/formats/to/msgpackz.rs @@ -70,7 +70,7 @@ impl Command for ToMsgpackz { .transpose()?; let value_span = input.span().unwrap_or(call.head); - let value = input.into_value(value_span); + let value = input.into_value(value_span)?; let mut out_buf = vec![]; let mut out = brotli::CompressorWriter::new( &mut out_buf, diff --git a/crates/nu-command/src/formats/to/nuon.rs b/crates/nu-command/src/formats/to/nuon.rs index e747ac58f6..f40b7b5c1d 100644 --- a/crates/nu-command/src/formats/to/nuon.rs +++ b/crates/nu-command/src/formats/to/nuon.rs @@ -53,7 +53,7 @@ impl Command for ToNuon { }; let span = call.head; - let value = input.into_value(span); + let value = input.into_value(span)?; match nuon::to_nuon(&value, style, Some(span)) { Ok(serde_nuon_string) => { diff --git a/crates/nu-command/src/formats/to/text.rs b/crates/nu-command/src/formats/to/text.rs index d6935b0b28..9d771a7097 100644 --- a/crates/nu-command/src/formats/to/text.rs +++ b/crates/nu-command/src/formats/to/text.rs @@ -1,6 +1,12 @@ use chrono_humanize::HumanTime; use nu_engine::command_prelude::*; -use nu_protocol::{format_duration, format_filesize_from_conf, Config, RawStream, ValueIterator}; +use nu_protocol::{format_duration, format_filesize_from_conf, ByteStream, Config}; + +const LINE_ENDING: &str = if cfg!(target_os = "windows") { + "\r\n" +} else { + "\n" +}; #[derive(Clone)] pub struct ToText; @@ -28,39 +34,28 @@ impl Command for ToText { input: PipelineData, ) -> Result { let span = call.head; - let config = engine_state.get_config(); - - let line_ending = if cfg!(target_os = "windows") { - "\r\n" - } else { - "\n" - }; let input = input.try_expand_range()?; - if let PipelineData::ListStream(stream, _) = input { - Ok(PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(ListStreamIterator { - stream: stream.into_inner(), - separator: line_ending.into(), - config: config.clone(), - }), - engine_state.ctrlc.clone(), - span, - None, - )), - stderr: None, - exit_code: None, - span, - metadata: None, - trim_end_newline: false, - }) - } else { - // FIXME: don't collect! stream the output wherever possible! - // Even if the data is collected when it arrives at `to text`, we should be able to stream it out - let collected_input = local_into_string(input.into_value(span), line_ending, config); - - Ok(Value::string(collected_input, span).into_pipeline_data()) + match input { + PipelineData::Empty => Ok(Value::string(String::new(), span).into_pipeline_data()), + PipelineData::Value(value, ..) => { + let str = local_into_string(value, LINE_ENDING, engine_state.get_config()); + Ok(Value::string(str, span).into_pipeline_data()) + } + PipelineData::ListStream(stream, meta) => { + let span = stream.span(); + let config = engine_state.get_config().clone(); + let iter = stream.into_inner().map(move |value| { + let mut str = local_into_string(value, LINE_ENDING, &config); + str.push_str(LINE_ENDING); + str + }); + Ok(PipelineData::ByteStream( + ByteStream::from_iter(iter, span, engine_state.ctrlc.clone()), + meta, + )) + } + PipelineData::ByteStream(stream, meta) => Ok(PipelineData::ByteStream(stream, meta)), } } @@ -85,26 +80,6 @@ impl Command for ToText { } } -struct ListStreamIterator { - stream: ValueIterator, - separator: String, - config: Config, -} - -impl Iterator for ListStreamIterator { - type Item = Result, ShellError>; - - fn next(&mut self) -> Option { - if let Some(item) = self.stream.next() { - let mut string = local_into_string(item, &self.separator, &self.config); - string.push_str(&self.separator); - Some(Ok(string.as_bytes().to_vec())) - } else { - None - } - } -} - fn local_into_string(value: Value, separator: &str, config: &Config) -> String { let span = value.span(); match value { diff --git a/crates/nu-command/src/formats/to/toml.rs b/crates/nu-command/src/formats/to/toml.rs index ffd6234e09..82b8082158 100644 --- a/crates/nu-command/src/formats/to/toml.rs +++ b/crates/nu-command/src/formats/to/toml.rs @@ -141,7 +141,7 @@ fn to_toml( input: PipelineData, span: Span, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; let toml_value = value_to_toml_value(engine_state, &value, span)?; match toml_value { diff --git a/crates/nu-command/src/formats/to/xml.rs b/crates/nu-command/src/formats/to/xml.rs index c08001b9b1..7ba608afe7 100644 --- a/crates/nu-command/src/formats/to/xml.rs +++ b/crates/nu-command/src/formats/to/xml.rs @@ -132,7 +132,7 @@ impl Job { } fn run(mut self, input: PipelineData, head: Span) -> Result { - let value = input.into_value(head); + let value = input.into_value(head)?; self.write_xml_entry(value, true).and_then(|_| { let b = self.writer.into_inner().into_inner(); diff --git a/crates/nu-command/src/formats/to/yaml.rs b/crates/nu-command/src/formats/to/yaml.rs index a9e8776cd3..85d3523966 100644 --- a/crates/nu-command/src/formats/to/yaml.rs +++ b/crates/nu-command/src/formats/to/yaml.rs @@ -95,7 +95,7 @@ pub fn value_to_yaml_value(v: &Value) -> Result { } fn to_yaml(input: PipelineData, head: Span) -> Result { - let value = input.into_value(head); + let value = input.into_value(head)?; let yaml_value = value_to_yaml_value(&value)?; match serde_yaml::to_string(&yaml_value) { diff --git a/crates/nu-command/src/generators/generate.rs b/crates/nu-command/src/generators/generate.rs index 905c19eb2f..35cb00b060 100644 --- a/crates/nu-command/src/generators/generate.rs +++ b/crates/nu-command/src/generators/generate.rs @@ -158,14 +158,16 @@ used as the next argument to the closure, otherwise generation stops. } Ok(other) => { - let val = other.into_value(head); - let error = ShellError::GenericError { - error: "Invalid block return".into(), - msg: format!("Expected record, found {}", val.get_type()), - span: Some(val.span()), - help: None, - inner: vec![], - }; + let error = other + .into_value(head) + .map(|val| ShellError::GenericError { + error: "Invalid block return".into(), + msg: format!("Expected record, found {}", val.get_type()), + span: Some(val.span()), + help: None, + inner: vec![], + }) + .unwrap_or_else(|err| err); (Some(Value::error(error, head)), None) } diff --git a/crates/nu-command/src/hash/generic_digest.rs b/crates/nu-command/src/hash/generic_digest.rs index 476915f07d..ab15ccae7a 100644 --- a/crates/nu-command/src/hash/generic_digest.rs +++ b/crates/nu-command/src/hash/generic_digest.rs @@ -1,7 +1,6 @@ use nu_cmd_base::input_handler::{operate, CmdArgument}; use nu_engine::command_prelude::*; - -use std::marker::PhantomData; +use std::{io::Write, marker::PhantomData}; pub trait HashDigest: digest::Digest + Clone { fn name() -> &'static str; @@ -38,7 +37,7 @@ impl CmdArgument for Arguments { impl Command for GenericDigest where - D: HashDigest + Send + Sync + 'static, + D: HashDigest + Write + Send + Sync + 'static, digest::Output: core::fmt::LowerHex, { fn name(&self) -> &str { @@ -81,54 +80,23 @@ where call: &Call, input: PipelineData, ) -> Result { + let head = call.head; let binary = call.has_flag(engine_state, stack, "binary")?; let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - let args = Arguments { binary, cell_paths }; let mut hasher = D::new(); - match input { - PipelineData::ExternalStream { - stdout: Some(stream), - span, - .. - } => { - for item in stream { - match item { - // String and binary data are valid byte patterns - Ok(Value::String { val, .. }) => hasher.update(val.as_bytes()), - Ok(Value::Binary { val, .. }) => hasher.update(val), - // If any Error value is output, echo it back - Ok(v @ Value::Error { .. }) => return Ok(v.into_pipeline_data()), - // Unsupported data - Ok(other) => { - return Ok(Value::error( - ShellError::OnlySupportsThisInputType { - exp_input_type: "string and binary".into(), - wrong_type: other.get_type().to_string(), - dst_span: span, - src_span: other.span(), - }, - span, - ) - .into_pipeline_data()); - } - Err(err) => return Err(err), - }; - } - let digest = hasher.finalize(); - if args.binary { - Ok(Value::binary(digest.to_vec(), span).into_pipeline_data()) - } else { - Ok(Value::string(format!("{digest:x}"), span).into_pipeline_data()) - } + + if let PipelineData::ByteStream(stream, ..) = input { + stream.write_to(&mut hasher)?; + let digest = hasher.finalize(); + if binary { + Ok(Value::binary(digest.to_vec(), head).into_pipeline_data()) + } else { + Ok(Value::string(format!("{digest:x}"), head).into_pipeline_data()) } - _ => operate( - action::, - args, - input, - call.head, - engine_state.ctrlc.clone(), - ), + } else { + let args = Arguments { binary, cell_paths }; + operate(action::, args, input, head, engine_state.ctrlc.clone()) } } } diff --git a/crates/nu-command/src/help/help_.rs b/crates/nu-command/src/help/help_.rs index cb123b6e82..4b8e0b9b22 100644 --- a/crates/nu-command/src/help/help_.rs +++ b/crates/nu-command/src/help/help_.rs @@ -2,7 +2,6 @@ use crate::help::{help_aliases, help_commands, help_modules}; use fancy_regex::Regex; use nu_ansi_term::Style; use nu_engine::command_prelude::*; -use nu_protocol::span; use nu_utils::IgnoreCaseExt; #[derive(Clone)] @@ -97,9 +96,8 @@ You can also learn more at https://www.nushell.sh/book/"#; span: _, }) = result { - let rest_spans: Vec = rest.iter().map(|arg| arg.span).collect(); Err(ShellError::NotFound { - span: span(&rest_spans), + span: Span::merge_many(rest.iter().map(|s| s.span)), }) } else { result diff --git a/crates/nu-command/src/help/help_aliases.rs b/crates/nu-command/src/help/help_aliases.rs index 2cc7c7f073..da03fa6398 100644 --- a/crates/nu-command/src/help/help_aliases.rs +++ b/crates/nu-command/src/help/help_aliases.rs @@ -1,7 +1,6 @@ use crate::help::highlight_search_in_table; use nu_color_config::StyleComputer; use nu_engine::{command_prelude::*, scope::ScopeData}; -use nu_protocol::span; #[derive(Clone)] pub struct HelpAliases; @@ -110,13 +109,13 @@ pub fn help_aliases( let Some(alias) = engine_state.find_decl(name.as_bytes(), &[]) else { return Err(ShellError::AliasNotFound { - span: span(&rest.iter().map(|r| r.span).collect::>()), + span: Span::merge_many(rest.iter().map(|s| s.span)), }); }; let Some(alias) = engine_state.get_decl(alias).as_alias() else { return Err(ShellError::AliasNotFound { - span: span(&rest.iter().map(|r| r.span).collect::>()), + span: Span::merge_many(rest.iter().map(|s| s.span)), }); }; diff --git a/crates/nu-command/src/help/help_commands.rs b/crates/nu-command/src/help/help_commands.rs index bc508b249b..bc0fd92d92 100644 --- a/crates/nu-command/src/help/help_commands.rs +++ b/crates/nu-command/src/help/help_commands.rs @@ -1,7 +1,6 @@ use crate::help::highlight_search_in_table; use nu_color_config::StyleComputer; use nu_engine::{command_prelude::*, get_full_help}; -use nu_protocol::span; #[derive(Clone)] pub struct HelpCommands; @@ -104,7 +103,7 @@ pub fn help_commands( ) } else { Err(ShellError::CommandNotFound { - span: span(&[rest[0].span, rest[rest.len() - 1].span]), + span: Span::merge_many(rest.iter().map(|s| s.span)), }) } } diff --git a/crates/nu-command/src/help/help_externs.rs b/crates/nu-command/src/help/help_externs.rs index 624a8d8060..22fb4a303c 100644 --- a/crates/nu-command/src/help/help_externs.rs +++ b/crates/nu-command/src/help/help_externs.rs @@ -1,7 +1,6 @@ use crate::help::highlight_search_in_table; use nu_color_config::StyleComputer; use nu_engine::{command_prelude::*, get_full_help, scope::ScopeData}; -use nu_protocol::span; #[derive(Clone)] pub struct HelpExterns; @@ -124,7 +123,7 @@ pub fn help_externs( ) } else { Err(ShellError::CommandNotFound { - span: span(&[rest[0].span, rest[rest.len() - 1].span]), + span: Span::merge_many(rest.iter().map(|s| s.span)), }) } } diff --git a/crates/nu-command/src/help/help_modules.rs b/crates/nu-command/src/help/help_modules.rs index f2ddf55f1d..690968251b 100644 --- a/crates/nu-command/src/help/help_modules.rs +++ b/crates/nu-command/src/help/help_modules.rs @@ -1,7 +1,7 @@ use crate::help::highlight_search_in_table; use nu_color_config::StyleComputer; use nu_engine::{command_prelude::*, scope::ScopeData}; -use nu_protocol::{span, DeclId}; +use nu_protocol::DeclId; #[derive(Clone)] pub struct HelpModules; @@ -117,7 +117,7 @@ pub fn help_modules( let Some(module_id) = engine_state.find_module(name.as_bytes(), &[]) else { return Err(ShellError::ModuleNotFoundAtRuntime { mod_name: name, - span: span(&rest.iter().map(|r| r.span).collect::>()), + span: Span::merge_many(rest.iter().map(|s| s.span)), }); }; diff --git a/crates/nu-command/src/misc/tutor.rs b/crates/nu-command/src/misc/tutor.rs index 8eeec6f393..6b1c43534b 100644 --- a/crates/nu-command/src/misc/tutor.rs +++ b/crates/nu-command/src/misc/tutor.rs @@ -409,15 +409,15 @@ fn display(help: &str, engine_state: &EngineState, stack: &mut Stack, span: Span //TODO: support no-color mode if let Some(highlighter) = engine_state.find_decl(b"nu-highlight", &[]) { let decl = engine_state.get_decl(highlighter); - - if let Ok(output) = decl.run( + let result = decl.run( engine_state, stack, &Call::new(span), Value::string(item, Span::unknown()).into_pipeline_data(), - ) { - let result = output.into_value(Span::unknown()); - match result.coerce_into_string() { + ); + + if let Ok(value) = result.and_then(|data| data.into_value(Span::unknown())) { + match value.coerce_into_string() { Ok(s) => { build.push_str(&s); } diff --git a/crates/nu-command/src/network/http/client.rs b/crates/nu-command/src/network/http/client.rs index 976e2d0786..e4fba47664 100644 --- a/crates/nu-command/src/network/http/client.rs +++ b/crates/nu-command/src/network/http/client.rs @@ -5,10 +5,9 @@ use base64::{ Engine, }; use nu_engine::command_prelude::*; -use nu_protocol::{BufferedReader, RawStream}; +use nu_protocol::ByteStream; use std::{ collections::HashMap, - io::BufReader, path::PathBuf, str::FromStr, sync::{ @@ -119,21 +118,11 @@ pub fn response_to_buffer( }; let reader = response.into_reader(); - let buffered_input = BufReader::new(reader); - PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(BufferedReader::new(buffered_input)), - engine_state.ctrlc.clone(), - span, - buffer_size, - )), - stderr: None, - exit_code: None, - span, - metadata: None, - trim_end_newline: false, - } + PipelineData::ByteStream( + ByteStream::read(reader, span, engine_state.ctrlc.clone()).with_known_size(buffer_size), + None, + ) } pub fn request_add_authorization_header( @@ -529,25 +518,25 @@ fn request_handle_response_content( if flags.full { let response_status = resp.status(); - let request_headers_value = match headers_to_nu(&extract_request_headers(&request), span) { - Ok(headers) => headers.into_value(span), - Err(_) => Value::nothing(span), - }; + let request_headers_value = headers_to_nu(&extract_request_headers(&request), span) + .and_then(|data| data.into_value(span)) + .unwrap_or(Value::nothing(span)); - let response_headers_value = match headers_to_nu(&extract_response_headers(&resp), span) { - Ok(headers) => headers.into_value(span), - Err(_) => Value::nothing(span), - }; + let response_headers_value = headers_to_nu(&extract_response_headers(&resp), span) + .and_then(|data| data.into_value(span)) + .unwrap_or(Value::nothing(span)); let headers = record! { "request" => request_headers_value, "response" => response_headers_value, }; + let body = consume_response_body(resp)?.into_value(span)?; + let full_response = Value::record( record! { "headers" => Value::record(headers, span), - "body" => consume_response_body(resp)?.into_value(span), + "body" => body, "status" => Value::int(response_status as i64, span), }, span, diff --git a/crates/nu-command/src/network/url/parse.rs b/crates/nu-command/src/network/url/parse.rs index 8a80553eca..e71c8d472a 100644 --- a/crates/nu-command/src/network/url/parse.rs +++ b/crates/nu-command/src/network/url/parse.rs @@ -42,7 +42,7 @@ impl Command for SubCommand { call: &Call, input: PipelineData, ) -> Result { - parse(input.into_value(call.head), call.head, engine_state) + parse(input.into_value(call.head)?, call.head, engine_state) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/path/join.rs b/crates/nu-command/src/path/join.rs index eb820606af..19d65b4c46 100644 --- a/crates/nu-command/src/path/join.rs +++ b/crates/nu-command/src/path/join.rs @@ -171,8 +171,8 @@ fn run(call: &Call, args: &Arguments, input: PipelineData) -> Result Ok(PipelineData::Value(handle_value(val, args, head), md)), - PipelineData::ListStream(..) => Ok(PipelineData::Value( - handle_value(input.into_value(head), args, head), + PipelineData::ListStream(stream, ..) => Ok(PipelineData::Value( + handle_value(stream.into_value(), args, head), metadata, )), PipelineData::Empty { .. } => Err(ShellError::PipelineEmpty { dst_span: head }), diff --git a/crates/nu-command/src/platform/ansi/ansi_.rs b/crates/nu-command/src/platform/ansi/ansi_.rs index 427c7c3a73..9e0b0bba66 100644 --- a/crates/nu-command/src/platform/ansi/ansi_.rs +++ b/crates/nu-command/src/platform/ansi/ansi_.rs @@ -784,10 +784,7 @@ fn heavy_lifting(code: Value, escape: bool, osc: bool, call: &Call) -> Result { return Err(ShellError::TypeMismatch { err_message: String::from("Unknown ansi code"), - span: call - .positional_nth(0) - .expect("Unexpected missing argument") - .span, + span: code.span(), }) } } diff --git a/crates/nu-command/src/platform/is_terminal.rs b/crates/nu-command/src/platform/is_terminal.rs index 770fa45289..c67329e839 100644 --- a/crates/nu-command/src/platform/is_terminal.rs +++ b/crates/nu-command/src/platform/is_terminal.rs @@ -1,5 +1,4 @@ use nu_engine::command_prelude::*; -use nu_protocol::span; use std::io::IsTerminal as _; #[derive(Clone)] @@ -57,12 +56,9 @@ impl Command for IsTerminal { }); } _ => { - let spans: Vec<_> = call.arguments.iter().map(|arg| arg.span()).collect(); - let span = span(&spans); - return Err(ShellError::IncompatibleParametersSingle { msg: "Only one stream may be checked".into(), - span, + span: Span::merge_many(call.arguments.iter().map(|arg| arg.span())), }); } }; diff --git a/crates/nu-command/src/platform/kill.rs b/crates/nu-command/src/platform/kill.rs index 59486cf1ad..2e47ee8c78 100644 --- a/crates/nu-command/src/platform/kill.rs +++ b/crates/nu-command/src/platform/kill.rs @@ -1,5 +1,4 @@ use nu_engine::command_prelude::*; -use nu_protocol::span; use std::process::{Command as CommandSys, Stdio}; #[derive(Clone)] @@ -96,7 +95,7 @@ impl Command for Kill { })? .span, right_message: "signal".to_string(), - right_span: span(&[ + right_span: Span::merge( call.get_named_arg("signal") .ok_or_else(|| ShellError::GenericError { error: "Flag error".into(), @@ -107,7 +106,7 @@ impl Command for Kill { })? .span, signal_span, - ]), + ), }); } cmd.arg("-9"); diff --git a/crates/nu-command/src/progress_bar.rs b/crates/nu-command/src/progress_bar.rs index 17ddeeb64e..db4d4e23b1 100644 --- a/crates/nu-command/src/progress_bar.rs +++ b/crates/nu-command/src/progress_bar.rs @@ -6,8 +6,6 @@ use std::fmt; pub struct NuProgressBar { pub pb: ProgressBar, - bytes_processed: u64, - total_bytes: Option, } impl NuProgressBar { @@ -40,8 +38,6 @@ impl NuProgressBar { NuProgressBar { pb: new_progress_bar, - total_bytes: None, - bytes_processed: 0, } } @@ -57,12 +53,4 @@ impl NuProgressBar { pub fn abandoned_msg(&self, msg: String) { self.pb.abandon_with_message(msg); } - - pub fn clone(&self) -> NuProgressBar { - NuProgressBar { - pb: self.pb.clone(), - bytes_processed: self.bytes_processed, - total_bytes: self.total_bytes, - } - } } diff --git a/crates/nu-command/src/stor/insert.rs b/crates/nu-command/src/stor/insert.rs index 2aeb076d44..e0c0ad4d28 100644 --- a/crates/nu-command/src/stor/insert.rs +++ b/crates/nu-command/src/stor/insert.rs @@ -1,5 +1,6 @@ -use crate::database::{SQLiteDatabase, MEMORY_DB}; +use crate::database::{values_to_sql, SQLiteDatabase, MEMORY_DB}; use nu_engine::command_prelude::*; +use rusqlite::params_from_iter; #[derive(Clone)] pub struct StorInsert; @@ -57,86 +58,81 @@ impl Command for StorInsert { // let config = engine_state.get_config(); let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); - if table_name.is_none() { - return Err(ShellError::MissingParameter { - param_name: "requires at table name".into(), - span, - }); - } - let new_table_name = table_name.unwrap_or("table".into()); - if let Ok(conn) = db.open_connection() { - match columns { - Some(record) => { - let mut create_stmt = format!("INSERT INTO {} ( ", new_table_name); - let cols = record.columns(); - cols.for_each(|col| { - create_stmt.push_str(&format!("{}, ", col)); - }); - if create_stmt.ends_with(", ") { - create_stmt.pop(); - create_stmt.pop(); - } + process(table_name, span, &db, columns)?; - create_stmt.push_str(") VALUES ( "); - let vals = record.values(); - vals.for_each(|val| match val { - Value::Int { val, .. } => { - create_stmt.push_str(&format!("{}, ", val)); - } - Value::Float { val, .. } => { - create_stmt.push_str(&format!("{}, ", val)); - } - Value::String { val, .. } => { - create_stmt.push_str(&format!("'{}', ", val)); - } - Value::Date { val, .. } => { - create_stmt.push_str(&format!("'{}', ", val)); - } - Value::Bool { val, .. } => { - create_stmt.push_str(&format!("{}, ", val)); - } - _ => { - // return Err(ShellError::UnsupportedInput { - // msg: format!("{} is not a valid datepart, expected one of year, month, day, hour, minute, second, millisecond, microsecond, nanosecond", part.item), - // input: "value originates from here".to_string(), - // msg_span: span, - // input_span: val.span(), - // }); - } - }); - if create_stmt.ends_with(", ") { - create_stmt.pop(); - create_stmt.pop(); - } - - create_stmt.push(')'); - - // dbg!(&create_stmt); - - conn.execute(&create_stmt, []) - .map_err(|err| ShellError::GenericError { - error: "Failed to open SQLite connection in memory from insert".into(), - msg: err.to_string(), - span: Some(Span::test_data()), - help: None, - inner: vec![], - })?; - } - None => { - return Err(ShellError::MissingParameter { - param_name: "requires at least one column".into(), - span: call.head, - }); - } - }; - } - // dbg!(db.clone()); Ok(Value::custom(db, span).into_pipeline_data()) } } +fn process( + table_name: Option, + span: Span, + db: &SQLiteDatabase, + columns: Option, +) -> Result<(), ShellError> { + if table_name.is_none() { + return Err(ShellError::MissingParameter { + param_name: "requires at table name".into(), + span, + }); + } + let new_table_name = table_name.unwrap_or("table".into()); + if let Ok(conn) = db.open_connection() { + match columns { + Some(record) => { + let mut create_stmt = format!("INSERT INTO {} ( ", new_table_name); + let cols = record.columns(); + cols.for_each(|col| { + create_stmt.push_str(&format!("{}, ", col)); + }); + if create_stmt.ends_with(", ") { + create_stmt.pop(); + create_stmt.pop(); + } + + // Values are set as placeholders. + create_stmt.push_str(") VALUES ( "); + for (index, _) in record.columns().enumerate() { + create_stmt.push_str(&format!("?{}, ", index + 1)); + } + + if create_stmt.ends_with(", ") { + create_stmt.pop(); + create_stmt.pop(); + } + + create_stmt.push(')'); + + // dbg!(&create_stmt); + + // Get the params from the passed values + let params = values_to_sql(record.values().cloned())?; + + conn.execute(&create_stmt, params_from_iter(params)) + .map_err(|err| ShellError::GenericError { + error: "Failed to open SQLite connection in memory from insert".into(), + msg: err.to_string(), + span: Some(Span::test_data()), + help: None, + inner: vec![], + })?; + } + None => { + return Err(ShellError::MissingParameter { + param_name: "requires at least one column".into(), + span, + }); + } + }; + } + // dbg!(db.clone()); + Ok(()) +} + #[cfg(test)] mod test { + use chrono::DateTime; + use super::*; #[test] @@ -145,4 +141,160 @@ mod test { test_examples(StorInsert {}) } + + #[test] + fn test_process_with_simple_parameters() { + let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let create_stmt = "CREATE TABLE test_process_with_simple_parameters ( + int_column INTEGER, + real_column REAL, + str_column VARCHAR(255), + bool_column BOOLEAN, + date_column DATETIME DEFAULT(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')) + )"; + + let conn = db + .open_connection() + .expect("Test was unable to open connection."); + conn.execute(create_stmt, []) + .expect("Failed to create table as part of test."); + let table_name = Some("test_process_with_simple_parameters".to_string()); + let span = Span::unknown(); + let mut columns = Record::new(); + columns.insert("int_column".to_string(), Value::test_int(42)); + columns.insert("real_column".to_string(), Value::test_float(3.1)); + columns.insert( + "str_column".to_string(), + Value::test_string("SimpleString".to_string()), + ); + columns.insert("bool_column".to_string(), Value::test_bool(true)); + columns.insert( + "date_column".to_string(), + Value::test_date( + DateTime::parse_from_str("2021-12-30 00:00:00 +0000", "%Y-%m-%d %H:%M:%S %z") + .expect("Date string should parse."), + ), + ); + + let result = process(table_name, span, &db, Some(columns)); + + assert!(result.is_ok()); + } + + #[test] + fn test_process_string_with_space() { + let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let create_stmt = "CREATE TABLE test_process_string_with_space ( + str_column VARCHAR(255) + )"; + + let conn = db + .open_connection() + .expect("Test was unable to open connection."); + conn.execute(create_stmt, []) + .expect("Failed to create table as part of test."); + let table_name = Some("test_process_string_with_space".to_string()); + let span = Span::unknown(); + let mut columns = Record::new(); + columns.insert( + "str_column".to_string(), + Value::test_string("String With Spaces".to_string()), + ); + + let result = process(table_name, span, &db, Some(columns)); + + assert!(result.is_ok()); + } + + #[test] + fn test_no_errors_when_string_too_long() { + let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let create_stmt = "CREATE TABLE test_errors_when_string_too_long ( + str_column VARCHAR(8) + )"; + + let conn = db + .open_connection() + .expect("Test was unable to open connection."); + conn.execute(create_stmt, []) + .expect("Failed to create table as part of test."); + let table_name = Some("test_errors_when_string_too_long".to_string()); + let span = Span::unknown(); + let mut columns = Record::new(); + columns.insert( + "str_column".to_string(), + Value::test_string("ThisIsALongString".to_string()), + ); + + let result = process(table_name, span, &db, Some(columns)); + // SQLite uses dynamic typing, making any length acceptable for a varchar column + assert!(result.is_ok()); + } + + #[test] + fn test_no_errors_when_param_is_wrong_type() { + let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let create_stmt = "CREATE TABLE test_errors_when_param_is_wrong_type ( + int_column INT + )"; + + let conn = db + .open_connection() + .expect("Test was unable to open connection."); + conn.execute(create_stmt, []) + .expect("Failed to create table as part of test."); + let table_name = Some("test_errors_when_param_is_wrong_type".to_string()); + let span = Span::unknown(); + let mut columns = Record::new(); + columns.insert( + "int_column".to_string(), + Value::test_string("ThisIsTheWrongType".to_string()), + ); + + let result = process(table_name, span, &db, Some(columns)); + // SQLite uses dynamic typing, making any type acceptable for a column + assert!(result.is_ok()); + } + + #[test] + fn test_errors_when_column_doesnt_exist() { + let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let create_stmt = "CREATE TABLE test_errors_when_column_doesnt_exist ( + int_column INT + )"; + + let conn = db + .open_connection() + .expect("Test was unable to open connection."); + conn.execute(create_stmt, []) + .expect("Failed to create table as part of test."); + let table_name = Some("test_errors_when_column_doesnt_exist".to_string()); + let span = Span::unknown(); + let mut columns = Record::new(); + columns.insert( + "not_a_column".to_string(), + Value::test_string("ThisIsALongString".to_string()), + ); + + let result = process(table_name, span, &db, Some(columns)); + + assert!(result.is_err()); + } + + #[test] + fn test_errors_when_table_doesnt_exist() { + let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + + let table_name = Some("test_errors_when_table_doesnt_exist".to_string()); + let span = Span::unknown(); + let mut columns = Record::new(); + columns.insert( + "str_column".to_string(), + Value::test_string("ThisIsALongString".to_string()), + ); + + let result = process(table_name, span, &db, Some(columns)); + + assert!(result.is_err()); + } } diff --git a/crates/nu-command/src/strings/char_.rs b/crates/nu-command/src/strings/char_.rs index e227f2667f..7d8d152b56 100644 --- a/crates/nu-command/src/strings/char_.rs +++ b/crates/nu-command/src/strings/char_.rs @@ -235,18 +235,18 @@ impl Command for Char { // handle -i flag if integer { - let int_args: Vec = call.rest_const(working_set, 0)?; - handle_integer_flag(int_args, call, call_span) + let int_args = call.rest_const(working_set, 0)?; + handle_integer_flag(int_args, call_span) } // handle -u flag else if unicode { - let string_args: Vec = call.rest_const(working_set, 0)?; - handle_unicode_flag(string_args, call, call_span) + let string_args = call.rest_const(working_set, 0)?; + handle_unicode_flag(string_args, call_span) } // handle the rest else { - let string_args: Vec = call.rest_const(working_set, 0)?; - handle_the_rest(string_args, call, call_span) + let string_args = call.rest_const(working_set, 0)?; + handle_the_rest(string_args, call_span) } } @@ -270,18 +270,18 @@ impl Command for Char { // handle -i flag if integer { - let int_args: Vec = call.rest(engine_state, stack, 0)?; - handle_integer_flag(int_args, call, call_span) + let int_args = call.rest(engine_state, stack, 0)?; + handle_integer_flag(int_args, call_span) } // handle -u flag else if unicode { - let string_args: Vec = call.rest(engine_state, stack, 0)?; - handle_unicode_flag(string_args, call, call_span) + let string_args = call.rest(engine_state, stack, 0)?; + handle_unicode_flag(string_args, call_span) } // handle the rest else { - let string_args: Vec = call.rest(engine_state, stack, 0)?; - handle_the_rest(string_args, call, call_span) + let string_args = call.rest(engine_state, stack, 0)?; + handle_the_rest(string_args, call_span) } } } @@ -309,8 +309,7 @@ fn generate_character_list(ctrlc: Option>, call_span: Span) -> P } fn handle_integer_flag( - int_args: Vec, - call: &Call, + int_args: Vec>, call_span: Span, ) -> Result { if int_args.is_empty() { @@ -319,20 +318,17 @@ fn handle_integer_flag( span: call_span, }); } - let mut multi_byte = String::new(); - for (i, &arg) in int_args.iter().enumerate() { - let span = call - .positional_nth(i) - .expect("Unexpected missing argument") - .span; - multi_byte.push(integer_to_unicode_char(arg, span)?) - } - Ok(Value::string(multi_byte, call_span).into_pipeline_data()) + + let str = int_args + .into_iter() + .map(integer_to_unicode_char) + .collect::>()?; + + Ok(Value::string(str, call_span).into_pipeline_data()) } fn handle_unicode_flag( - string_args: Vec, - call: &Call, + string_args: Vec>, call_span: Span, ) -> Result { if string_args.is_empty() { @@ -341,57 +337,53 @@ fn handle_unicode_flag( span: call_span, }); } - let mut multi_byte = String::new(); - for (i, arg) in string_args.iter().enumerate() { - let span = call - .positional_nth(i) - .expect("Unexpected missing argument") - .span; - multi_byte.push(string_to_unicode_char(arg, span)?) - } - Ok(Value::string(multi_byte, call_span).into_pipeline_data()) + + let str = string_args + .into_iter() + .map(string_to_unicode_char) + .collect::>()?; + + Ok(Value::string(str, call_span).into_pipeline_data()) } fn handle_the_rest( - string_args: Vec, - call: &Call, + string_args: Vec>, call_span: Span, ) -> Result { - if string_args.is_empty() { + let Some(s) = string_args.first() else { return Err(ShellError::MissingParameter { param_name: "missing name of the character".into(), span: call_span, }); - } - let special_character = str_to_character(&string_args[0]); + }; + + let special_character = str_to_character(&s.item); + if let Some(output) = special_character { Ok(Value::string(output, call_span).into_pipeline_data()) } else { Err(ShellError::TypeMismatch { err_message: "error finding named character".into(), - span: call - .positional_nth(0) - .expect("Unexpected missing argument") - .span, + span: s.span, }) } } -fn integer_to_unicode_char(value: i64, t: Span) -> Result { - let decoded_char = value.try_into().ok().and_then(std::char::from_u32); +fn integer_to_unicode_char(value: Spanned) -> Result { + let decoded_char = value.item.try_into().ok().and_then(std::char::from_u32); if let Some(ch) = decoded_char { Ok(ch) } else { Err(ShellError::TypeMismatch { err_message: "not a valid Unicode codepoint".into(), - span: t, + span: value.span, }) } } -fn string_to_unicode_char(s: &str, t: Span) -> Result { - let decoded_char = u32::from_str_radix(s, 16) +fn string_to_unicode_char(s: Spanned) -> Result { + let decoded_char = u32::from_str_radix(&s.item, 16) .ok() .and_then(std::char::from_u32); @@ -400,7 +392,7 @@ fn string_to_unicode_char(s: &str, t: Span) -> Result { } else { Err(ShellError::TypeMismatch { err_message: "error decoding Unicode character".into(), - span: t, + span: s.span, }) } } diff --git a/crates/nu-command/src/strings/encode_decode/decode.rs b/crates/nu-command/src/strings/encode_decode/decode.rs index 25b8f59ec2..9b13fad202 100644 --- a/crates/nu-command/src/strings/encode_decode/decode.rs +++ b/crates/nu-command/src/strings/encode_decode/decode.rs @@ -57,16 +57,12 @@ documentation link at https://docs.rs/encoding_rs/latest/encoding_rs/#statics"# let encoding: Option> = call.opt(engine_state, stack, 0)?; match input { - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - span: input_span, - .. - } => { - let bytes: Vec = stream.into_bytes()?.item; + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + let bytes = stream.into_bytes()?; match encoding { Some(encoding_name) => super::encoding::decode(head, encoding_name, &bytes), - None => super::encoding::detect_encoding_name(head, input_span, &bytes) + None => super::encoding::detect_encoding_name(head, span, &bytes) .map(|encoding| encoding.decode(&bytes).0.into_owned()) .map(|s| Value::string(s, head)), } diff --git a/crates/nu-command/src/strings/encode_decode/encode.rs b/crates/nu-command/src/strings/encode_decode/encode.rs index 98fcc34179..113c0fe548 100644 --- a/crates/nu-command/src/strings/encode_decode/encode.rs +++ b/crates/nu-command/src/strings/encode_decode/encode.rs @@ -81,13 +81,10 @@ documentation link at https://docs.rs/encoding_rs/latest/encoding_rs/#statics"# let ignore_errors = call.has_flag(engine_state, stack, "ignore-errors")?; match input { - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); let s = stream.into_string()?; - super::encoding::encode(head, encoding, &s.item, s.span, ignore_errors) + super::encoding::encode(head, encoding, &s, span, ignore_errors) .map(|val| val.into_pipeline_data()) } PipelineData::Value(v, ..) => { diff --git a/crates/nu-command/src/strings/parse.rs b/crates/nu-command/src/strings/parse.rs index 8f6d35b0b3..bc70d4679c 100644 --- a/crates/nu-command/src/strings/parse.rs +++ b/crates/nu-command/src/strings/parse.rs @@ -1,9 +1,9 @@ -use fancy_regex::Regex; +use fancy_regex::{Captures, Regex}; use nu_engine::command_prelude::*; -use nu_protocol::{ListStream, ValueIterator}; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, +use nu_protocol::ListStream; +use std::{ + collections::VecDeque, + sync::{atomic::AtomicBool, Arc}, }; #[derive(Clone)] @@ -119,7 +119,6 @@ fn operate( let head = call.head; let pattern: Spanned = call.req(engine_state, stack, 0)?; let regex: bool = call.has_flag(engine_state, stack, "regex")?; - let ctrlc = engine_state.ctrlc.clone(); let pattern_item = pattern.item; let pattern_span = pattern.span; @@ -130,7 +129,7 @@ fn operate( build_regex(&pattern_item, pattern_span)? }; - let regex_pattern = Regex::new(&item_to_parse).map_err(|e| ShellError::GenericError { + let regex = Regex::new(&item_to_parse).map_err(|e| ShellError::GenericError { error: "Error with regular expression".into(), msg: e.to_string(), span: Some(pattern_span), @@ -138,92 +137,99 @@ fn operate( inner: vec![], })?; - let columns = column_names(®ex_pattern); + let columns = regex + .capture_names() + .skip(1) + .enumerate() + .map(|(i, name)| { + name.map(String::from) + .unwrap_or_else(|| format!("capture{i}")) + }) + .collect::>(); + + let ctrlc = engine_state.ctrlc.clone(); match input { PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::Value(..) => { - let mut parsed: Vec = Vec::new(); + PipelineData::Value(value, ..) => match value { + Value::String { val, .. } => { + let captures = regex + .captures_iter(&val) + .map(|captures| captures_to_value(captures, &columns, head)) + .collect::>()?; - for v in input { - let v_span = v.span(); - match v.coerce_into_string() { - Ok(s) => { - let results = regex_pattern.captures_iter(&s); - - for c in results { - let captures = match c { - Ok(c) => c, - Err(e) => { - return Err(ShellError::GenericError { - error: "Error with regular expression captures".into(), - msg: e.to_string(), - span: None, - help: None, - inner: vec![], - }) - } - }; - - let record = columns - .iter() - .zip(captures.iter().skip(1)) - .map(|(column_name, cap)| { - let cap_string = cap.map(|v| v.as_str()).unwrap_or(""); - (column_name.clone(), Value::string(cap_string, v_span)) - }) - .collect(); - - parsed.push(Value::record(record, head)); - } - } - Err(_) => { - return Err(ShellError::PipelineMismatch { - exp_input_type: "string".into(), - dst_span: head, - src_span: v_span, - }) - } - } + Ok(Value::list(captures, head).into_pipeline_data()) } + Value::List { vals, .. } => { + let iter = vals.into_iter().map(move |val| { + let span = val.span(); + val.into_string().map_err(|_| ShellError::PipelineMismatch { + exp_input_type: "string".into(), + dst_span: head, + src_span: span, + }) + }); - Ok(ListStream::new(parsed.into_iter(), head, ctrlc).into()) - } + let iter = ParseIter { + captures: VecDeque::new(), + regex, + columns, + iter, + span: head, + ctrlc, + }; + + Ok(ListStream::new(iter, head, None).into()) + } + value => Err(ShellError::PipelineMismatch { + exp_input_type: "string".into(), + dst_span: head, + src_span: value.span(), + }), + }, PipelineData::ListStream(stream, ..) => Ok(stream - .modify(|stream| ParseStreamer { - span: head, - excess: Vec::new(), - regex: regex_pattern, - columns, - stream, - ctrlc, + .modify(|stream| { + let iter = stream.map(move |val| { + let span = val.span(); + val.into_string().map_err(|_| ShellError::PipelineMismatch { + exp_input_type: "string".into(), + dst_span: head, + src_span: span, + }) + }); + + ParseIter { + captures: VecDeque::new(), + regex, + columns, + iter, + span: head, + ctrlc, + } }) .into()), + PipelineData::ByteStream(stream, ..) => { + if let Some(lines) = stream.lines() { + let iter = ParseIter { + captures: VecDeque::new(), + regex, + columns, + iter: lines, + span: head, + ctrlc, + }; - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::Empty), - - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => Ok(ListStream::new( - ParseStreamerExternal { - span: head, - excess: Vec::new(), - regex: regex_pattern, - columns, - stream: stream.stream, - }, - head, - ctrlc, - ) - .into()), + Ok(ListStream::new(iter, head, None).into()) + } else { + Ok(PipelineData::Empty) + } + } } } fn build_regex(input: &str, span: Span) -> Result { let mut output = "(?s)\\A".to_string(); - //let mut loop_input = input; let mut loop_input = input.chars().peekable(); loop { let mut before = String::new(); @@ -274,172 +280,73 @@ fn build_regex(input: &str, span: Span) -> Result { Ok(output) } -fn column_names(regex: &Regex) -> Vec { - regex - .capture_names() - .enumerate() - .skip(1) - .map(|(i, name)| { - name.map(String::from) - .unwrap_or_else(|| format!("capture{}", i - 1)) - }) - .collect() -} - -pub struct ParseStreamer { - span: Span, - excess: Vec, +struct ParseIter>> { + captures: VecDeque, regex: Regex, columns: Vec, - stream: ValueIterator, + iter: I, + span: Span, ctrlc: Option>, } -impl Iterator for ParseStreamer { - type Item = Value; - fn next(&mut self) -> Option { - if !self.excess.is_empty() { - return Some(self.excess.remove(0)); +impl>> ParseIter { + fn populate_captures(&mut self, str: &str) -> Result<(), ShellError> { + for captures in self.regex.captures_iter(str) { + self.captures + .push_back(captures_to_value(captures, &self.columns, self.span)?); } + Ok(()) + } +} +impl>> Iterator for ParseIter { + type Item = Value; + + fn next(&mut self) -> Option { loop { - if let Some(ctrlc) = &self.ctrlc { - if ctrlc.load(Ordering::SeqCst) { - break None; - } + if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { + return None; } - let v = self.stream.next()?; - let span = v.span(); + if let Some(val) = self.captures.pop_front() { + return Some(val); + } - let Ok(s) = v.coerce_into_string() else { - return Some(Value::error( - ShellError::PipelineMismatch { - exp_input_type: "string".into(), - dst_span: self.span, - src_span: span, - }, - span, - )); - }; + let result = self + .iter + .next()? + .and_then(|str| self.populate_captures(&str)); - let parsed = stream_helper( - self.regex.clone(), - span, - s, - self.columns.clone(), - &mut self.excess, - ); - - if parsed.is_none() { - continue; - }; - - return parsed; + if let Err(err) = result { + return Some(Value::error(err, self.span)); + } } } } -pub struct ParseStreamerExternal { +fn captures_to_value( + captures: Result, + columns: &[String], span: Span, - excess: Vec, - regex: Regex, - columns: Vec, - stream: Box, ShellError>> + Send + 'static>, -} +) -> Result { + let captures = captures.map_err(|err| ShellError::GenericError { + error: "Error with regular expression captures".into(), + msg: err.to_string(), + span: Some(span), + help: None, + inner: vec![], + })?; -impl Iterator for ParseStreamerExternal { - type Item = Value; - fn next(&mut self) -> Option { - if !self.excess.is_empty() { - return Some(self.excess.remove(0)); - } + let record = columns + .iter() + .zip(captures.iter().skip(1)) + .map(|(column, match_)| { + let match_str = match_.map(|m| m.as_str()).unwrap_or(""); + (column.clone(), Value::string(match_str, span)) + }) + .collect(); - let mut chunk = self.stream.next(); - - // Collect all `stream` chunks into a single `chunk` to be able to deal with matches that - // extend across chunk boundaries. - // This is a stop-gap solution until the `regex` crate supports streaming or an alternative - // solution is found. - // See https://github.com/nushell/nushell/issues/9795 - while let Some(Ok(chunks)) = &mut chunk { - match self.stream.next() { - Some(Ok(mut next_chunk)) => chunks.append(&mut next_chunk), - error @ Some(Err(_)) => chunk = error, - None => break, - } - } - - let chunk = match chunk { - Some(Ok(chunk)) => chunk, - Some(Err(err)) => return Some(Value::error(err, self.span)), - _ => return None, - }; - - let Ok(chunk) = String::from_utf8(chunk) else { - return Some(Value::error( - ShellError::PipelineMismatch { - exp_input_type: "string".into(), - dst_span: self.span, - src_span: self.span, - }, - self.span, - )); - }; - - stream_helper( - self.regex.clone(), - self.span, - chunk, - self.columns.clone(), - &mut self.excess, - ) - } -} - -fn stream_helper( - regex: Regex, - span: Span, - s: String, - columns: Vec, - excess: &mut Vec, -) -> Option { - let results = regex.captures_iter(&s); - - for c in results { - let captures = match c { - Ok(c) => c, - Err(e) => { - return Some(Value::error( - ShellError::GenericError { - error: "Error with regular expression captures".into(), - msg: e.to_string(), - span: Some(span), - help: Some(e.to_string()), - inner: vec![], - }, - span, - )) - } - }; - - let record = columns - .iter() - .zip(captures.iter().skip(1)) - .map(|(column_name, cap)| { - let cap_string = cap.map(|v| v.as_str()).unwrap_or(""); - (column_name.clone(), Value::string(cap_string, span)) - }) - .collect(); - - excess.push(Value::record(record, span)); - } - - if !excess.is_empty() { - Some(excess.remove(0)) - } else { - None - } + Ok(Value::record(record, span)) } #[cfg(test)] diff --git a/crates/nu-command/src/strings/str_/contains.rs b/crates/nu-command/src/strings/str_/contains.rs index abb0ce1a2b..bc1d5bcf11 100644 --- a/crates/nu-command/src/strings/str_/contains.rs +++ b/crates/nu-command/src/strings/str_/contains.rs @@ -40,7 +40,7 @@ impl Command for SubCommand { "For a data structure input, check strings at the given cell paths, and replace with result.", ) .switch("ignore-case", "search is case insensitive", Some('i')) - .switch("not", "does not contain", Some('n')) + .switch("not", "DEPRECATED OPTION: does not contain", Some('n')) .category(Category::Strings) } @@ -59,6 +59,20 @@ impl Command for SubCommand { call: &Call, input: PipelineData, ) -> Result { + if call.has_flag(engine_state, stack, "not")? { + nu_protocol::report_error_new( + engine_state, + &ShellError::GenericError { + error: "Deprecated option".into(), + msg: "`str contains --not {string}` is deprecated and will be removed in 0.95." + .into(), + span: Some(call.head), + help: Some("Please use the `not` operator instead.".into()), + inner: vec![], + }, + ); + } + let cell_paths: Vec = call.rest(engine_state, stack, 1)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); let args = Arguments { @@ -120,15 +134,6 @@ impl Command for SubCommand { Value::test_bool(false), ])), }, - Example { - description: "Check if list does not contain string", - example: "[one two three] | str contains --not o", - result: Some(Value::test_list(vec![ - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(true), - ])), - }, ] } } diff --git a/crates/nu-command/src/system/complete.rs b/crates/nu-command/src/system/complete.rs index c622c86f3c..409cef1f27 100644 --- a/crates/nu-command/src/system/complete.rs +++ b/crates/nu-command/src/system/complete.rs @@ -1,6 +1,5 @@ use nu_engine::command_prelude::*; use nu_protocol::OutDest; -use std::thread; #[derive(Clone)] pub struct Complete; @@ -31,78 +30,53 @@ impl Command for Complete { call: &Call, input: PipelineData, ) -> Result { + let head = call.head; match input { - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - .. - } => { - let mut record = Record::new(); - - // use a thread to receive stderr message. - // Or we may get a deadlock if child process sends out too much bytes to stdout. - // - // For example: in normal linux system, stdout pipe's limit is 65535 bytes. - // if child process sends out 65536 bytes, the process will be hanged because no consumer - // consumes the first 65535 bytes - // So we need a thread to receive stderr message, then the current thread can continue to consume - // stdout messages. - let stderr_handler = stderr - .map(|stderr| { - let stderr_span = stderr.span; - thread::Builder::new() - .name("stderr consumer".to_string()) - .spawn(move || { - let stderr = stderr.into_bytes()?; - if let Ok(st) = String::from_utf8(stderr.item.clone()) { - Ok::<_, ShellError>(Value::string(st, stderr.span)) - } else { - Ok::<_, ShellError>(Value::binary(stderr.item, stderr.span)) - } - }) - .map(|handle| (handle, stderr_span)) - .err_span(call.head) - }) - .transpose()?; - - if let Some(stdout) = stdout { - let stdout = stdout.into_bytes()?; - record.push( - "stdout", - if let Ok(st) = String::from_utf8(stdout.item.clone()) { - Value::string(st, stdout.span) - } else { - Value::binary(stdout.item, stdout.span) - }, - ) - } - - if let Some((handler, stderr_span)) = stderr_handler { - let res = handler.join().map_err(|err| ShellError::ExternalCommand { - label: "Fail to receive external commands stderr message".to_string(), - help: format!("{err:?}"), - span: stderr_span, - })??; - record.push("stderr", res) + PipelineData::ByteStream(stream, ..) => { + let Ok(child) = stream.into_child() else { + return Err(ShellError::GenericError { + error: "Complete only works with external commands".into(), + msg: "complete only works on external commands".into(), + span: Some(call.head), + help: None, + inner: vec![], + }); }; - if let Some(exit_code) = exit_code { - let mut v: Vec<_> = exit_code.into_iter().collect(); + let output = child.wait_with_output()?; + let exit_code = output.exit_status.code(); + let mut record = Record::new(); - if let Some(v) = v.pop() { - record.push("exit_code", v); - } + if let Some(stdout) = output.stdout { + record.push( + "stdout", + match String::from_utf8(stdout) { + Ok(str) => Value::string(str, head), + Err(err) => Value::binary(err.into_bytes(), head), + }, + ); } + if let Some(stderr) = output.stderr { + record.push( + "stderr", + match String::from_utf8(stderr) { + Ok(str) => Value::string(str, head), + Err(err) => Value::binary(err.into_bytes(), head), + }, + ); + } + + record.push("exit_code", Value::int(exit_code.into(), head)); + Ok(Value::record(record, call.head).into_pipeline_data()) } // bubble up errors from the previous command PipelineData::Value(Value::Error { error, .. }, _) => Err(*error), _ => Err(ShellError::GenericError { - error: "Complete only works with external streams".into(), - msg: "complete only works on external streams".into(), - span: Some(call.head), + error: "Complete only works with external commands".into(), + msg: "complete only works on external commands".into(), + span: Some(head), help: None, inner: vec![], }), diff --git a/crates/nu-command/src/system/nu_check.rs b/crates/nu-command/src/system/nu_check.rs index 260a1c7a59..f9e0879c00 100644 --- a/crates/nu-command/src/system/nu_check.rs +++ b/crates/nu-command/src/system/nu_check.rs @@ -69,18 +69,8 @@ impl Command for NuCheck { parse_script(&mut working_set, None, &contents, is_debug, call.head) } } - PipelineData::ExternalStream { - stdout: Some(stream), - .. - } => { - let mut contents = vec![]; - let raw_stream: Vec<_> = stream.stream.collect(); - for r in raw_stream { - match r { - Ok(v) => contents.extend(v), - Err(error) => return Err(error), - }; - } + PipelineData::ByteStream(stream, ..) => { + let contents = stream.into_bytes()?; if as_module { parse_module(&mut working_set, None, &contents, is_debug, call.head) @@ -160,7 +150,7 @@ impl Command for NuCheck { result: None, }, Example { - description: "Parse an external stream as script by showing error message", + description: "Parse a byte stream as script by showing error message", example: "open foo.nu | nu-check --debug script.nu", result: None, }, diff --git a/crates/nu-command/src/system/run_external.rs b/crates/nu-command/src/system/run_external.rs index e73bd4ab50..2941d80de3 100644 --- a/crates/nu-command/src/system/run_external.rs +++ b/crates/nu-command/src/system/run_external.rs @@ -1,16 +1,16 @@ use nu_cmd_base::hook::eval_hook; use nu_engine::{command_prelude::*, env_to_strings, get_eval_expression}; -use nu_protocol::{ast::Expr, did_you_mean, ListStream, NuGlob, OutDest, RawStream}; +use nu_protocol::{ast::Expr, did_you_mean, process::ChildProcess, ByteStream, NuGlob, OutDest}; use nu_system::ForegroundChild; use nu_utils::IgnoreCaseExt; use os_pipe::PipeReader; use pathdiff::diff_paths; use std::{ collections::HashMap, - io::{BufRead, BufReader, Read, Write}, + io::Write, path::{Path, PathBuf}, process::{Command as CommandSys, Stdio}, - sync::{mpsc, Arc}, + sync::Arc, thread, }; @@ -163,89 +163,124 @@ impl ExternalCommand { ) -> Result { let head = self.name.span; - #[allow(unused_mut)] - let (cmd, mut reader) = self.create_process(&input, false, head)?; - - #[cfg(all(not(unix), not(windows)))] // are there any systems like this? - let child = ForegroundChild::spawn(cmd); - #[cfg(windows)] - let child = match ForegroundChild::spawn(cmd) { - Ok(child) => Ok(child), - Err(err) => { - // Running external commands on Windows has 2 points of complication: - // 1. Some common Windows commands are actually built in to cmd.exe, not executables in their own right. - // 2. We need to let users run batch scripts etc. (.bat, .cmd) without typing their extension + let (child, reader, input) = { + // We may need to run `create_process` again, so we have to clone the underlying + // file or pipe in `input` here first. + let (input_consumed, stdin) = match &input { + PipelineData::ByteStream(stream, ..) => match stream.source() { + nu_protocol::ByteStreamSource::Read(_) => (false, Stdio::piped()), + nu_protocol::ByteStreamSource::File(file) => { + (true, file.try_clone().err_span(head)?.into()) + } + nu_protocol::ByteStreamSource::Child(child) => { + if let Some(nu_protocol::process::ChildPipe::Pipe(pipe)) = &child.stdout { + (true, pipe.try_clone().err_span(head)?.into()) + } else { + (false, Stdio::piped()) + } + } + }, + PipelineData::Empty => (false, Stdio::inherit()), + _ => (false, Stdio::piped()), + }; - // To support these situations, we have a fallback path that gets run if a command - // fails to be run as a normal executable: - // 1. "shell out" to cmd.exe if the command is a known cmd.exe internal command - // 2. Otherwise, use `which-rs` to look for batch files etc. then run those in cmd.exe + let mut input = input; + let (cmd, mut reader) = self.create_process(stdin, false, head)?; + let child = match ForegroundChild::spawn(cmd) { + Ok(child) => { + if input_consumed { + input = PipelineData::Empty; + } + Ok(child) + } + Err(err) => { + // Running external commands on Windows has 2 points of complication: + // 1. Some common Windows commands are actually built in to cmd.exe, not executables in their own right. + // 2. We need to let users run batch scripts etc. (.bat, .cmd) without typing their extension - // set the default value, maybe we'll override it later - let mut child = Err(err); + // To support these situations, we have a fallback path that gets run if a command + // fails to be run as a normal executable: + // 1. "shell out" to cmd.exe if the command is a known cmd.exe internal command + // 2. Otherwise, use `which-rs` to look for batch files etc. then run those in cmd.exe - // This has the full list of cmd.exe "internal" commands: https://ss64.com/nt/syntax-internal.html - // I (Reilly) went through the full list and whittled it down to ones that are potentially useful: - const CMD_INTERNAL_COMMANDS: [&str; 9] = [ - "ASSOC", "CLS", "ECHO", "FTYPE", "MKLINK", "PAUSE", "START", "VER", "VOL", - ]; - let command_name = &self.name.item; - let looks_like_cmd_internal = CMD_INTERNAL_COMMANDS - .iter() - .any(|&cmd| command_name.eq_ignore_ascii_case(cmd)); + // set the default value, maybe we'll override it later + let mut child = Err(err); - if looks_like_cmd_internal { - let (cmd, new_reader) = self.create_process(&input, true, head)?; - reader = new_reader; - child = ForegroundChild::spawn(cmd); - } else { - #[cfg(feature = "which-support")] - { - // maybe it's a batch file (foo.cmd) and the user typed `foo`. Try to find it with `which-rs` - // TODO: clean this up with an if-let chain once those are stable - if let Ok(path) = - nu_engine::env::path_str(engine_state, stack, self.name.span) + // This has the full list of cmd.exe "internal" commands: https://ss64.com/nt/syntax-internal.html + // I (Reilly) went through the full list and whittled it down to ones that are potentially useful: + const CMD_INTERNAL_COMMANDS: [&str; 9] = [ + "ASSOC", "CLS", "ECHO", "FTYPE", "MKLINK", "PAUSE", "START", "VER", "VOL", + ]; + let command_name = &self.name.item; + let looks_like_cmd_internal = CMD_INTERNAL_COMMANDS + .iter() + .any(|&cmd| command_name.eq_ignore_ascii_case(cmd)); + + let (data, stdin) = extract_stdio(input); + input = data; + + if looks_like_cmd_internal { + let (cmd, new_reader) = self.create_process(stdin, true, head)?; + reader = new_reader; + child = ForegroundChild::spawn(cmd); + } else { + #[cfg(feature = "which-support")] { - if let Some(cwd) = self.env_vars.get("PWD") { - // append cwd to PATH so `which-rs` looks in the cwd too. - // this approximates what cmd.exe does. - let path_with_cwd = format!("{};{}", cwd, path); - if let Ok(which_path) = - which::which_in(&self.name.item, Some(path_with_cwd), cwd) - { - if let Some(file_name) = which_path.file_name() { - if !file_name.to_string_lossy().eq_ignore_case(command_name) - { - // which-rs found an executable file with a slightly different name - // than the one the user tried. Let's try running it - let mut new_command = self.clone(); - new_command.name = Spanned { - item: file_name.to_string_lossy().to_string(), - span: self.name.span, - }; - let (cmd, new_reader) = - new_command.create_process(&input, true, head)?; - reader = new_reader; - child = ForegroundChild::spawn(cmd); + // maybe it's a batch file (foo.cmd) and the user typed `foo`. Try to find it with `which-rs` + // TODO: clean this up with an if-let chain once those are stable + if let Ok(path) = + nu_engine::env::path_str(engine_state, stack, self.name.span) + { + if let Some(cwd) = self.env_vars.get("PWD") { + // append cwd to PATH so `which-rs` looks in the cwd too. + // this approximates what cmd.exe does. + let path_with_cwd = format!("{};{}", cwd, path); + if let Ok(which_path) = + which::which_in(&self.name.item, Some(path_with_cwd), cwd) + { + if let Some(file_name) = which_path.file_name() { + if !file_name + .to_string_lossy() + .eq_ignore_case(command_name) + { + // which-rs found an executable file with a slightly different name + // than the one the user tried. Let's try running it + let mut new_command = self.clone(); + new_command.name = Spanned { + item: file_name.to_string_lossy().to_string(), + span: self.name.span, + }; + let (cmd, new_reader) = new_command + .create_process(stdin, true, head)?; + reader = new_reader; + child = ForegroundChild::spawn(cmd); + } } } } } } } - } - child - } + child + } + }; + + (child, reader, input) }; #[cfg(unix)] - let child = ForegroundChild::spawn( - cmd, - engine_state.is_interactive, - &engine_state.pipeline_externals_state, - ); + let (child, reader, input) = { + let (input, stdin) = extract_stdio(input); + let (cmd, reader) = self.create_process(stdin, false, head)?; + let child = ForegroundChild::spawn( + cmd, + engine_state.is_interactive, + &engine_state.pipeline_externals_state, + ); + (child, reader, input) + }; match child { Err(err) => { @@ -381,9 +416,8 @@ impl ExternalCommand { .name("external stdin worker".to_string()) .spawn(move || { let input = match input { - input @ PipelineData::Value(Value::Binary { .. }, ..) => { - Ok(input) - } + input @ PipelineData::ByteStream(..) => input, + input @ PipelineData::Value(Value::Binary { .. }, ..) => input, input => { let stack = &mut stack.start_capture(); // Attempt to render the input as a table before piping it to the external. @@ -397,143 +431,39 @@ impl ExternalCommand { stack, &Call::new(head), input, - ) + )? } }; - if let Ok(input) = input { + if let PipelineData::ByteStream(stream, ..) = input { + stream.write_to(&mut stdin_write)?; + } else { for value in input.into_iter() { - let buf = match value { - Value::String { val, .. } => val.into_bytes(), - Value::Binary { val, .. } => val, - _ => return Err(()), - }; - if stdin_write.write(&buf).is_err() { - return Ok(()); - } + let buf = value.coerce_into_binary()?; + stdin_write.write_all(&buf)?; } } - Ok(()) + Ok::<_, ShellError>(()) }) .err_span(head)?; } } - #[cfg(unix)] - let commandname = self.name.item.clone(); - let span = self.name.span; - let (exit_code_tx, exit_code_rx) = mpsc::channel(); + let child = + ChildProcess::new(child, reader, matches!(self.err, OutDest::Pipe), head)?; - let (stdout, stderr) = if let Some(combined) = reader { - ( - Some(RawStream::new( - Box::new(ByteLines::new(combined)), - engine_state.ctrlc.clone(), - head, - None, - )), - None, - ) - } else { - let stdout = child.as_mut().stdout.take().map(|out| { - RawStream::new( - Box::new(ByteLines::new(out)), - engine_state.ctrlc.clone(), - head, - None, - ) - }); - - let stderr = child.as_mut().stderr.take().map(|err| { - RawStream::new( - Box::new(ByteLines::new(err)), - engine_state.ctrlc.clone(), - head, - None, - ) - }); - - if matches!(self.err, OutDest::Pipe) { - (stderr, stdout) - } else { - (stdout, stderr) - } - }; - - // Create a thread to wait for an exit code. - thread::Builder::new() - .name("exit code waiter".into()) - .spawn(move || match child.as_mut().wait() { - Err(err) => Err(ShellError::ExternalCommand { - label: "External command exited with error".into(), - help: err.to_string(), - span, - }), - Ok(x) => { - #[cfg(unix)] - { - use nix::sys::signal::Signal; - use nu_ansi_term::{Color, Style}; - use std::os::unix::process::ExitStatusExt; - - if x.core_dumped() { - let cause = x - .signal() - .and_then(|sig| { - Signal::try_from(sig).ok().map(Signal::as_str) - }) - .unwrap_or("Something went wrong"); - - let style = Style::new().bold().on(Color::Red); - let message = format!( - "{cause}: child process '{commandname}' core dumped" - ); - eprintln!("{}", style.paint(&message)); - let _ = exit_code_tx.send(Value::error( - ShellError::ExternalCommand { - label: "core dumped".into(), - help: message, - span: head, - }, - head, - )); - return Ok(()); - } - } - if let Some(code) = x.code() { - let _ = exit_code_tx.send(Value::int(code as i64, head)); - } else if x.success() { - let _ = exit_code_tx.send(Value::int(0, head)); - } else { - let _ = exit_code_tx.send(Value::int(-1, head)); - } - Ok(()) - } - }) - .err_span(head)?; - - let exit_code = Some(ListStream::new( - ValueReceiver::new(exit_code_rx), - head, + Ok(PipelineData::ByteStream( + ByteStream::child(child, head), None, - )); - - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span: head, - metadata: None, - trim_end_newline: true, - }) + )) } } } pub fn create_process( &self, - input: &PipelineData, + stdin: Stdio, use_cmd: bool, span: Span, ) -> Result<(CommandSys, Option), ShellError> { @@ -578,11 +508,7 @@ impl ExternalCommand { None }; - // If there is an input from the pipeline. The stdin from the process - // is piped so it can be used to send the input information - if !input.is_nothing() { - process.stdin(Stdio::piped()); - } + process.stdin(stdin); Ok((process, reader)) } @@ -764,51 +690,14 @@ fn remove_quotes(input: String) -> String { } } -struct ByteLines(BufReader); - -impl ByteLines { - fn new(read: R) -> Self { - Self(BufReader::new(read)) - } -} - -impl Iterator for ByteLines { - type Item = Result, ShellError>; - - fn next(&mut self) -> Option { - let mut buf = Vec::new(); - // `read_until` will never stop reading unless `\n` or EOF is encountered, - // so let's limit the number of bytes using `take` as the Rust docs suggest. - let capacity = self.0.capacity() as u64; - let mut reader = (&mut self.0).take(capacity); - match reader.read_until(b'\n', &mut buf) { - Ok(0) => None, - Ok(_) => Some(Ok(buf)), - Err(e) => Some(Err(e.into())), - } - } -} - -// Receiver used for the ListStream -// It implements iterator so it can be used as a ListStream -struct ValueReceiver { - rx: mpsc::Receiver, -} - -impl ValueReceiver { - pub fn new(rx: mpsc::Receiver) -> Self { - Self { rx } - } -} - -impl Iterator for ValueReceiver { - type Item = Value; - - fn next(&mut self) -> Option { - match self.rx.recv() { - Ok(v) => Some(v), - Err(_) => None, - } +fn extract_stdio(pipeline: PipelineData) -> (PipelineData, Stdio) { + match pipeline { + PipelineData::ByteStream(stream, metadata) => match stream.into_stdio() { + Ok(pipe) => (PipelineData::Empty, pipe), + Err(stream) => (PipelineData::ByteStream(stream, metadata), Stdio::piped()), + }, + PipelineData::Empty => (PipelineData::Empty, Stdio::inherit()), + data => (data, Stdio::piped()), } } diff --git a/crates/nu-command/src/system/sys/mod.rs b/crates/nu-command/src/system/sys/mod.rs index 5e0467c251..fcb40fd37d 100644 --- a/crates/nu-command/src/system/sys/mod.rs +++ b/crates/nu-command/src/system/sys/mod.rs @@ -16,9 +16,8 @@ pub use sys_::Sys; pub use temp::SysTemp; pub use users::SysUsers; -use chrono::{DateTime, Local}; +use chrono::{DateTime, FixedOffset, Local}; use nu_protocol::{record, Record, Span, Value}; -use std::time::{Duration, UNIX_EPOCH}; use sysinfo::{ Components, CpuRefreshKind, Disks, Networks, System, Users, MINIMUM_CPU_UPDATE_INTERVAL, }; @@ -171,22 +170,31 @@ pub fn host(span: Span) -> Record { record.push("hostname", Value::string(trim_cstyle_null(hostname), span)); } - record.push( - "uptime", - Value::duration(1000000000 * System::uptime() as i64, span), - ); + let uptime = System::uptime() + .saturating_mul(1_000_000_000) + .try_into() + .unwrap_or(i64::MAX); - // Creates a new SystemTime from the specified number of whole seconds - let d = UNIX_EPOCH + Duration::from_secs(System::boot_time()); - // Create DateTime from SystemTime - let datetime = DateTime::::from(d); - // Convert to local time and then rfc3339 - let timestamp_str = datetime.with_timezone(datetime.offset()).to_rfc3339(); - record.push("boot_time", Value::string(timestamp_str, span)); + record.push("uptime", Value::duration(uptime, span)); + + let boot_time = boot_time() + .map(|time| Value::date(time, span)) + .unwrap_or(Value::nothing(span)); + + record.push("boot_time", boot_time); record } +fn boot_time() -> Option> { + // Broken systems can apparently return really high values. + // See: https://github.com/nushell/nushell/issues/10155 + // First, try to convert u64 to i64, and then try to create a `DateTime`. + let secs = System::boot_time().try_into().ok()?; + let time = DateTime::from_timestamp(secs, 0)?; + Some(time.with_timezone(&Local).fixed_offset()) +} + pub fn temp(span: Span) -> Value { let components = Components::new_with_refreshed_list() .iter() diff --git a/crates/nu-command/src/viewers/table.rs b/crates/nu-command/src/viewers/table.rs index f86857d81e..63d7439193 100644 --- a/crates/nu-command/src/viewers/table.rs +++ b/crates/nu-command/src/viewers/table.rs @@ -6,7 +6,7 @@ use lscolors::{LsColors, Style}; use nu_color_config::{color_from_hex, StyleComputer, TextStyle}; use nu_engine::{command_prelude::*, env::get_config, env_to_string}; use nu_protocol::{ - Config, DataSource, ListStream, PipelineMetadata, RawStream, TableMode, ValueIterator, + ByteStream, Config, DataSource, ListStream, PipelineMetadata, TableMode, ValueIterator, }; use nu_table::{ common::create_nu_table_config, CollapsedTable, ExpandedTable, JustTable, NuTable, NuTableCell, @@ -14,8 +14,12 @@ use nu_table::{ }; use nu_utils::get_ls_colors; use std::{ - collections::VecDeque, io::IsTerminal, path::PathBuf, str::FromStr, sync::atomic::AtomicBool, - sync::Arc, time::Instant, + collections::VecDeque, + io::{Cursor, IsTerminal}, + path::PathBuf, + str::FromStr, + sync::{atomic::AtomicBool, Arc}, + time::Instant, }; use terminal_size::{Height, Width}; use url::Url; @@ -360,25 +364,16 @@ fn handle_table_command( ) -> Result { let span = input.data.span().unwrap_or(input.call.head); match input.data { - PipelineData::ExternalStream { .. } => Ok(input.data), + PipelineData::ByteStream(..) => Ok(input.data), PipelineData::Value(Value::Binary { val, .. }, ..) => { - let bytes = format!("{}\n", nu_pretty_hex::pretty_hex(&val)).into_bytes(); + let bytes = { + let mut str = nu_pretty_hex::pretty_hex(&val); + str.push('\n'); + str.into_bytes() + }; let ctrlc = input.engine_state.ctrlc.clone(); - let stream = RawStream::new( - Box::new([Ok(bytes)].into_iter()), - ctrlc, - input.call.head, - None, - ); - - Ok(PipelineData::ExternalStream { - stdout: Some(stream), - stderr: None, - exit_code: None, - span: input.call.head, - metadata: None, - trim_end_newline: false, - }) + let stream = ByteStream::read(Cursor::new(bytes), input.call.head, ctrlc); + Ok(PipelineData::ByteStream(stream, None)) } // None of these two receive a StyleComputer because handle_row_stream() can produce it by itself using engine_state and stack. PipelineData::Value(Value::List { vals, .. }, metadata) => { @@ -613,16 +608,8 @@ fn handle_row_stream( ctrlc.clone(), cfg, ); - let stream = RawStream::new(Box::new(paginator), ctrlc, input.call.head, None); - - Ok(PipelineData::ExternalStream { - stdout: Some(stream), - stderr: None, - exit_code: None, - span: input.call.head, - metadata: None, - trim_end_newline: false, - }) + let stream = ByteStream::from_result_iter(paginator, input.call.head, None); + Ok(PipelineData::ByteStream(stream, None)) } fn make_clickable_link( diff --git a/crates/nu-command/tests/commands/cd.rs b/crates/nu-command/tests/commands/cd.rs index c7c30528e8..87af52aa4d 100644 --- a/crates/nu-command/tests/commands/cd.rs +++ b/crates/nu-command/tests/commands/cd.rs @@ -312,3 +312,17 @@ fn cd_permission_denied_folder() { assert!(actual.err.contains("Folder is not able to read")); }); } + +#[test] +#[cfg(unix)] +fn pwd_recovery() { + let nu = nu_test_support::fs::executable_path().display().to_string(); + let tmpdir = std::env::temp_dir().join("foobar").display().to_string(); + + // We `cd` into a temporary directory, then spawn another `nu` process to + // delete that directory. Then we attempt to recover by running `cd /`. + let cmd = format!("mkdir {tmpdir}; cd {tmpdir}; {nu} -c 'cd /; rm -r {tmpdir}'; cd /; pwd"); + let actual = nu!(cmd); + + assert_eq!(actual.out, "/"); +} diff --git a/crates/nu-command/tests/commands/complete.rs b/crates/nu-command/tests/commands/complete.rs index cc398d4c8c..a5a8b6a256 100644 --- a/crates/nu-command/tests/commands/complete.rs +++ b/crates/nu-command/tests/commands/complete.rs @@ -92,3 +92,16 @@ fn capture_error_with_both_stdout_stderr_messages_not_hang_nushell() { }, ) } + +#[test] +fn combined_pipe_redirection() { + let actual = nu!("$env.FOO = hello; $env.BAR = world; nu --testbin echo_env_mixed out-err FOO BAR o+e>| complete | get stdout"); + assert_eq!(actual.out, "helloworld"); +} + +#[test] +fn err_pipe_redirection() { + let actual = + nu!("$env.FOO = hello; nu --testbin echo_env_stderr FOO e>| complete | get stdout"); + assert_eq!(actual.out, "hello"); +} diff --git a/crates/nu-command/tests/commands/save.rs b/crates/nu-command/tests/commands/save.rs index b5776a7bb9..ef0304dc7c 100644 --- a/crates/nu-command/tests/commands/save.rs +++ b/crates/nu-command/tests/commands/save.rs @@ -407,3 +407,20 @@ fn save_same_file_without_extension_pipeline() { .contains("pipeline input and output are the same file")); }) } + +#[test] +fn save_with_custom_converter() { + Playground::setup("save_with_custom_converter", |dirs, _| { + let file = dirs.test().join("test.ndjson"); + + nu!(cwd: dirs.test(), pipeline( + r#" + def "to ndjson" []: any -> string { each { to json --raw } | to text } ; + {a: 1, b: 2} | save test.ndjson + "# + )); + + let actual = file_contents(file); + assert_eq!(actual, r#"{"a":1,"b":2}"#); + }) +} diff --git a/crates/nu-command/tests/format_conversions/csv.rs b/crates/nu-command/tests/format_conversions/csv.rs index 5915b3c4d4..a9be76d5c3 100644 --- a/crates/nu-command/tests/format_conversions/csv.rs +++ b/crates/nu-command/tests/format_conversions/csv.rs @@ -183,6 +183,7 @@ fn from_csv_text_with_tab_separator_to_table() { } #[test] +#[ignore = "csv crate has a bug when the last line is a comment: https://github.com/BurntSushi/rust-csv/issues/363"] fn from_csv_text_with_comments_to_table() { Playground::setup("filter_from_csv_test_5", |dirs, sandbox| { sandbox.with_files(&[FileWithContentToBeTrimmed( diff --git a/crates/nu-command/tests/format_conversions/tsv.rs b/crates/nu-command/tests/format_conversions/tsv.rs index 9627d0d0be..be57c60242 100644 --- a/crates/nu-command/tests/format_conversions/tsv.rs +++ b/crates/nu-command/tests/format_conversions/tsv.rs @@ -106,6 +106,7 @@ fn from_tsv_text_to_table() { } #[test] +#[ignore = "csv crate has a bug when the last line is a comment: https://github.com/BurntSushi/rust-csv/issues/363"] fn from_tsv_text_with_comments_to_table() { Playground::setup("filter_from_tsv_test_2", |dirs, sandbox| { sandbox.with_files(&[FileWithContentToBeTrimmed( diff --git a/crates/nu-engine/src/documentation.rs b/crates/nu-engine/src/documentation.rs index d4ae27a065..a113727be2 100644 --- a/crates/nu-engine/src/documentation.rs +++ b/crates/nu-engine/src/documentation.rs @@ -53,7 +53,7 @@ fn nu_highlight_string(code_string: &str, engine_state: &EngineState, stack: &mu Value::string(code_string, Span::unknown()).into_pipeline_data(), ) { let result = output.into_value(Span::unknown()); - if let Ok(s) = result.coerce_into_string() { + if let Ok(s) = result.and_then(Value::coerce_into_string) { return s; // successfully highlighted string } } @@ -280,7 +280,7 @@ fn get_documentation( ) { Ok(output) => { let result = output.into_value(Span::unknown()); - match result.coerce_into_string() { + match result.and_then(Value::coerce_into_string) { Ok(s) => { let _ = write!(long_desc, "\n > {s}\n"); } diff --git a/crates/nu-engine/src/env.rs b/crates/nu-engine/src/env.rs index ae226c4421..048d9bfb99 100644 --- a/crates/nu-engine/src/env.rs +++ b/crates/nu-engine/src/env.rs @@ -32,7 +32,7 @@ enum ConversionResult { /// It returns Option instead of Result since we do want to translate all the values we can and /// skip errors. This function is called in the main() so we want to keep running, we cannot just /// exit. -pub fn convert_env_values(engine_state: &mut EngineState, stack: &Stack) -> Option { +pub fn convert_env_values(engine_state: &mut EngineState, stack: &Stack) -> Result<(), ShellError> { let mut error = None; let mut new_scope = HashMap::new(); @@ -85,7 +85,11 @@ pub fn convert_env_values(engine_state: &mut EngineState, stack: &Stack) -> Opti }); } - error + if let Some(err) = error { + Err(err) + } else { + Ok(()) + } } /// Translate one environment variable from Value to String @@ -346,14 +350,15 @@ fn get_converted_value( .and_then(|record| record.get(direction)); if let Some(conversion) = conversion { - match conversion.as_closure() { - Ok(closure) => ClosureEvalOnce::new(engine_state, stack, closure.clone()) - .debug(false) - .run_with_value(orig_val.clone()) - .map(|data| ConversionResult::Ok(data.into_value(orig_val.span()))) - .unwrap_or_else(ConversionResult::ConversionError), - Err(e) => ConversionResult::ConversionError(e), - } + conversion + .as_closure() + .and_then(|closure| { + ClosureEvalOnce::new(engine_state, stack, closure.clone()) + .debug(false) + .run_with_value(orig_val.clone()) + }) + .and_then(|data| data.into_value(orig_val.span())) + .map_or_else(ConversionResult::ConversionError, ConversionResult::Ok) } else { ConversionResult::CellPathError } diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index 6324b35ae7..0bc0c3727c 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -9,8 +9,8 @@ use nu_protocol::{ debugger::DebugContext, engine::{Closure, EngineState, Redirection, Stack}, eval_base::Eval, - Config, FromValue, IntoPipelineData, OutDest, PipelineData, ShellError, Span, Spanned, Type, - Value, VarId, ENV_VARIABLE_ID, + ByteStreamSource, Config, FromValue, IntoPipelineData, OutDest, PipelineData, ShellError, Span, + Spanned, Type, Value, VarId, ENV_VARIABLE_ID, }; use nu_utils::IgnoreCaseExt; use std::{borrow::Cow, fs::OpenOptions, path::PathBuf}; @@ -209,7 +209,6 @@ pub fn redirect_env(engine_state: &EngineState, caller_stack: &mut Stack, callee } } -#[allow(clippy::too_many_arguments)] fn eval_external( engine_state: &EngineState, stack: &mut Stack, @@ -284,7 +283,7 @@ pub fn eval_expression_with_input( let stack = &mut stack.start_capture(); // FIXME: protect this collect with ctrl-c input = eval_subexpression::(engine_state, stack, block, input)? - .into_value(*span) + .into_value(*span)? .follow_cell_path(&full_cell_path.tail, false)? .into_pipeline_data() } else { @@ -301,7 +300,7 @@ pub fn eval_expression_with_input( } }; - // If input is PipelineData::ExternalStream, + // If input an external command, // then `might_consume_external_result` will consume `stderr` if `stdout` is `None`. // This should not happen if the user wants to capture stderr. if !matches!(stack.stdout(), OutDest::Pipe | OutDest::Capture) @@ -309,15 +308,10 @@ pub fn eval_expression_with_input( { Ok((input, false)) } else { - Ok(might_consume_external_result(input)) + input.check_external_failed() } } -// Try to catch and detect if external command runs to failed. -fn might_consume_external_result(input: PipelineData) -> (PipelineData, bool) { - input.check_external_failed() -} - fn eval_redirection( engine_state: &EngineState, stack: &mut Stack, @@ -340,7 +334,13 @@ fn eval_redirection( } Ok(Redirection::file(options.create(true).open(path)?)) } - RedirectionTarget::Pipe { .. } => Ok(Redirection::Pipe(next_out.unwrap_or(OutDest::Pipe))), + RedirectionTarget::Pipe { .. } => { + let dest = match next_out { + None | Some(OutDest::Capture) => OutDest::Pipe, + Some(next) => next, + }; + Ok(Redirection::Pipe(dest)) + } } } @@ -367,11 +367,12 @@ fn eval_element_redirection( } => { let stderr = eval_redirection::(engine_state, stack, target, None)?; if matches!(stderr, Redirection::Pipe(OutDest::Pipe)) { + let dest = match next_out { + None | Some(OutDest::Capture) => OutDest::Pipe, + Some(next) => next, + }; // e>| redirection, don't override current stack `stdout` - Ok(( - None, - Some(next_out.map(Redirection::Pipe).unwrap_or(stderr)), - )) + Ok((None, Some(Redirection::Pipe(dest)))) } else { Ok((next_out.map(Redirection::Pipe), Some(stderr))) } @@ -403,10 +404,17 @@ fn eval_element_with_input_inner( element: &PipelineElement, input: PipelineData, ) -> Result<(PipelineData, bool), ShellError> { - let (data, ok) = eval_expression_with_input::(engine_state, stack, &element.expr, input)?; + let (data, failed) = + eval_expression_with_input::(engine_state, stack, &element.expr, input)?; - if !matches!(data, PipelineData::ExternalStream { .. }) { - if let Some(redirection) = element.redirection.as_ref() { + if let Some(redirection) = element.redirection.as_ref() { + let is_external = if let PipelineData::ByteStream(stream, ..) = &data { + matches!(stream.source(), ByteStreamSource::Child(..)) + } else { + false + }; + + if !is_external { match redirection { &PipelineRedirection::Single { source: RedirectionSource::Stderr, @@ -417,8 +425,8 @@ fn eval_element_with_input_inner( .. } => { return Err(ShellError::GenericError { - error: "`e>|` only works with external streams".into(), - msg: "`e>|` only works on external streams".into(), + error: "`e>|` only works on external commands".into(), + msg: "`e>|` only works on external commands".into(), span: Some(span), help: None, inner: vec![], @@ -429,8 +437,8 @@ fn eval_element_with_input_inner( target: RedirectionTarget::Pipe { span }, } => { return Err(ShellError::GenericError { - error: "`o+e>|` only works with external streams".into(), - msg: "`o+e>|` only works on external streams".into(), + error: "`o+e>|` only works on external commands".into(), + msg: "`o+e>|` only works on external commands".into(), span: Some(span), help: None, inner: vec![], @@ -441,15 +449,33 @@ fn eval_element_with_input_inner( } } - let data = if matches!(stack.pipe_stdout(), Some(OutDest::File(_))) - && !matches!(stack.pipe_stderr(), Some(OutDest::Pipe)) - { - data.write_to_out_dests(engine_state, stack)? - } else { - data + let has_stdout_file = matches!(stack.pipe_stdout(), Some(OutDest::File(_))); + + let data = match &data { + PipelineData::Value(..) | PipelineData::ListStream(..) => { + if has_stdout_file { + data.write_to_out_dests(engine_state, stack)?; + PipelineData::Empty + } else { + data + } + } + PipelineData::ByteStream(stream, ..) => { + let write = match stream.source() { + ByteStreamSource::Read(_) | ByteStreamSource::File(_) => has_stdout_file, + ByteStreamSource::Child(_) => false, + }; + if write { + data.write_to_out_dests(engine_state, stack)?; + PipelineData::Empty + } else { + data + } + } + PipelineData::Empty => PipelineData::Empty, }; - Ok((data, ok)) + Ok((data, failed)) } fn eval_element_with_input( @@ -459,12 +485,18 @@ fn eval_element_with_input( input: PipelineData, ) -> Result<(PipelineData, bool), ShellError> { D::enter_element(engine_state, element); - - let result = eval_element_with_input_inner::(engine_state, stack, element, input); - - D::leave_element(engine_state, element, &result); - - result + match eval_element_with_input_inner::(engine_state, stack, element, input) { + Ok((data, failed)) => { + let res = Ok(data); + D::leave_element(engine_state, element, &res); + res.map(|data| (data, failed)) + } + Err(err) => { + let res = Err(err); + D::leave_element(engine_state, element, &res); + res.map(|data| (data, false)) + } + } } pub fn eval_block_with_early_return( @@ -548,17 +580,20 @@ pub fn eval_block( } input = PipelineData::Empty; match output { - stream @ PipelineData::ExternalStream { .. } => { - let exit_code = stream.drain_with_exit_code()?; - stack.add_env_var( - "LAST_EXIT_CODE".into(), - Value::int(exit_code, last.expr.span), - ); - if exit_code != 0 { - break; + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + let status = stream.drain()?; + if let Some(status) = status { + stack.add_env_var( + "LAST_EXIT_CODE".into(), + Value::int(status.code().into(), span), + ); + if status.code() != 0 { + break; + } } } - PipelineData::ListStream(stream, _) => { + PipelineData::ListStream(stream, ..) => { stream.drain()?; } PipelineData::Value(..) | PipelineData::Empty => {} @@ -654,7 +689,7 @@ impl Eval for EvalRuntime { } else if quoted { Ok(Value::string(path, span)) } else { - let cwd = engine_state.cwd(Some(stack))?; + let cwd = engine_state.cwd(Some(stack)).unwrap_or_default(); let path = expand_path_with(path, cwd, true); Ok(Value::string(path.to_string_lossy(), span)) @@ -677,7 +712,7 @@ impl Eval for EvalRuntime { _: Span, ) -> Result { // FIXME: protect this collect with ctrl-c - Ok(eval_call::(engine_state, stack, call, PipelineData::empty())?.into_value(call.head)) + eval_call::(engine_state, stack, call, PipelineData::empty())?.into_value(call.head) } fn eval_external_call( @@ -689,7 +724,7 @@ impl Eval for EvalRuntime { ) -> Result { let span = head.span; // FIXME: protect this collect with ctrl-c - Ok(eval_external(engine_state, stack, head, args, PipelineData::empty())?.into_value(span)) + eval_external(engine_state, stack, head, args, PipelineData::empty())?.into_value(span) } fn eval_subexpression( @@ -699,12 +734,8 @@ impl Eval for EvalRuntime { span: Span, ) -> Result { let block = engine_state.get_block(block_id); - // FIXME: protect this collect with ctrl-c - Ok( - eval_subexpression::(engine_state, stack, block, PipelineData::empty())? - .into_value(span), - ) + eval_subexpression::(engine_state, stack, block, PipelineData::empty())?.into_value(span) } fn regex_match( diff --git a/crates/nu-explore/src/nu_common/value.rs b/crates/nu-explore/src/nu_common/value.rs index fde4394dac..9f9a1ce3fe 100644 --- a/crates/nu-explore/src/nu_common/value.rs +++ b/crates/nu-explore/src/nu_common/value.rs @@ -1,7 +1,7 @@ use super::NuSpan; use anyhow::Result; use nu_engine::get_columns; -use nu_protocol::{record, ListStream, PipelineData, PipelineMetadata, RawStream, Value}; +use nu_protocol::{record, ByteStream, ListStream, PipelineData, PipelineMetadata, Value}; use std::collections::HashMap; pub fn collect_pipeline(input: PipelineData) -> Result<(Vec, Vec>)> { @@ -9,16 +9,7 @@ pub fn collect_pipeline(input: PipelineData) -> Result<(Vec, Vec Ok((vec![], vec![])), PipelineData::Value(value, ..) => collect_input(value), PipelineData::ListStream(stream, ..) => Ok(collect_list_stream(stream)), - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - metadata, - span, - .. - } => Ok(collect_external_stream( - stdout, stderr, exit_code, metadata, span, - )), + PipelineData::ByteStream(stream, metadata) => Ok(collect_byte_stream(stream, metadata)), } } @@ -42,49 +33,60 @@ fn collect_list_stream(stream: ListStream) -> (Vec, Vec>) { (cols, data) } -fn collect_external_stream( - stdout: Option, - stderr: Option, - exit_code: Option, +fn collect_byte_stream( + stream: ByteStream, metadata: Option, - span: NuSpan, ) -> (Vec, Vec>) { + let span = stream.span(); + let mut columns = vec![]; let mut data = vec![]; - if let Some(stdout) = stdout { - let value = stdout.into_string().map_or_else( - |error| Value::error(error, span), - |string| Value::string(string.item, span), - ); - columns.push(String::from("stdout")); - data.push(value); - } - if let Some(stderr) = stderr { - let value = stderr.into_string().map_or_else( - |error| Value::error(error, span), - |string| Value::string(string.item, span), - ); + match stream.into_child() { + Ok(child) => match child.wait_with_output() { + Ok(output) => { + let exit_code = output.exit_status.code(); + if let Some(stdout) = output.stdout { + columns.push(String::from("stdout")); + data.push(string_or_binary(stdout, span)); + } + if let Some(stderr) = output.stderr { + columns.push(String::from("stderr")); + data.push(string_or_binary(stderr, span)); + } + columns.push(String::from("exit_code")); + data.push(Value::int(exit_code.into(), span)); + } + Err(err) => { + columns.push("".into()); + data.push(Value::error(err, span)); + } + }, + Err(stream) => { + let value = stream + .into_value() + .unwrap_or_else(|err| Value::error(err, span)); - columns.push(String::from("stderr")); - data.push(value); + columns.push("".into()); + data.push(value); + } } - if let Some(exit_code) = exit_code { - let list = exit_code.into_iter().collect::>(); - let val = Value::list(list, span); - columns.push(String::from("exit_code")); - data.push(val); - } if metadata.is_some() { let val = Value::record(record! { "data_source" => Value::string("ls", span) }, span); - columns.push(String::from("metadata")); data.push(val); } (columns, vec![data]) } +fn string_or_binary(bytes: Vec, span: NuSpan) -> Value { + match String::from_utf8(bytes) { + Ok(str) => Value::string(str, span), + Err(err) => Value::binary(err.into_bytes(), span), + } +} + /// Try to build column names and a table grid. pub fn collect_input(value: Value) -> Result<(Vec, Vec>)> { let span = value.span(); diff --git a/crates/nu-explore/src/views/record/mod.rs b/crates/nu-explore/src/views/record/mod.rs index 6534ba3589..0ed8e3e761 100644 --- a/crates/nu-explore/src/views/record/mod.rs +++ b/crates/nu-explore/src/views/record/mod.rs @@ -20,7 +20,7 @@ use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; use nu_color_config::{get_color_map, StyleComputer}; use nu_protocol::{ engine::{EngineState, Stack}, - Record, Value, + Record, Span, Value, }; use ratatui::{layout::Rect, widgets::Block}; use std::{borrow::Cow, collections::HashMap}; @@ -180,7 +180,11 @@ impl<'a> RecordView<'a> { Orientation::Left => (column, row), }; - layer.records[row][column].clone() + if layer.records.len() > row && layer.records[row].len() > column { + layer.records[row][column].clone() + } else { + Value::nothing(Span::unknown()) + } } fn create_tablew(&'a self, cfg: ViewConfig<'a>) -> TableW<'a> { diff --git a/crates/nu-lsp/src/diagnostics.rs b/crates/nu-lsp/src/diagnostics.rs index a9bbab77a9..423fffbf6c 100644 --- a/crates/nu-lsp/src/diagnostics.rs +++ b/crates/nu-lsp/src/diagnostics.rs @@ -7,8 +7,7 @@ use miette::{IntoDiagnostic, Result}; use nu_parser::parse; use nu_protocol::{ engine::{EngineState, StateWorkingSet}, - eval_const::create_nu_constant, - Span, Value, NU_VARIABLE_ID, + Value, }; impl LanguageServer { @@ -19,11 +18,7 @@ impl LanguageServer { ) -> Result<()> { let cwd = std::env::current_dir().expect("Could not get current working directory."); engine_state.add_env_var("PWD".into(), Value::test_string(cwd.to_string_lossy())); - - let Ok(nu_const) = create_nu_constant(engine_state, Span::unknown()) else { - return Ok(()); - }; - engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const); + engine_state.generate_nu_constant(); let mut working_set = StateWorkingSet::new(engine_state); diff --git a/crates/nu-lsp/src/lib.rs b/crates/nu-lsp/src/lib.rs index 939bb0e4e0..47535d9bd4 100644 --- a/crates/nu-lsp/src/lib.rs +++ b/crates/nu-lsp/src/lib.rs @@ -552,8 +552,8 @@ impl LanguageServer { ¶ms.text_document_position.text_document.uri, )?; - let stack = Stack::new(); - let mut completer = NuCompleter::new(Arc::new(engine_state.clone()), stack); + let mut completer = + NuCompleter::new(Arc::new(engine_state.clone()), Arc::new(Stack::new())); let location = Self::lsp_position_to_location(¶ms.text_document_position.position, rope_of_file); diff --git a/crates/nu-parser/src/flatten.rs b/crates/nu-parser/src/flatten.rs index e70a48e9f1..92b424783a 100644 --- a/crates/nu-parser/src/flatten.rs +++ b/crates/nu-parser/src/flatten.rs @@ -180,12 +180,12 @@ fn flatten_expression_into( flatten_expression_into(working_set, op, output); flatten_expression_into(working_set, rhs, output); } - Expr::UnaryNot(expr) => { + Expr::UnaryNot(not) => { output.push(( Span::new(expr.span.start, expr.span.start + 3), FlatShape::Operator, )); - flatten_expression_into(working_set, expr, output); + flatten_expression_into(working_set, not, output); } Expr::Closure(block_id) => { let outer_span = expr.span; diff --git a/crates/nu-parser/src/lex.rs b/crates/nu-parser/src/lex.rs index f9e75d13f0..4639cae25d 100644 --- a/crates/nu-parser/src/lex.rs +++ b/crates/nu-parser/src/lex.rs @@ -129,7 +129,7 @@ pub fn lex_item( }, Some(ParseError::UnexpectedEof( (start as char).to_string(), - Span::new(span.end, span.end), + Span::new(span.end - 1, span.end), )), ); } @@ -247,21 +247,6 @@ pub fn lex_item( let span = Span::new(span_offset + token_start, span_offset + *curr_offset); - // If there is still unclosed opening delimiters, remember they were missing - if let Some(block) = block_level.last() { - let delim = block.closing(); - let cause = - ParseError::UnexpectedEof((delim as char).to_string(), Span::new(span.end, span.end)); - - return ( - Token { - contents: TokenContents::Item, - span, - }, - Some(cause), - ); - } - if let Some(delim) = quote_start { // The non-lite parse trims quotes on both sides, so we add the expected quote so that // anyone wanting to consume this partial parse (e.g., completions) will be able to get @@ -273,11 +258,28 @@ pub fn lex_item( }, Some(ParseError::UnexpectedEof( (delim as char).to_string(), - Span::new(span.end, span.end), + Span::new(span.end - 1, span.end), )), ); } + // If there is still unclosed opening delimiters, remember they were missing + if let Some(block) = block_level.last() { + let delim = block.closing(); + let cause = ParseError::UnexpectedEof( + (delim as char).to_string(), + Span::new(span.end - 1, span.end), + ); + + return ( + Token { + contents: TokenContents::Item, + span, + }, + Some(cause), + ); + } + // If we didn't accumulate any characters, it's an unexpected error. if *curr_offset - token_start == 0 { return ( @@ -395,9 +397,11 @@ fn lex_raw_string( *curr_offset += 1 } if !matches { + let mut expected = '\''.to_string(); + expected.push_str(&"#".repeat(prefix_sharp_cnt)); return Err(ParseError::UnexpectedEof( - "#".to_string(), - Span::new(span_offset + *curr_offset, span_offset + *curr_offset), + expected, + Span::new(span_offset + *curr_offset - 1, span_offset + *curr_offset), )); } Ok(()) diff --git a/crates/nu-parser/src/parse_keywords.rs b/crates/nu-parser/src/parse_keywords.rs index 381a86d19e..015873e69a 100644 --- a/crates/nu-parser/src/parse_keywords.rs +++ b/crates/nu-parser/src/parse_keywords.rs @@ -15,8 +15,8 @@ use nu_protocol::{ }, engine::{StateWorkingSet, DEFAULT_OVERLAY_NAME}, eval_const::eval_constant, - span, Alias, BlockId, DeclId, Module, ModuleId, ParseError, PositionalArg, - ResolvedImportPattern, Span, Spanned, SyntaxShape, Type, Value, VarId, + Alias, BlockId, DeclId, Module, ModuleId, ParseError, PositionalArg, ResolvedImportPattern, + Span, Spanned, SyntaxShape, Type, Value, VarId, }; use std::{ collections::{HashMap, HashSet}, @@ -77,14 +77,14 @@ pub const UNALIASABLE_PARSER_KEYWORDS: &[&[u8]] = &[ /// Check whether spans start with a parser keyword that can be aliased pub fn is_unaliasable_parser_keyword(working_set: &StateWorkingSet, spans: &[Span]) -> bool { // try two words - if let (Some(span1), Some(span2)) = (spans.first(), spans.get(1)) { - let cmd_name = working_set.get_span_contents(span(&[*span1, *span2])); + if let (Some(&span1), Some(&span2)) = (spans.first(), spans.get(1)) { + let cmd_name = working_set.get_span_contents(Span::append(span1, span2)); return UNALIASABLE_PARSER_KEYWORDS.contains(&cmd_name); } // try one word - if let Some(span1) = spans.first() { - let cmd_name = working_set.get_span_contents(*span1); + if let Some(&span1) = spans.first() { + let cmd_name = working_set.get_span_contents(span1); UNALIASABLE_PARSER_KEYWORDS.contains(&cmd_name) } else { false @@ -254,7 +254,7 @@ pub fn parse_for(working_set: &mut StateWorkingSet, lite_command: &LiteCommand) if working_set.get_span_contents(spans[0]) != b"for" { working_set.error(ParseError::UnknownState( "internal error: Wrong call name for 'for' function".into(), - span(spans), + Span::concat(spans), )); return garbage(spans[0]); } @@ -270,7 +270,7 @@ pub fn parse_for(working_set: &mut StateWorkingSet, lite_command: &LiteCommand) None => { working_set.error(ParseError::UnknownState( "internal error: for declaration not found".into(), - span(spans), + Span::concat(spans), )); return garbage(spans[0]); } @@ -281,7 +281,7 @@ pub fn parse_for(working_set: &mut StateWorkingSet, lite_command: &LiteCommand) working_set.exit_scope(); - let call_span = span(spans); + let call_span = Span::concat(spans); let decl = working_set.get_decl(decl_id); let sig = decl.signature(); @@ -395,7 +395,7 @@ pub fn parse_def( if def_call != b"def" { working_set.error(ParseError::UnknownState( "internal error: Wrong call name for def function".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), None); } @@ -411,7 +411,7 @@ pub fn parse_def( None => { working_set.error(ParseError::UnknownState( "internal error: def declaration not found".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), None); } @@ -442,8 +442,12 @@ pub fn parse_def( } let starting_error_count = working_set.parse_errors.len(); - let ParsedInternalCall { call, output } = - parse_internal_call(working_set, span(command_spans), rest_spans, decl_id); + let ParsedInternalCall { call, output } = parse_internal_call( + working_set, + Span::concat(command_spans), + rest_spans, + decl_id, + ); // This is to preserve the order of the errors so that // the check errors below come first let mut new_errors = working_set.parse_errors[starting_error_count..].to_vec(); @@ -451,7 +455,7 @@ pub fn parse_def( working_set.exit_scope(); - let call_span = span(spans); + let call_span = Span::concat(spans); let decl = working_set.get_decl(decl_id); let sig = decl.signature(); @@ -673,7 +677,7 @@ pub fn parse_extern( if extern_call != b"extern" { working_set.error(ParseError::UnknownState( "internal error: Wrong call name for extern command".into(), - span(spans), + Span::concat(spans), )); return garbage_pipeline(spans); } @@ -689,7 +693,7 @@ pub fn parse_extern( None => { working_set.error(ParseError::UnknownState( "internal error: def declaration not found".into(), - span(spans), + Span::concat(spans), )); return garbage_pipeline(spans); } @@ -709,11 +713,15 @@ pub fn parse_extern( } } - let ParsedInternalCall { call, .. } = - parse_internal_call(working_set, span(command_spans), rest_spans, decl_id); + let ParsedInternalCall { call, .. } = parse_internal_call( + working_set, + Span::concat(command_spans), + rest_spans, + decl_id, + ); working_set.exit_scope(); - let call_span = span(spans); + let call_span = Span::concat(spans); //let decl = working_set.get_decl(decl_id); //let sig = decl.signature(); @@ -824,8 +832,9 @@ fn check_alias_name<'a>(working_set: &mut StateWorkingSet, spans: &'a [Span]) -> None } else if spans.len() < command_len + 3 { if working_set.get_span_contents(spans[command_len]) == b"=" { - let name = - String::from_utf8_lossy(working_set.get_span_contents(span(&spans[..command_len]))); + let name = String::from_utf8_lossy( + working_set.get_span_contents(Span::concat(&spans[..command_len])), + ); working_set.error(ParseError::AssignmentMismatch( format!("{name} missing name"), "missing name".into(), @@ -836,8 +845,9 @@ fn check_alias_name<'a>(working_set: &mut StateWorkingSet, spans: &'a [Span]) -> None } } else if working_set.get_span_contents(spans[command_len + 1]) != b"=" { - let name = - String::from_utf8_lossy(working_set.get_span_contents(span(&spans[..command_len]))); + let name = String::from_utf8_lossy( + working_set.get_span_contents(Span::concat(&spans[..command_len])), + ); working_set.error(ParseError::AssignmentMismatch( format!("{name} missing sign"), "missing equal sign".into(), @@ -868,7 +878,7 @@ pub fn parse_alias( if name != b"alias" { working_set.error(ParseError::InternalError( "Alias statement unparsable".into(), - span(spans), + Span::concat(spans), )); return garbage_pipeline(spans); } @@ -890,7 +900,12 @@ pub fn parse_alias( call: alias_call, output, .. - } = parse_internal_call(working_set, span(command_spans), rest_spans, decl_id); + } = parse_internal_call( + working_set, + Span::concat(command_spans), + rest_spans, + decl_id, + ); working_set .parse_errors @@ -902,7 +917,7 @@ pub fn parse_alias( let alias_pipeline = Pipeline::from_vec(vec![Expression { expr: Expr::Call(alias_call.clone()), - span: span(spans), + span: Span::concat(spans), ty: output, custom_completion: None, }]); @@ -914,7 +929,7 @@ pub fn parse_alias( let Some(alias_name_expr) = alias_call.positional_nth(0) else { working_set.error(ParseError::UnknownState( "Missing positional after call check".to_string(), - span(spans), + Span::concat(spans), )); return garbage_pipeline(spans); }; @@ -1090,7 +1105,7 @@ pub fn parse_alias( } else if spans.len() < 4 { working_set.error(ParseError::IncorrectValue( "Incomplete alias".into(), - span(&spans[..split_id]), + Span::concat(&spans[..split_id]), "incomplete alias".into(), )); } @@ -1100,7 +1115,7 @@ pub fn parse_alias( working_set.error(ParseError::InternalError( "Alias statement unparsable".into(), - span(spans), + Span::concat(spans), )); garbage_pipeline(spans) @@ -1111,7 +1126,7 @@ pub fn parse_export_in_block( working_set: &mut StateWorkingSet, lite_command: &LiteCommand, ) -> Pipeline { - let call_span = span(&lite_command.parts); + let call_span = Span::concat(&lite_command.parts); let full_name = if lite_command.parts.len() > 1 { let sub = working_set.get_span_contents(lite_command.parts[1]); @@ -1139,7 +1154,7 @@ pub fn parse_export_in_block( if full_name == "export" { lite_command.parts[0] } else { - span(&lite_command.parts[0..2]) + Span::concat(&lite_command.parts[0..2]) }, if full_name == "export" { &lite_command.parts[1..] @@ -1169,7 +1184,7 @@ pub fn parse_export_in_block( } else { working_set.error(ParseError::UnknownState( format!("internal error: '{full_name}' declaration not found",), - span(&lite_command.parts), + Span::concat(&lite_command.parts), )); return garbage_pipeline(&lite_command.parts); }; @@ -1213,7 +1228,7 @@ pub fn parse_export_in_module( if working_set.get_span_contents(*sp) != b"export" { working_set.error(ParseError::UnknownState( "expected export statement".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), vec![]); } @@ -1222,7 +1237,7 @@ pub fn parse_export_in_module( } else { working_set.error(ParseError::UnknownState( "got empty input for parsing export statement".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), vec![]); }; @@ -1280,12 +1295,12 @@ pub fn parse_export_in_module( if let Some(Expr::Call(def_call)) = pipeline.elements.first().map(|e| &e.expr.expr) { call.clone_from(def_call); - call.head = span(&spans[0..=1]); + call.head = Span::concat(&spans[0..=1]); call.decl_id = export_def_decl_id; } else { working_set.error(ParseError::InternalError( "unexpected output from parsing a definition".into(), - span(&spans[1..]), + Span::concat(&spans[1..]), )); }; @@ -1316,12 +1331,12 @@ pub fn parse_export_in_module( if let Some(Expr::Call(def_call)) = pipeline.elements.first().map(|e| &e.expr.expr) { call.clone_from(def_call); - call.head = span(&spans[0..=1]); + call.head = Span::concat(&spans[0..=1]); call.decl_id = export_def_decl_id; } else { working_set.error(ParseError::InternalError( "unexpected output from parsing a definition".into(), - span(&spans[1..]), + Span::concat(&spans[1..]), )); }; @@ -1341,7 +1356,7 @@ pub fn parse_export_in_module( } else { working_set.error(ParseError::InternalError( "failed to find added declaration".into(), - span(&spans[1..]), + Span::concat(&spans[1..]), )); } @@ -1373,12 +1388,12 @@ pub fn parse_export_in_module( { call.clone_from(alias_call); - call.head = span(&spans[0..=1]); + call.head = Span::concat(&spans[0..=1]); call.decl_id = export_alias_decl_id; } else { working_set.error(ParseError::InternalError( "unexpected output from parsing a definition".into(), - span(&spans[1..]), + Span::concat(&spans[1..]), )); }; @@ -1398,7 +1413,7 @@ pub fn parse_export_in_module( } else { working_set.error(ParseError::InternalError( "failed to find added alias".into(), - span(&spans[1..]), + Span::concat(&spans[1..]), )); } @@ -1428,12 +1443,12 @@ pub fn parse_export_in_module( { call.clone_from(use_call); - call.head = span(&spans[0..=1]); + call.head = Span::concat(&spans[0..=1]); call.decl_id = export_use_decl_id; } else { working_set.error(ParseError::InternalError( "unexpected output from parsing a definition".into(), - span(&spans[1..]), + Span::concat(&spans[1..]), )); }; @@ -1460,12 +1475,12 @@ pub fn parse_export_in_module( { call.clone_from(module_call); - call.head = span(&spans[0..=1]); + call.head = Span::concat(&spans[0..=1]); call.decl_id = export_module_decl_id; } else { working_set.error(ParseError::InternalError( "unexpected output from parsing a definition".into(), - span(&spans[1..]), + Span::concat(&spans[1..]), )); }; @@ -1486,7 +1501,7 @@ pub fn parse_export_in_module( "failed to find added module '{}'", String::from_utf8_lossy(module_name) ), - span(&spans[1..]), + Span::concat(&spans[1..]), )); } } @@ -1511,12 +1526,12 @@ pub fn parse_export_in_module( { call.clone_from(def_call); - call.head = span(&spans[0..=1]); + call.head = Span::concat(&spans[0..=1]); call.decl_id = export_const_decl_id; } else { working_set.error(ParseError::InternalError( "unexpected output from parsing a definition".into(), - span(&spans[1..]), + Span::concat(&spans[1..]), )); }; @@ -1538,7 +1553,7 @@ pub fn parse_export_in_module( } else { working_set.error(ParseError::InternalError( "failed to find added variable".into(), - span(&spans[1..]), + Span::concat(&spans[1..]), )); } } @@ -1567,7 +1582,7 @@ pub fn parse_export_in_module( ( Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, }]), @@ -1582,7 +1597,7 @@ pub fn parse_export_env( if !spans.is_empty() && working_set.get_span_contents(spans[0]) != b"export-env" { working_set.error(ParseError::UnknownState( "internal error: Wrong call name for 'export-env' command".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), None); } @@ -1590,7 +1605,7 @@ pub fn parse_export_env( if spans.len() < 2 { working_set.error(ParseError::MissingPositional( "block".into(), - span(spans), + Span::concat(spans), "export-env ".into(), )); return (garbage_pipeline(spans), None); @@ -1602,7 +1617,7 @@ pub fn parse_export_env( parse_internal_call(working_set, spans[0], &[spans[1]], decl_id); let decl = working_set.get_decl(decl_id); - let call_span = span(spans); + let call_span = Span::concat(spans); let starting_error_count = working_set.parse_errors.len(); check_call(working_set, call_span, &decl.signature(), &call); @@ -1628,7 +1643,7 @@ pub fn parse_export_env( None => { working_set.error(ParseError::UnknownState( "internal error: 'export-env' declaration not found".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), None); } @@ -1647,14 +1662,14 @@ pub fn parse_export_env( } else { working_set.error(ParseError::UnknownState( "internal error: 'export-env' block is missing".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), None); }; let pipeline = Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, }]); @@ -2050,11 +2065,15 @@ pub fn parse_module( Some(decl_id) => { let (command_spans, rest_spans) = spans.split_at(split_id); - let ParsedInternalCall { call, output } = - parse_internal_call(working_set, span(command_spans), rest_spans, decl_id); + let ParsedInternalCall { call, output } = parse_internal_call( + working_set, + Span::concat(command_spans), + rest_spans, + decl_id, + ); let decl = working_set.get_decl(decl_id); - let call_span = span(spans); + let call_span = Span::concat(spans); let starting_error_count = working_set.parse_errors.len(); check_call(working_set, call_span, &decl.signature(), &call); @@ -2080,7 +2099,7 @@ pub fn parse_module( None => { working_set.error(ParseError::UnknownState( "internal error: 'module' or 'export module' declaration not found".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), None); } @@ -2112,14 +2131,14 @@ pub fn parse_module( } else { working_set.error(ParseError::UnknownState( "internal error: name not a string".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), None); } } else { working_set.error(ParseError::UnknownState( "internal error: missing positional".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), None); }; @@ -2151,7 +2170,7 @@ pub fn parse_module( if spans.len() < split_id + 2 { working_set.error(ParseError::UnknownState( "Expected structure: module or module ".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), None); @@ -2199,7 +2218,7 @@ pub fn parse_module( .expect("internal error: missing module command"); let call = Box::new(Call { - head: span(&spans[..split_id]), + head: Span::concat(&spans[..split_id]), decl_id: module_decl_id, arguments: vec![ Argument::Positional(module_name_or_path_expr), @@ -2211,7 +2230,7 @@ pub fn parse_module( ( Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, }]), @@ -2236,7 +2255,7 @@ pub fn parse_use( if use_call != b"use" { working_set.error(ParseError::UnknownState( "internal error: Wrong call name for 'use' command".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), vec![]); } @@ -2244,7 +2263,7 @@ pub fn parse_use( if working_set.get_span_contents(name_span) != b"use" { working_set.error(ParseError::UnknownState( "internal error: Wrong call name for 'use' command".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), vec![]); } @@ -2258,11 +2277,15 @@ pub fn parse_use( Some(decl_id) => { let (command_spans, rest_spans) = spans.split_at(split_id); - let ParsedInternalCall { call, output } = - parse_internal_call(working_set, span(command_spans), rest_spans, decl_id); + let ParsedInternalCall { call, output } = parse_internal_call( + working_set, + Span::concat(command_spans), + rest_spans, + decl_id, + ); let decl = working_set.get_decl(decl_id); - let call_span = span(spans); + let call_span = Span::concat(spans); let starting_error_count = working_set.parse_errors.len(); check_call(working_set, call_span, &decl.signature(), &call); @@ -2288,7 +2311,7 @@ pub fn parse_use( None => { working_set.error(ParseError::UnknownState( "internal error: 'use' declaration not found".into(), - span(spans), + Span::concat(spans), )); return (garbage_pipeline(spans), vec![]); } @@ -2418,7 +2441,7 @@ pub fn parse_use( // Create a new Use command call to pass the import pattern as parser info let import_pattern_expr = Expression { expr: Expr::ImportPattern(Box::new(import_pattern)), - span: span(args_spans), + span: Span::concat(args_spans), ty: Type::Any, custom_completion: None, }; @@ -2429,7 +2452,7 @@ pub fn parse_use( ( Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, }]), @@ -2443,7 +2466,7 @@ pub fn parse_hide(working_set: &mut StateWorkingSet, lite_command: &LiteCommand) if working_set.get_span_contents(spans[0]) != b"hide" { working_set.error(ParseError::UnknownState( "internal error: Wrong call name for 'hide' command".into(), - span(spans), + Span::concat(spans), )); return garbage_pipeline(spans); } @@ -2458,7 +2481,7 @@ pub fn parse_hide(working_set: &mut StateWorkingSet, lite_command: &LiteCommand) parse_internal_call(working_set, spans[0], &spans[1..], decl_id); let decl = working_set.get_decl(decl_id); - let call_span = span(spans); + let call_span = Span::concat(spans); let starting_error_count = working_set.parse_errors.len(); check_call(working_set, call_span, &decl.signature(), &call); @@ -2481,7 +2504,7 @@ pub fn parse_hide(working_set: &mut StateWorkingSet, lite_command: &LiteCommand) None => { working_set.error(ParseError::UnknownState( "internal error: 'hide' declaration not found".into(), - span(spans), + Span::concat(spans), )); return garbage_pipeline(spans); } @@ -2602,7 +2625,7 @@ pub fn parse_hide(working_set: &mut StateWorkingSet, lite_command: &LiteCommand) // Create a new Use command call to pass the new import pattern let import_pattern_expr = Expression { expr: Expr::ImportPattern(Box::new(import_pattern)), - span: span(args_spans), + span: Span::concat(args_spans), ty: Type::Any, custom_completion: None, }; @@ -2612,14 +2635,14 @@ pub fn parse_hide(working_set: &mut StateWorkingSet, lite_command: &LiteCommand) Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, }]) } else { working_set.error(ParseError::UnknownState( "Expected structure: hide ".into(), - span(spans), + Span::concat(spans), )); garbage_pipeline(spans) } @@ -2975,7 +2998,7 @@ pub fn parse_let(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline // let x = 'f', = at least start from index 2 if item == b"=" && spans.len() > (span.0 + 1) && span.0 > 1 { let (tokens, parse_error) = lex( - working_set.get_span_contents(nu_protocol::span(&spans[(span.0 + 1)..])), + working_set.get_span_contents(Span::concat(&spans[(span.0 + 1)..])), spans[span.0 + 1].start, &[], &[], @@ -2986,7 +3009,7 @@ pub fn parse_let(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline working_set.error(parse_error) } - let rvalue_span = nu_protocol::span(&spans[(span.0 + 1)..]); + let rvalue_span = Span::concat(&spans[(span.0 + 1)..]); let rvalue_block = parse_block(working_set, &tokens, rvalue_span, false, true); let output_type = rvalue_block.output_type(); @@ -3025,7 +3048,7 @@ pub fn parse_let(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline working_set.error(ParseError::TypeMismatch( explicit_type.clone(), rhs_type.clone(), - nu_protocol::span(&spans[(span.0 + 1)..]), + Span::concat(&spans[(span.0 + 1)..]), )); } } @@ -3045,7 +3068,7 @@ pub fn parse_let(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: nu_protocol::span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, }]); @@ -3057,20 +3080,20 @@ pub fn parse_let(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: nu_protocol::span(spans), + span: Span::concat(spans), ty: output, custom_completion: None, }]); } else { working_set.error(ParseError::UnknownState( "internal error: let or const statements not found in core language".into(), - span(spans), + Span::concat(spans), )) } working_set.error(ParseError::UnknownState( "internal error: let or const statement unparsable".into(), - span(spans), + Span::concat(spans), )); garbage_pipeline(spans) @@ -3134,7 +3157,7 @@ pub fn parse_const(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipelin working_set.error(ParseError::TypeMismatch( explicit_type.clone(), rhs_type.clone(), - nu_protocol::span(&spans[(span.0 + 1)..]), + Span::concat(&spans[(span.0 + 1)..]), )); } } @@ -3155,7 +3178,7 @@ pub fn parse_const(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipelin working_set.error(ParseError::TypeMismatch( explicit_type.clone(), const_type.clone(), - nu_protocol::span(&spans[(span.0 + 1)..]), + Span::concat(&spans[(span.0 + 1)..]), )); } let val_span = value.span(); @@ -3191,7 +3214,7 @@ pub fn parse_const(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipelin return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: nu_protocol::span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, }]); @@ -3203,20 +3226,20 @@ pub fn parse_const(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipelin return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: nu_protocol::span(spans), + span: Span::concat(spans), ty: output, custom_completion: None, }]); } else { working_set.error(ParseError::UnknownState( "internal error: let or const statements not found in core language".into(), - span(spans), + Span::concat(spans), )) } working_set.error(ParseError::UnknownState( "internal error: let or const statement unparsable".into(), - span(spans), + Span::concat(spans), )); garbage_pipeline(spans) @@ -3239,7 +3262,7 @@ pub fn parse_mut(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline // mut x = 'f', = at least start from index 2 if item == b"=" && spans.len() > (span.0 + 1) && span.0 > 1 { let (tokens, parse_error) = lex( - working_set.get_span_contents(nu_protocol::span(&spans[(span.0 + 1)..])), + working_set.get_span_contents(Span::concat(&spans[(span.0 + 1)..])), spans[span.0 + 1].start, &[], &[], @@ -3250,7 +3273,7 @@ pub fn parse_mut(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline working_set.error(parse_error); } - let rvalue_span = nu_protocol::span(&spans[(span.0 + 1)..]); + let rvalue_span = Span::concat(&spans[(span.0 + 1)..]); let rvalue_block = parse_block(working_set, &tokens, rvalue_span, false, true); let output_type = rvalue_block.output_type(); @@ -3290,7 +3313,7 @@ pub fn parse_mut(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline working_set.error(ParseError::TypeMismatch( explicit_type.clone(), rhs_type.clone(), - nu_protocol::span(&spans[(span.0 + 1)..]), + Span::concat(&spans[(span.0 + 1)..]), )); } } @@ -3310,7 +3333,7 @@ pub fn parse_mut(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: nu_protocol::span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, }]); @@ -3322,20 +3345,20 @@ pub fn parse_mut(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: nu_protocol::span(spans), + span: Span::concat(spans), ty: output, custom_completion: None, }]); } else { working_set.error(ParseError::UnknownState( "internal error: let or const statements not found in core language".into(), - span(spans), + Span::concat(spans), )) } working_set.error(ParseError::UnknownState( "internal error: let or const statement unparsable".into(), - span(spans), + Span::concat(spans), )); garbage_pipeline(spans) @@ -3375,7 +3398,7 @@ pub fn parse_source(working_set: &mut StateWorkingSet, lite_command: &LiteComman if is_help { return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: span(spans), + span: Span::concat(spans), ty: output, custom_completion: None, }]); @@ -3388,10 +3411,10 @@ pub fn parse_source(working_set: &mut StateWorkingSet, lite_command: &LiteComman let val = match eval_constant(working_set, &expr) { Ok(val) => val, Err(err) => { - working_set.error(err.wrap(working_set, span(&spans[1..]))); + working_set.error(err.wrap(working_set, Span::concat(&spans[1..]))); return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: span(&spans[1..]), + span: Span::concat(&spans[1..]), ty: Type::Any, custom_completion: None, }]); @@ -3401,10 +3424,10 @@ pub fn parse_source(working_set: &mut StateWorkingSet, lite_command: &LiteComman let filename = match val.coerce_into_string() { Ok(s) => s, Err(err) => { - working_set.error(err.wrap(working_set, span(&spans[1..]))); + working_set.error(err.wrap(working_set, Span::concat(&spans[1..]))); return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: span(&spans[1..]), + span: Span::concat(&spans[1..]), ty: Type::Any, custom_completion: None, }]); @@ -3450,7 +3473,7 @@ pub fn parse_source(working_set: &mut StateWorkingSet, lite_command: &LiteComman return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call_with_block), - span: span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, }]); @@ -3461,7 +3484,7 @@ pub fn parse_source(working_set: &mut StateWorkingSet, lite_command: &LiteComman } return Pipeline::from_vec(vec![Expression { expr: Expr::Call(call), - span: span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, }]); @@ -3469,7 +3492,7 @@ pub fn parse_source(working_set: &mut StateWorkingSet, lite_command: &LiteComman } working_set.error(ParseError::UnknownState( "internal error: source statement unparsable".into(), - span(spans), + Span::concat(spans), )); garbage_pipeline(spans) } @@ -3480,18 +3503,18 @@ pub fn parse_where_expr(working_set: &mut StateWorkingSet, spans: &[Span]) -> Ex if !spans.is_empty() && working_set.get_span_contents(spans[0]) != b"where" { working_set.error(ParseError::UnknownState( "internal error: Wrong call name for 'where' command".into(), - span(spans), + Span::concat(spans), )); - return garbage(span(spans)); + return garbage(Span::concat(spans)); } if spans.len() < 2 { working_set.error(ParseError::MissingPositional( "row condition".into(), - span(spans), + Span::concat(spans), "where ".into(), )); - return garbage(span(spans)); + return garbage(Span::concat(spans)); } let call = match working_set.find_decl(b"where") { @@ -3500,13 +3523,13 @@ pub fn parse_where_expr(working_set: &mut StateWorkingSet, spans: &[Span]) -> Ex parse_internal_call(working_set, spans[0], &spans[1..], decl_id); let decl = working_set.get_decl(decl_id); - let call_span = span(spans); + let call_span = Span::concat(spans); let starting_error_count = working_set.parse_errors.len(); check_call(working_set, call_span, &decl.signature(), &call); let Ok(is_help) = has_flag_const(working_set, &call, "help") else { - return garbage(span(spans)); + return garbage(Span::concat(spans)); }; if starting_error_count != working_set.parse_errors.len() || is_help { @@ -3523,15 +3546,15 @@ pub fn parse_where_expr(working_set: &mut StateWorkingSet, spans: &[Span]) -> Ex None => { working_set.error(ParseError::UnknownState( "internal error: 'where' declaration not found".into(), - span(spans), + Span::concat(spans), )); - return garbage(span(spans)); + return garbage(Span::concat(spans)); } }; Expression { expr: Expr::Call(call), - span: span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, } @@ -3574,7 +3597,7 @@ pub fn parse_register(working_set: &mut StateWorkingSet, lite_command: &LiteComm if working_set.get_span_contents(spans[0]) != b"register" { working_set.error(ParseError::UnknownState( "internal error: Wrong call name for 'register' function".into(), - span(spans), + Span::concat(spans), )); return garbage_pipeline(spans); } @@ -3590,7 +3613,7 @@ pub fn parse_register(working_set: &mut StateWorkingSet, lite_command: &LiteComm None => { working_set.error(ParseError::UnknownState( "internal error: Register declaration not found".into(), - span(spans), + Span::concat(spans), )); return garbage_pipeline(spans); } @@ -3599,7 +3622,7 @@ pub fn parse_register(working_set: &mut StateWorkingSet, lite_command: &LiteComm parse_internal_call(working_set, spans[0], &spans[1..], decl_id); let decl = working_set.get_decl(decl_id); - let call_span = span(spans); + let call_span = Span::concat(spans); let starting_error_count = working_set.parse_errors.len(); check_call(working_set, call_span, &decl.signature(), &call); diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 7dfccf7bad..bd09f5b52a 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -11,7 +11,7 @@ use itertools::Itertools; use log::trace; use nu_engine::DIR_VAR_PARSER_INFO; use nu_protocol::{ - ast::*, engine::StateWorkingSet, eval_const::eval_constant, span, BlockId, DidYouMean, Flag, + ast::*, engine::StateWorkingSet, eval_const::eval_constant, BlockId, DidYouMean, Flag, ParseError, PositionalArg, Signature, Span, Spanned, SyntaxShape, Type, VarId, ENV_VARIABLE_ID, IN_VARIABLE_ID, }; @@ -27,7 +27,7 @@ pub fn garbage(span: Span) -> Expression { } pub fn garbage_pipeline(spans: &[Span]) -> Pipeline { - Pipeline::from_vec(vec![garbage(span(spans))]) + Pipeline::from_vec(vec![garbage(Span::concat(spans))]) } fn is_identifier_byte(b: u8) -> bool { @@ -228,6 +228,8 @@ fn parse_external_arg(working_set: &mut StateWorkingSet, span: Span) -> External ExternalArgument::Regular(parse_dollar_expr(working_set, span)) } else if contents.starts_with(b"[") { ExternalArgument::Regular(parse_list_expression(working_set, span, &SyntaxShape::Any)) + } else if contents.starts_with(b"r#") { + ExternalArgument::Regular(parse_raw_string(working_set, span)) } else if contents.len() > 3 && contents.starts_with(b"...") && (contents[3] == b'$' || contents[3] == b'[' || contents[3] == b'(') @@ -296,7 +298,7 @@ pub fn parse_external_call(working_set: &mut StateWorkingSet, spans: &[Span]) -> Expression { expr: Expr::ExternalCall(head, args), - span: span(spans), + span: Span::concat(spans), ty: Type::Any, custom_completion: None, } @@ -484,7 +486,7 @@ fn parse_short_flags( && matches!( sig.get_positional(positional_idx), Some(PositionalArg { - shape: SyntaxShape::Int | SyntaxShape::Number, + shape: SyntaxShape::Int | SyntaxShape::Number | SyntaxShape::Float, .. }) ) @@ -1055,7 +1057,7 @@ pub fn parse_call(working_set: &mut StateWorkingSet, spans: &[Span], head: Span) if spans.is_empty() { working_set.error(ParseError::UnknownState( "Encountered command with zero spans".into(), - span(spans), + Span::concat(spans), )); return garbage(head); } @@ -1117,9 +1119,9 @@ pub fn parse_call(working_set: &mut StateWorkingSet, spans: &[Span], head: Span) working_set.error(ParseError::UnknownState( "Incomplete statement".into(), - span(spans), + Span::concat(spans), )); - return garbage(span(spans)); + return garbage(Span::concat(spans)); } } @@ -1147,7 +1149,7 @@ pub fn parse_call(working_set: &mut StateWorkingSet, spans: &[Span], head: Span) return Expression { expr: Expr::ExternalCall(head, final_args.into()), - span: span(spans), + span: Span::concat(spans), ty: ty.clone(), custom_completion: *custom_completion, }; @@ -1155,7 +1157,7 @@ pub fn parse_call(working_set: &mut StateWorkingSet, spans: &[Span], head: Span) trace!("parsing: alias of internal call"); parse_internal_call( working_set, - span(&spans[cmd_start..pos]), + Span::concat(&spans[cmd_start..pos]), &spans[pos..], decl_id, ) @@ -1164,7 +1166,7 @@ pub fn parse_call(working_set: &mut StateWorkingSet, spans: &[Span], head: Span) trace!("parsing: internal call"); parse_internal_call( working_set, - span(&spans[cmd_start..pos]), + Span::concat(&spans[cmd_start..pos]), &spans[pos..], decl_id, ) @@ -1172,7 +1174,7 @@ pub fn parse_call(working_set: &mut StateWorkingSet, spans: &[Span], head: Span) Expression { expr: Expr::Call(parsed_call.call), - span: span(spans), + span: Span::concat(spans), ty: parsed_call.output, custom_completion: None, } @@ -2795,9 +2797,9 @@ pub fn parse_import_pattern(working_set: &mut StateWorkingSet, spans: &[Span]) - let Some(head_span) = spans.first() else { working_set.error(ParseError::WrongImportPattern( "needs at least one component of import pattern".to_string(), - span(spans), + Span::concat(spans), )); - return garbage(span(spans)); + return garbage(Span::concat(spans)); }; let head_expr = parse_value(working_set, *head_span, &SyntaxShape::Any); @@ -2806,13 +2808,13 @@ pub fn parse_import_pattern(working_set: &mut StateWorkingSet, spans: &[Span]) - Ok(val) => match val.coerce_into_string() { Ok(s) => (working_set.find_module(s.as_bytes()), s.into_bytes()), Err(err) => { - working_set.error(err.wrap(working_set, span(spans))); - return garbage(span(spans)); + working_set.error(err.wrap(working_set, Span::concat(spans))); + return garbage(Span::concat(spans)); } }, Err(err) => { - working_set.error(err.wrap(working_set, span(spans))); - return garbage(span(spans)); + working_set.error(err.wrap(working_set, Span::concat(spans))); + return garbage(Span::concat(spans)); } }; @@ -2892,7 +2894,7 @@ pub fn parse_import_pattern(working_set: &mut StateWorkingSet, spans: &[Span]) - working_set.error(ParseError::ExportNotFound(result.span)); return Expression { expr: Expr::ImportPattern(Box::new(import_pattern)), - span: span(spans), + span: Span::concat(spans), ty: Type::List(Box::new(Type::String)), custom_completion: None, }; @@ -2912,7 +2914,7 @@ pub fn parse_import_pattern(working_set: &mut StateWorkingSet, spans: &[Span]) - Expression { expr: Expr::ImportPattern(Box::new(import_pattern)), - span: span(&spans[1..]), + span: Span::concat(&spans[1..]), ty: Type::List(Box::new(Type::String)), custom_completion: None, } @@ -2946,7 +2948,7 @@ pub fn parse_var_with_opt_type( *spans_idx += 1; // signature like record is broken into multiple spans due to // whitespaces. Collect the rest into one span and work on it - let full_span = span(&spans[*spans_idx..]); + let full_span = Span::concat(&spans[*spans_idx..]); let type_bytes = working_set.get_span_contents(full_span).to_vec(); let (tokens, parse_error) = @@ -2974,7 +2976,7 @@ pub fn parse_var_with_opt_type( ( Expression { expr: Expr::VarDecl(id), - span: span(&spans[span_beginning..*spans_idx + 1]), + span: Span::concat(&spans[span_beginning..*spans_idx + 1]), ty: ty.clone(), custom_completion: None, }, @@ -3017,7 +3019,7 @@ pub fn parse_var_with_opt_type( let id = working_set.add_variable( var_name, - span(&spans[*spans_idx..*spans_idx + 1]), + Span::concat(&spans[*spans_idx..*spans_idx + 1]), Type::Any, mutable, ); @@ -3065,7 +3067,7 @@ pub fn parse_input_output_types( working_set: &mut StateWorkingSet, spans: &[Span], ) -> Vec<(Type, Type)> { - let mut full_span = span(spans); + let mut full_span = Span::concat(spans); let mut bytes = working_set.get_span_contents(full_span); @@ -3143,7 +3145,7 @@ pub fn parse_full_signature(working_set: &mut StateWorkingSet, spans: &[Span]) - } = &mut arg_signature { sig.input_output_types = input_output_types; - expr_span.end = span(&spans[1..]).end; + expr_span.end = Span::concat(&spans[1..]).end; } arg_signature } else { @@ -3152,9 +3154,9 @@ pub fn parse_full_signature(working_set: &mut StateWorkingSet, spans: &[Span]) - } pub fn parse_row_condition(working_set: &mut StateWorkingSet, spans: &[Span]) -> Expression { - let var_id = working_set.add_variable(b"$it".to_vec(), span(spans), Type::Any, false); + let var_id = working_set.add_variable(b"$it".to_vec(), Span::concat(spans), Type::Any, false); let expression = parse_math_expression(working_set, spans, Some(var_id)); - let span = span(spans); + let span = Span::concat(spans); let block_id = match expression.expr { Expr::Block(block_id) => block_id, @@ -5058,7 +5060,7 @@ pub fn parse_math_expression( working_set.error(err); } - let op_span = span(&[lhs.span, rhs.span]); + let op_span = Span::append(lhs.span, rhs.span); expr_stack.push(Expression { expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), span: op_span, @@ -5094,7 +5096,7 @@ pub fn parse_math_expression( working_set.error(err) } - let binary_op_span = span(&[lhs.span, rhs.span]); + let binary_op_span = Span::append(lhs.span, rhs.span); expr_stack.push(Expression { expr: Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)), span: binary_op_span, @@ -5165,7 +5167,7 @@ pub fn parse_expression(working_set: &mut StateWorkingSet, spans: &[Span]) -> Ex if pos == spans.len() { working_set.error(ParseError::UnknownCommand(spans[0])); - return garbage(span(spans)); + return garbage(Span::concat(spans)); } let output = if is_math_expression_like(working_set, spans[pos]) { @@ -5260,13 +5262,13 @@ pub fn parse_expression(working_set: &mut StateWorkingSet, spans: &[Span]) -> Ex let arguments = vec![ Argument::Positional(Expression { expr: Expr::Record(env_vars), - span: span(&spans[..pos]), + span: Span::concat(&spans[..pos]), ty: Type::Any, custom_completion: None, }), Argument::Positional(Expression { expr: Expr::Closure(block_id), - span: span(&spans[pos..]), + span: Span::concat(&spans[pos..]), ty: Type::Closure, custom_completion: None, }), @@ -5282,7 +5284,7 @@ pub fn parse_expression(working_set: &mut StateWorkingSet, spans: &[Span]) -> Ex Expression { expr, custom_completion: None, - span: span(spans), + span: Span::concat(spans), ty, } } else { @@ -5634,7 +5636,7 @@ pub fn parse_pipeline( // if the 'let' is complete enough, use it, if not, fall through for now if new_command.parts.len() > 3 { - let rhs_span = nu_protocol::span(&new_command.parts[3..]); + let rhs_span = Span::concat(&new_command.parts[3..]); new_command.parts.truncate(3); new_command.parts.push(rhs_span); diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index e73f0f2e02..9b8ae94317 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -1,6 +1,6 @@ use nu_parser::*; use nu_protocol::{ - ast::{Argument, Call, Expr, PathMember, Range}, + ast::{Argument, Call, Expr, ExternalArgument, PathMember, Range}, engine::{Command, EngineState, Stack, StateWorkingSet}, ParseError, PipelineData, ShellError, Signature, Span, SyntaxShape, }; @@ -926,6 +926,28 @@ mod string { assert!(working_set.parse_errors.is_empty()); } } + + #[test] + fn parse_raw_string_as_external_argument() { + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); + + let block = parse(&mut working_set, None, b"^echo r#'text'#", true); + + assert!(working_set.parse_errors.is_empty()); + assert_eq!(block.len(), 1); + let pipeline = &block.pipelines[0]; + assert_eq!(pipeline.len(), 1); + let element = &pipeline.elements[0]; + assert!(element.redirection.is_none()); + if let Expr::ExternalCall(_, args) = &element.expr.expr { + if let [ExternalArgument::Regular(expr)] = args.as_ref() { + assert_eq!(expr.expr, Expr::RawString("text".into())); + return; + } + } + panic!("wrong expression: {:?}", element.expr.expr) + } } #[rstest] diff --git a/crates/nu-plugin-core/src/interface/mod.rs b/crates/nu-plugin-core/src/interface/mod.rs index 3fb86aee36..b4a2bc9a25 100644 --- a/crates/nu-plugin-core/src/interface/mod.rs +++ b/crates/nu-plugin-core/src/interface/mod.rs @@ -1,15 +1,10 @@ //! Implements the stream multiplexing interface for both the plugin side and the engine side. -use nu_plugin_protocol::{ - ExternalStreamInfo, ListStreamInfo, PipelineDataHeader, RawStreamInfo, StreamMessage, -}; -use nu_protocol::{ListStream, PipelineData, RawStream, ShellError}; +use nu_plugin_protocol::{ByteStreamInfo, ListStreamInfo, PipelineDataHeader, StreamMessage}; +use nu_protocol::{ByteStream, IntoSpanned, ListStream, PipelineData, Reader, ShellError}; use std::{ - io::Write, - sync::{ - atomic::{AtomicBool, Ordering::Relaxed}, - Arc, Mutex, - }, + io::{Read, Write}, + sync::{atomic::AtomicBool, Arc, Mutex}, thread, }; @@ -185,31 +180,10 @@ pub trait InterfaceManager { let reader = handle.read_stream(info.id, self.get_interface())?; ListStream::new(reader, info.span, ctrlc.cloned()).into() } - PipelineDataHeader::ExternalStream(info) => { + PipelineDataHeader::ByteStream(info) => { let handle = self.stream_manager().get_handle(); - let span = info.span; - let new_raw_stream = |raw_info: RawStreamInfo| { - let reader = handle.read_stream(raw_info.id, self.get_interface())?; - let mut stream = - RawStream::new(Box::new(reader), ctrlc.cloned(), span, raw_info.known_size); - stream.is_binary = raw_info.is_binary; - Ok::<_, ShellError>(stream) - }; - PipelineData::ExternalStream { - stdout: info.stdout.map(new_raw_stream).transpose()?, - stderr: info.stderr.map(new_raw_stream).transpose()?, - exit_code: info - .exit_code - .map(|list_info| { - handle - .read_stream(list_info.id, self.get_interface()) - .map(|reader| ListStream::new(reader, info.span, ctrlc.cloned())) - }) - .transpose()?, - span: info.span, - metadata: None, - trim_end_newline: info.trim_end_newline, - } + let reader = handle.read_stream(info.id, self.get_interface())?; + ByteStream::from_result_iter(reader, info.span, ctrlc.cloned()).into() } }) } @@ -271,11 +245,11 @@ pub trait Interface: Clone + Send { Ok::<_, ShellError>((id, writer)) }; match self.prepare_pipeline_data(data, context)? { - PipelineData::Value(value, _) => { + PipelineData::Value(value, ..) => { Ok((PipelineDataHeader::Value(value), PipelineDataWriter::None)) } PipelineData::Empty => Ok((PipelineDataHeader::Empty, PipelineDataWriter::None)), - PipelineData::ListStream(stream, _) => { + PipelineData::ListStream(stream, ..) => { let (id, writer) = new_stream(LIST_STREAM_HIGH_PRESSURE)?; Ok(( PipelineDataHeader::ListStream(ListStreamInfo { @@ -285,50 +259,15 @@ pub trait Interface: Clone + Send { PipelineDataWriter::ListStream(writer, stream), )) } - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata: _, - trim_end_newline, - } => { - // Create the writers and stream ids - let stdout_stream = stdout - .is_some() - .then(|| new_stream(RAW_STREAM_HIGH_PRESSURE)) - .transpose()?; - let stderr_stream = stderr - .is_some() - .then(|| new_stream(RAW_STREAM_HIGH_PRESSURE)) - .transpose()?; - let exit_code_stream = exit_code - .is_some() - .then(|| new_stream(LIST_STREAM_HIGH_PRESSURE)) - .transpose()?; - // Generate the header, with the stream ids - let header = PipelineDataHeader::ExternalStream(ExternalStreamInfo { - span, - stdout: stdout - .as_ref() - .zip(stdout_stream.as_ref()) - .map(|(stream, (id, _))| RawStreamInfo::new(*id, stream)), - stderr: stderr - .as_ref() - .zip(stderr_stream.as_ref()) - .map(|(stream, (id, _))| RawStreamInfo::new(*id, stream)), - exit_code: exit_code_stream - .as_ref() - .map(|&(id, _)| ListStreamInfo { id, span }), - trim_end_newline, - }); - // Collect the writers - let writer = PipelineDataWriter::ExternalStream { - stdout: stdout_stream.map(|(_, writer)| writer).zip(stdout), - stderr: stderr_stream.map(|(_, writer)| writer).zip(stderr), - exit_code: exit_code_stream.map(|(_, writer)| writer).zip(exit_code), - }; - Ok((header, writer)) + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + if let Some(reader) = stream.reader() { + let (id, writer) = new_stream(RAW_STREAM_HIGH_PRESSURE)?; + let header = PipelineDataHeader::ByteStream(ByteStreamInfo { id, span }); + Ok((header, PipelineDataWriter::ByteStream(writer, reader))) + } else { + Ok((PipelineDataHeader::Empty, PipelineDataWriter::None)) + } } } } @@ -355,11 +294,7 @@ pub enum PipelineDataWriter { #[default] None, ListStream(StreamWriter, ListStream), - ExternalStream { - stdout: Option<(StreamWriter, RawStream)>, - stderr: Option<(StreamWriter, RawStream)>, - exit_code: Option<(StreamWriter, ListStream)>, - }, + ByteStream(StreamWriter, Reader), } impl PipelineDataWriter @@ -376,49 +311,16 @@ where writer.write_all(stream)?; Ok(()) } - // Write all three possible streams of an ExternalStream on separate threads. - PipelineDataWriter::ExternalStream { - stdout, - stderr, - exit_code, - } => { - thread::scope(|scope| { - let stderr_thread = stderr - .map(|(mut writer, stream)| { - thread::Builder::new() - .name("plugin stderr writer".into()) - .spawn_scoped(scope, move || { - writer.write_all(raw_stream_iter(stream)) - }) - }) - .transpose()?; - let exit_code_thread = exit_code - .map(|(mut writer, stream)| { - thread::Builder::new() - .name("plugin exit_code writer".into()) - .spawn_scoped(scope, move || writer.write_all(stream)) - }) - .transpose()?; - // Optimize for stdout: if only stdout is present, don't spawn any other - // threads. - if let Some((mut writer, stream)) = stdout { - writer.write_all(raw_stream_iter(stream))?; - } - let panicked = |thread_name: &str| { - Err(ShellError::NushellFailed { - msg: format!( - "{thread_name} thread panicked in PipelineDataWriter::write" - ), - }) - }; - stderr_thread - .map(|t| t.join().unwrap_or_else(|_| panicked("stderr"))) - .transpose()?; - exit_code_thread - .map(|t| t.join().unwrap_or_else(|_| panicked("exit_code"))) - .transpose()?; - Ok(()) - }) + // Write a byte stream. + PipelineDataWriter::ByteStream(mut writer, mut reader) => { + let span = reader.span(); + let buf = &mut [0; 8192]; + writer.write_all(std::iter::from_fn(move || match reader.read(buf) { + Ok(0) => None, + Ok(len) => Some(Ok(buf[..len].to_vec())), + Err(err) => Some(Err(ShellError::from(err.into_spanned(span)))), + }))?; + Ok(()) } } } @@ -446,11 +348,3 @@ where } } } - -/// Custom iterator for [`RawStream`] that respects ctrlc, but still has binary chunks -fn raw_stream_iter(stream: RawStream) -> impl Iterator, ShellError>> { - let ctrlc = stream.ctrlc; - stream - .stream - .take_while(move |_| ctrlc.as_ref().map(|b| !b.load(Relaxed)).unwrap_or(true)) -} diff --git a/crates/nu-plugin-core/src/interface/tests.rs b/crates/nu-plugin-core/src/interface/tests.rs index ce7be52f30..fb3d737190 100644 --- a/crates/nu-plugin-core/src/interface/tests.rs +++ b/crates/nu-plugin-core/src/interface/tests.rs @@ -6,11 +6,12 @@ use super::{ Interface, InterfaceManager, PluginRead, PluginWrite, }; use nu_plugin_protocol::{ - ExternalStreamInfo, ListStreamInfo, PipelineDataHeader, PluginInput, PluginOutput, - RawStreamInfo, StreamData, StreamMessage, + ByteStreamInfo, ListStreamInfo, PipelineDataHeader, PluginInput, PluginOutput, StreamData, + StreamMessage, }; use nu_protocol::{ - DataSource, ListStream, PipelineData, PipelineMetadata, RawStream, ShellError, Span, Value, + ByteStream, ByteStreamSource, DataSource, ListStream, PipelineData, PipelineMetadata, + ShellError, Span, Value, }; use std::{path::Path, sync::Arc}; @@ -140,9 +141,9 @@ fn read_pipeline_data_value() -> Result<(), ShellError> { let header = PipelineDataHeader::Value(value.clone()); match manager.read_pipeline_data(header, None)? { - PipelineData::Value(read_value, _) => assert_eq!(value, read_value), - PipelineData::ListStream(_, _) => panic!("unexpected ListStream"), - PipelineData::ExternalStream { .. } => panic!("unexpected ExternalStream"), + PipelineData::Value(read_value, ..) => assert_eq!(value, read_value), + PipelineData::ListStream(..) => panic!("unexpected ListStream"), + PipelineData::ByteStream(..) => panic!("unexpected ByteStream"), PipelineData::Empty => panic!("unexpected Empty"), } @@ -188,47 +189,25 @@ fn read_pipeline_data_list_stream() -> Result<(), ShellError> { } #[test] -fn read_pipeline_data_external_stream() -> Result<(), ShellError> { +fn read_pipeline_data_byte_stream() -> Result<(), ShellError> { let test = TestCase::new(); let mut manager = TestInterfaceManager::new(&test); let iterations = 100; let out_pattern = b"hello".to_vec(); - let err_pattern = vec![5, 4, 3, 2]; - test.add(StreamMessage::Data(14, Value::test_int(1).into())); for _ in 0..iterations { test.add(StreamMessage::Data( 12, StreamData::Raw(Ok(out_pattern.clone())), )); - test.add(StreamMessage::Data( - 13, - StreamData::Raw(Ok(err_pattern.clone())), - )); } test.add(StreamMessage::End(12)); - test.add(StreamMessage::End(13)); - test.add(StreamMessage::End(14)); let test_span = Span::new(10, 13); - let header = PipelineDataHeader::ExternalStream(ExternalStreamInfo { + let header = PipelineDataHeader::ByteStream(ByteStreamInfo { + id: 12, span: test_span, - stdout: Some(RawStreamInfo { - id: 12, - is_binary: false, - known_size: Some((out_pattern.len() * iterations) as u64), - }), - stderr: Some(RawStreamInfo { - id: 13, - is_binary: true, - known_size: None, - }), - exit_code: Some(ListStreamInfo { - id: 14, - span: Span::test_data(), - }), - trim_end_newline: true, }); let pipe = manager.read_pipeline_data(header, None)?; @@ -237,52 +216,28 @@ fn read_pipeline_data_external_stream() -> Result<(), ShellError> { manager.consume_all()?; match pipe { - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - } => { - let stdout = stdout.expect("stdout is None"); - let stderr = stderr.expect("stderr is None"); - let exit_code = exit_code.expect("exit_code is None"); - assert_eq!(test_span, span); + PipelineData::ByteStream(stream, metadata) => { + assert_eq!(test_span, stream.span()); assert!( metadata.is_some(), "expected metadata to be Some due to prepare_pipeline_data()" ); - assert!(trim_end_newline); - assert!(!stdout.is_binary); - assert!(stderr.is_binary); - - assert_eq!( - Some((out_pattern.len() * iterations) as u64), - stdout.known_size - ); - assert_eq!(None, stderr.known_size); - - // check the streams - let mut count = 0; - for chunk in stdout.stream { - assert_eq!(out_pattern, chunk?); - count += 1; + match stream.into_source() { + ByteStreamSource::Read(mut read) => { + let mut buf = Vec::new(); + read.read_to_end(&mut buf)?; + let iter = buf.chunks_exact(out_pattern.len()); + assert_eq!(iter.len(), iterations); + for chunk in iter { + assert_eq!(out_pattern, chunk) + } + } + ByteStreamSource::File(..) => panic!("unexpected byte stream source: file"), + ByteStreamSource::Child(..) => { + panic!("unexpected byte stream source: child") + } } - assert_eq!(iterations, count, "stdout length"); - let mut count = 0; - - for chunk in stderr.stream { - assert_eq!(err_pattern, chunk?); - count += 1; - } - assert_eq!(iterations, count, "stderr length"); - - assert_eq!( - vec![Value::test_int(1)], - exit_code.into_iter().collect::>() - ); } _ => panic!("unexpected PipelineData: {pipe:?}"), } @@ -436,120 +391,51 @@ fn write_pipeline_data_list_stream() -> Result<(), ShellError> { } #[test] -fn write_pipeline_data_external_stream() -> Result<(), ShellError> { +fn write_pipeline_data_byte_stream() -> Result<(), ShellError> { let test = TestCase::new(); let manager = TestInterfaceManager::new(&test); let interface = manager.get_interface(); - let stdout_bufs = vec![ - b"hello".to_vec(), - b"world".to_vec(), - b"these are tests".to_vec(), - ]; - let stdout_len = stdout_bufs.iter().map(|b| b.len() as u64).sum::(); - let stderr_bufs = vec![b"error messages".to_vec(), b"go here".to_vec()]; - let exit_code = Value::test_int(7); - + let expected = "hello\nworld\nthese are tests"; let span = Span::new(400, 500); - // Set up pipeline data for an external stream - let pipe = PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(stdout_bufs.clone().into_iter().map(Ok)), - None, - span, - Some(stdout_len), - )), - stderr: Some(RawStream::new( - Box::new(stderr_bufs.clone().into_iter().map(Ok)), - None, - span, - None, - )), - exit_code: Some(ListStream::new( - std::iter::once(exit_code.clone()), - Span::test_data(), - None, - )), - span, - metadata: None, - trim_end_newline: true, - }; + // Set up pipeline data for a byte stream + let data = PipelineData::ByteStream( + ByteStream::read(std::io::Cursor::new(expected), span, None), + None, + ); - let (header, writer) = interface.init_write_pipeline_data(pipe, &())?; + let (header, writer) = interface.init_write_pipeline_data(data, &())?; let info = match header { - PipelineDataHeader::ExternalStream(info) => info, + PipelineDataHeader::ByteStream(info) => info, _ => panic!("unexpected header: {header:?}"), }; writer.write()?; - let stdout_info = info.stdout.as_ref().expect("stdout info is None"); - let stderr_info = info.stderr.as_ref().expect("stderr info is None"); - let exit_code_info = info.exit_code.as_ref().expect("exit code info is None"); - assert_eq!(span, info.span); - assert!(info.trim_end_newline); - - assert_eq!(Some(stdout_len), stdout_info.known_size); - assert_eq!(None, stderr_info.known_size); // Now make sure the stream messages have been written - let mut stdout_iter = stdout_bufs.into_iter(); - let mut stderr_iter = stderr_bufs.into_iter(); - let mut exit_code_iter = std::iter::once(exit_code); + let mut actual = Vec::new(); + let mut ended = false; - let mut stdout_ended = false; - let mut stderr_ended = false; - let mut exit_code_ended = false; - - // There's no specific order these messages must come in with respect to how the streams are - // interleaved, but all of the data for each stream must be in its original order, and the - // End must come after all Data for msg in test.written() { match msg { PluginOutput::Data(id, data) => { - if id == stdout_info.id { - let result: Result, ShellError> = - data.try_into().expect("wrong data in stdout stream"); - assert_eq!( - stdout_iter.next().expect("too much data in stdout"), - result.expect("unexpected error in stdout stream") - ); - } else if id == stderr_info.id { - let result: Result, ShellError> = - data.try_into().expect("wrong data in stderr stream"); - assert_eq!( - stderr_iter.next().expect("too much data in stderr"), - result.expect("unexpected error in stderr stream") - ); - } else if id == exit_code_info.id { - let code: Value = data.try_into().expect("wrong data in stderr stream"); - assert_eq!( - exit_code_iter.next().expect("too much data in stderr"), - code - ); + if id == info.id { + let data: Result, ShellError> = + data.try_into().expect("wrong data in stream"); + + let data = data.expect("unexpected error in stream"); + actual.extend(data); } else { panic!("unrecognized stream id: {id}"); } } PluginOutput::End(id) => { - if id == stdout_info.id { - assert!(!stdout_ended, "double End of stdout"); - assert!(stdout_iter.next().is_none(), "unexpected end of stdout"); - stdout_ended = true; - } else if id == stderr_info.id { - assert!(!stderr_ended, "double End of stderr"); - assert!(stderr_iter.next().is_none(), "unexpected end of stderr"); - stderr_ended = true; - } else if id == exit_code_info.id { - assert!(!exit_code_ended, "double End of exit_code"); - assert!( - exit_code_iter.next().is_none(), - "unexpected end of exit_code" - ); - exit_code_ended = true; + if id == info.id { + ended = true; } else { panic!("unrecognized stream id: {id}"); } @@ -558,9 +444,8 @@ fn write_pipeline_data_external_stream() -> Result<(), ShellError> { } } - assert!(stdout_ended, "stdout did not End"); - assert!(stderr_ended, "stderr did not End"); - assert!(exit_code_ended, "exit_code did not End"); + assert_eq!(expected.as_bytes(), actual); + assert!(ended, "stream did not End"); Ok(()) } diff --git a/crates/nu-plugin-engine/src/context.rs b/crates/nu-plugin-engine/src/context.rs index 3f77d85477..0b1d56c050 100644 --- a/crates/nu-plugin-engine/src/context.rs +++ b/crates/nu-plugin-engine/src/context.rs @@ -106,9 +106,9 @@ impl<'a> PluginExecutionContext for PluginExecutionCommandContext<'a> { let span = value.span(); match value { Value::Closure { val, .. } => { - ClosureEvalOnce::new(&self.engine_state, &self.stack, val) + ClosureEvalOnce::new(&self.engine_state, &self.stack, *val) .run_with_input(PipelineData::Empty) - .map(|data| data.into_value(span)) + .and_then(|data| data.into_value(span)) .unwrap_or_else(|err| Value::error(err, self.call.head)) } _ => value.clone(), diff --git a/crates/nu-plugin-engine/src/init.rs b/crates/nu-plugin-engine/src/init.rs index 0ba70b49c0..198a01cd1c 100644 --- a/crates/nu-plugin-engine/src/init.rs +++ b/crates/nu-plugin-engine/src/init.rs @@ -26,7 +26,7 @@ use crate::{ /// This should be larger than the largest commonly sent message to avoid excessive fragmentation. /// -/// The buffers coming from external streams are typically each 8192 bytes, so double that. +/// The buffers coming from byte streams are typically each 8192 bytes, so double that. pub(crate) const OUTPUT_BUFFER_SIZE: usize = 16384; /// Spawn the command for a plugin, in the given `mode`. After spawning, it can be passed to diff --git a/crates/nu-plugin-engine/src/interface/mod.rs b/crates/nu-plugin-engine/src/interface/mod.rs index 3447d6a907..adab9dc68d 100644 --- a/crates/nu-plugin-engine/src/interface/mod.rs +++ b/crates/nu-plugin-engine/src/interface/mod.rs @@ -519,8 +519,8 @@ impl InterfaceManager for PluginInterfaceManager { .map_data(|data| { let ctrlc = self.get_ctrlc(id)?; - // Register the streams in the response - for stream_id in data.stream_ids() { + // Register the stream in the response + if let Some(stream_id) = data.stream_id() { self.recv_stream_started(id, stream_id); } @@ -602,7 +602,7 @@ impl InterfaceManager for PluginInterfaceManager { meta, )) } - PipelineData::Empty | PipelineData::ExternalStream { .. } => Ok(data), + PipelineData::Empty | PipelineData::ByteStream(..) => Ok(data), } } @@ -953,7 +953,7 @@ impl PluginInterface { let call = PluginCall::CustomValueOp(value.map(|cv| cv.without_source()), op); match self.plugin_call(call, None)? { - PluginCallResponse::PipelineData(out_data) => Ok(out_data.into_value(span)), + PluginCallResponse::PipelineData(out_data) => out_data.into_value(span), PluginCallResponse::Error(err) => Err(err.into()), _ => Err(ShellError::PluginFailedToDecode { msg: format!("Received unexpected response to custom value {op_name}() call"), @@ -1091,7 +1091,7 @@ impl Interface for PluginInterface { meta, )) } - PipelineData::Empty | PipelineData::ExternalStream { .. } => Ok(data), + PipelineData::Empty | PipelineData::ByteStream(..) => Ok(data), } } } diff --git a/crates/nu-plugin-engine/src/interface/tests.rs b/crates/nu-plugin-engine/src/interface/tests.rs index 7548703191..aca59a664e 100644 --- a/crates/nu-plugin-engine/src/interface/tests.rs +++ b/crates/nu-plugin-engine/src/interface/tests.rs @@ -9,10 +9,10 @@ use crate::{ use nu_plugin_core::{interface_test_util::TestCase, Interface, InterfaceManager}; use nu_plugin_protocol::{ test_util::{expected_test_custom_value, test_plugin_custom_value}, - CallInfo, CustomValueOp, EngineCall, EngineCallResponse, EvaluatedCall, ExternalStreamInfo, + ByteStreamInfo, CallInfo, CustomValueOp, EngineCall, EngineCallResponse, EvaluatedCall, ListStreamInfo, PipelineDataHeader, PluginCall, PluginCallId, PluginCallResponse, - PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, RawStreamInfo, - StreamData, StreamMessage, + PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, StreamData, + StreamMessage, }; use nu_protocol::{ ast::{Math, Operator}, @@ -154,16 +154,9 @@ fn manager_consume_all_propagates_message_error_to_readers() -> Result<(), Shell test.add(invalid_output()); let stream = manager.read_pipeline_data( - PipelineDataHeader::ExternalStream(ExternalStreamInfo { + PipelineDataHeader::ByteStream(ByteStreamInfo { + id: 0, span: Span::test_data(), - stdout: Some(RawStreamInfo { - id: 0, - is_binary: false, - known_size: None, - }), - stderr: None, - exit_code: None, - trim_end_newline: false, }), None, )?; @@ -378,7 +371,7 @@ fn manager_consume_call_response_registers_streams() -> Result<(), ShellError> { fake_plugin_call(&mut manager, n); } - // Check list streams, external streams + // Check list streams, byte streams manager.consume(PluginOutput::CallResponse( 0, PluginCallResponse::PipelineData(PipelineDataHeader::ListStream(ListStreamInfo { @@ -388,23 +381,9 @@ fn manager_consume_call_response_registers_streams() -> Result<(), ShellError> { ))?; manager.consume(PluginOutput::CallResponse( 1, - PluginCallResponse::PipelineData(PipelineDataHeader::ExternalStream(ExternalStreamInfo { + PluginCallResponse::PipelineData(PipelineDataHeader::ByteStream(ByteStreamInfo { + id: 1, span: Span::test_data(), - stdout: Some(RawStreamInfo { - id: 1, - is_binary: false, - known_size: None, - }), - stderr: Some(RawStreamInfo { - id: 2, - is_binary: false, - known_size: None, - }), - exit_code: Some(ListStreamInfo { - id: 3, - span: Span::test_data(), - }), - trim_end_newline: false, })), ))?; @@ -423,22 +402,20 @@ fn manager_consume_call_response_registers_streams() -> Result<(), ShellError> { "plugin_call_input_streams[0] should be Some(0)" ); - // ExternalStream should have three + // ByteStream should have one if let Some(sub) = manager.plugin_call_states.get(&1) { assert_eq!( - 3, sub.remaining_streams_to_read, - "ExternalStream remaining_streams_to_read should be 3" + 1, sub.remaining_streams_to_read, + "ByteStream remaining_streams_to_read should be 1" ); } else { - panic!("failed to find subscription for ExternalStream (1), maybe it was removed"); - } - for n in [1, 2, 3] { - assert_eq!( - Some(&1), - manager.plugin_call_input_streams.get(&n), - "plugin_call_input_streams[{n}] should be Some(1)" - ); + panic!("failed to find subscription for ByteStream (1), maybe it was removed"); } + assert_eq!( + Some(&1), + manager.plugin_call_input_streams.get(&1), + "plugin_call_input_streams[1] should be Some(1)" + ); Ok(()) } @@ -1087,7 +1064,7 @@ fn interface_run() -> Result<(), ShellError> { assert_eq!( Value::test_int(number), - result.into_value(Span::test_data()) + result.into_value(Span::test_data())?, ); assert!(test.has_unconsumed_write()); Ok(()) @@ -1136,7 +1113,7 @@ fn interface_prepare_pipeline_data_accepts_normal_values() -> Result<(), ShellEr match interface.prepare_pipeline_data(PipelineData::Value(value.clone(), None), &state) { Ok(data) => assert_eq!( value.get_type(), - data.into_value(Span::test_data()).get_type() + data.into_value(Span::test_data())?.get_type(), ), Err(err) => panic!("failed to accept {value:?}: {err}"), } diff --git a/crates/nu-plugin-protocol/src/lib.rs b/crates/nu-plugin-protocol/src/lib.rs index e40136ca56..ea27f82654 100644 --- a/crates/nu-plugin-protocol/src/lib.rs +++ b/crates/nu-plugin-protocol/src/lib.rs @@ -22,7 +22,7 @@ mod tests; pub mod test_util; use nu_protocol::{ - ast::Operator, engine::Closure, Config, LabeledError, PipelineData, PluginSignature, RawStream, + ast::Operator, engine::Closure, Config, LabeledError, PipelineData, PluginSignature, ShellError, Span, Spanned, Value, }; use serde::{Deserialize, Serialize}; @@ -82,32 +82,20 @@ pub enum PipelineDataHeader { /// /// Items are sent via [`StreamData`] ListStream(ListStreamInfo), - /// Initiate [`nu_protocol::PipelineData::ExternalStream`]. + /// Initiate [`nu_protocol::PipelineData::ByteStream`]. /// /// Items are sent via [`StreamData`] - ExternalStream(ExternalStreamInfo), + ByteStream(ByteStreamInfo), } impl PipelineDataHeader { - /// Return a list of stream IDs embedded in the header - pub fn stream_ids(&self) -> Vec { + /// Return the stream ID, if any, embedded in the header + pub fn stream_id(&self) -> Option { match self { - PipelineDataHeader::Empty => vec![], - PipelineDataHeader::Value(_) => vec![], - PipelineDataHeader::ListStream(info) => vec![info.id], - PipelineDataHeader::ExternalStream(info) => { - let mut out = vec![]; - if let Some(stdout) = &info.stdout { - out.push(stdout.id); - } - if let Some(stderr) = &info.stderr { - out.push(stderr.id); - } - if let Some(exit_code) = &info.exit_code { - out.push(exit_code.id); - } - out - } + PipelineDataHeader::Empty => None, + PipelineDataHeader::Value(_) => None, + PipelineDataHeader::ListStream(info) => Some(info.id), + PipelineDataHeader::ByteStream(info) => Some(info.id), } } } @@ -119,32 +107,11 @@ pub struct ListStreamInfo { pub span: Span, } -/// Additional information about external streams +/// Additional information about byte streams #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] -pub struct ExternalStreamInfo { - pub span: Span, - pub stdout: Option, - pub stderr: Option, - pub exit_code: Option, - pub trim_end_newline: bool, -} - -/// Additional information about raw (byte) streams -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] -pub struct RawStreamInfo { +pub struct ByteStreamInfo { pub id: StreamId, - pub is_binary: bool, - pub known_size: Option, -} - -impl RawStreamInfo { - pub fn new(id: StreamId, stream: &RawStream) -> Self { - RawStreamInfo { - id, - is_binary: stream.is_binary, - known_size: stream.known_size, - } - } + pub span: Span, } /// Calls that a plugin can execute. The type parameter determines the input type. @@ -380,7 +347,7 @@ impl PluginCallResponse { PipelineData::Empty => false, PipelineData::Value(..) => false, PipelineData::ListStream(..) => true, - PipelineData::ExternalStream { .. } => true, + PipelineData::ByteStream(..) => true, }, _ => false, } diff --git a/crates/nu-plugin-test-support/src/lib.rs b/crates/nu-plugin-test-support/src/lib.rs index 8aa675fd1f..caa7cbac1a 100644 --- a/crates/nu-plugin-test-support/src/lib.rs +++ b/crates/nu-plugin-test-support/src/lib.rs @@ -82,7 +82,7 @@ //! let input = vec![Value::test_string("FooBar")].into_pipeline_data(Span::test_data(), None); //! let output = PluginTest::new("lowercase", LowercasePlugin.into())? //! .eval_with("lowercase", input)? -//! .into_value(Span::test_data()); +//! .into_value(Span::test_data())?; //! //! assert_eq!( //! Value::test_list(vec![ diff --git a/crates/nu-plugin-test-support/src/plugin_test.rs b/crates/nu-plugin-test-support/src/plugin_test.rs index 5f34fbee46..d505ca42a6 100644 --- a/crates/nu-plugin-test-support/src/plugin_test.rs +++ b/crates/nu-plugin-test-support/src/plugin_test.rs @@ -93,7 +93,7 @@ impl PluginTest { /// "my-command", /// vec![Value::test_int(42)].into_pipeline_data(Span::test_data(), None) /// )? - /// .into_value(Span::test_data()); + /// .into_value(Span::test_data())?; /// assert_eq!(Value::test_string("42"), result); /// # Ok(()) /// # } @@ -136,33 +136,44 @@ impl PluginTest { // Serialize custom values in the input let source = self.source.clone(); - let input = input.map( - move |mut value| { - let result = PluginCustomValue::serialize_custom_values_in(&mut value) - // Make sure to mark them with the source so they pass correctly, too. - .and_then(|_| PluginCustomValueWithSource::add_source_in(&mut value, &source)); - match result { - Ok(()) => value, - Err(err) => Value::error(err, value.span()), - } - }, - None, - )?; + let input = if matches!(input, PipelineData::ByteStream(..)) { + input + } else { + input.map( + move |mut value| { + let result = PluginCustomValue::serialize_custom_values_in(&mut value) + // Make sure to mark them with the source so they pass correctly, too. + .and_then(|_| { + PluginCustomValueWithSource::add_source_in(&mut value, &source) + }); + match result { + Ok(()) => value, + Err(err) => Value::error(err, value.span()), + } + }, + None, + )? + }; // Eval the block with the input let mut stack = Stack::new().capture(); - eval_block::(&self.engine_state, &mut stack, &block, input)?.map( - |mut value| { - // Make sure to deserialize custom values - let result = PluginCustomValueWithSource::remove_source_in(&mut value) - .and_then(|_| PluginCustomValue::deserialize_custom_values_in(&mut value)); - match result { - Ok(()) => value, - Err(err) => Value::error(err, value.span()), - } - }, - None, - ) + let data = eval_block::(&self.engine_state, &mut stack, &block, input)?; + if matches!(data, PipelineData::ByteStream(..)) { + Ok(data) + } else { + data.map( + |mut value| { + // Make sure to deserialize custom values + let result = PluginCustomValueWithSource::remove_source_in(&mut value) + .and_then(|_| PluginCustomValue::deserialize_custom_values_in(&mut value)); + match result { + Ok(()) => value, + Err(err) => Value::error(err, value.span()), + } + }, + None, + ) + } } /// Evaluate some Nushell source code with the plugin commands in scope. @@ -176,7 +187,7 @@ impl PluginTest { /// # fn test(MyPlugin: impl Plugin + Send + 'static) -> Result<(), ShellError> { /// let result = PluginTest::new("my_plugin", MyPlugin.into())? /// .eval("42 | my-command")? - /// .into_value(Span::test_data()); + /// .into_value(Span::test_data())?; /// assert_eq!(Value::test_string("42"), result); /// # Ok(()) /// # } @@ -219,7 +230,7 @@ impl PluginTest { if let Some(expectation) = &example.result { match self.eval(example.example) { Ok(data) => { - let mut value = data.into_value(Span::test_data()); + let mut value = data.into_value(Span::test_data())?; // Set all of the spans in the value to test_data() to avoid unnecessary // differences when printing diff --git a/crates/nu-plugin-test-support/tests/custom_value/mod.rs b/crates/nu-plugin-test-support/tests/custom_value/mod.rs index aaae5538ff..f703a92e33 100644 --- a/crates/nu-plugin-test-support/tests/custom_value/mod.rs +++ b/crates/nu-plugin-test-support/tests/custom_value/mod.rs @@ -143,7 +143,7 @@ fn test_into_int_from_u32() -> Result<(), ShellError> { "into int from u32", PipelineData::Value(CustomU32(42).into_value(Span::test_data()), None), )? - .into_value(Span::test_data()); + .into_value(Span::test_data())?; assert_eq!(Value::test_int(42), result); Ok(()) } diff --git a/crates/nu-plugin-test-support/tests/hello/mod.rs b/crates/nu-plugin-test-support/tests/hello/mod.rs index 00886f1888..424940f156 100644 --- a/crates/nu-plugin-test-support/tests/hello/mod.rs +++ b/crates/nu-plugin-test-support/tests/hello/mod.rs @@ -80,7 +80,7 @@ fn test_requiring_nu_cmd_lang_commands() -> Result<(), ShellError> { let result = PluginTest::new("hello", HelloPlugin.into())? .eval("do { let greeting = hello; $greeting }")? - .into_value(Span::test_data()); + .into_value(Span::test_data())?; assert_eq!(Value::test_string("Hello, World!"), result); diff --git a/crates/nu-plugin-test-support/tests/lowercase/mod.rs b/crates/nu-plugin-test-support/tests/lowercase/mod.rs index 33446cea86..0072a08aa2 100644 --- a/crates/nu-plugin-test-support/tests/lowercase/mod.rs +++ b/crates/nu-plugin-test-support/tests/lowercase/mod.rs @@ -73,7 +73,7 @@ fn test_lowercase_using_eval_with() -> Result<(), ShellError> { assert_eq!( Value::test_list(vec![Value::test_string("hello world")]), - result.into_value(Span::test_data()) + result.into_value(Span::test_data())? ); Ok(()) diff --git a/crates/nu-plugin/src/plugin/command.rs b/crates/nu-plugin/src/plugin/command.rs index ad8ecd7d9c..5def950b0b 100644 --- a/crates/nu-plugin/src/plugin/command.rs +++ b/crates/nu-plugin/src/plugin/command.rs @@ -313,7 +313,7 @@ where // Unwrap the PipelineData from input, consuming the potential stream, and pass it to the // simpler signature in Plugin let span = input.span().unwrap_or(call.head); - let input_value = input.into_value(span); + let input_value = input.into_value(span)?; // Wrap the output in PipelineData::Value ::run(self, plugin, engine, call, &input_value) .map(|value| PipelineData::Value(value, None)) diff --git a/crates/nu-plugin/src/plugin/interface/mod.rs b/crates/nu-plugin/src/plugin/interface/mod.rs index 70e143ece5..e3e9679471 100644 --- a/crates/nu-plugin/src/plugin/interface/mod.rs +++ b/crates/nu-plugin/src/plugin/interface/mod.rs @@ -345,7 +345,7 @@ impl InterfaceManager for EngineInterfaceManager { }); Ok(PipelineData::ListStream(stream, meta)) } - PipelineData::Empty | PipelineData::ExternalStream { .. } => Ok(data), + PipelineData::Empty | PipelineData::ByteStream(..) => Ok(data), } } } @@ -850,7 +850,7 @@ impl EngineInterface { let input = input.map_or_else(|| PipelineData::Empty, |v| PipelineData::Value(v, None)); let output = self.eval_closure_with_stream(closure, positional, input, true, false)?; // Unwrap an error value - match output.into_value(closure.span) { + match output.into_value(closure.span)? { Value::Error { error, .. } => Err(*error), value => Ok(value), } @@ -920,7 +920,7 @@ impl Interface for EngineInterface { }); Ok(PipelineData::ListStream(stream, meta)) } - PipelineData::Empty | PipelineData::ExternalStream { .. } => Ok(data), + PipelineData::Empty | PipelineData::ByteStream(..) => Ok(data), } } } diff --git a/crates/nu-plugin/src/plugin/interface/tests.rs b/crates/nu-plugin/src/plugin/interface/tests.rs index 17018cbc00..ed04190712 100644 --- a/crates/nu-plugin/src/plugin/interface/tests.rs +++ b/crates/nu-plugin/src/plugin/interface/tests.rs @@ -4,10 +4,9 @@ use super::{EngineInterfaceManager, ReceivedPluginCall}; use nu_plugin_core::{interface_test_util::TestCase, Interface, InterfaceManager}; use nu_plugin_protocol::{ test_util::{expected_test_custom_value, test_plugin_custom_value, TestCustomValue}, - CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, EvaluatedCall, - ExternalStreamInfo, ListStreamInfo, PipelineDataHeader, PluginCall, PluginCallResponse, - PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, RawStreamInfo, - StreamData, + ByteStreamInfo, CallInfo, CustomValueOp, EngineCall, EngineCallId, EngineCallResponse, + EvaluatedCall, ListStreamInfo, PipelineDataHeader, PluginCall, PluginCallResponse, + PluginCustomValue, PluginInput, PluginOutput, Protocol, ProtocolInfo, StreamData, }; use nu_protocol::{ engine::Closure, Config, CustomValue, IntoInterruptiblePipelineData, LabeledError, @@ -158,16 +157,9 @@ fn manager_consume_all_propagates_message_error_to_readers() -> Result<(), Shell test.add(invalid_input()); let stream = manager.read_pipeline_data( - PipelineDataHeader::ExternalStream(ExternalStreamInfo { + PipelineDataHeader::ByteStream(ByteStreamInfo { + id: 0, span: Span::test_data(), - stdout: Some(RawStreamInfo { - id: 0, - is_binary: false, - known_size: None, - }), - stderr: None, - exit_code: None, - trim_end_newline: false, }), None, )?; @@ -1046,7 +1038,7 @@ fn interface_eval_closure_with_stream() -> Result<(), ShellError> { true, false, )? - .into_value(Span::test_data()); + .into_value(Span::test_data())?; assert_eq!(Value::test_int(2), result); diff --git a/crates/nu-plugin/src/plugin/mod.rs b/crates/nu-plugin/src/plugin/mod.rs index 0ec170f4cd..85283aadd0 100644 --- a/crates/nu-plugin/src/plugin/mod.rs +++ b/crates/nu-plugin/src/plugin/mod.rs @@ -30,7 +30,7 @@ pub use interface::{EngineInterface, EngineInterfaceManager}; /// This should be larger than the largest commonly sent message to avoid excessive fragmentation. /// -/// The buffers coming from external streams are typically each 8192 bytes, so double that. +/// The buffers coming from byte streams are typically each 8192 bytes, so double that. #[allow(dead_code)] pub(crate) const OUTPUT_BUFFER_SIZE: usize = 16384; @@ -260,7 +260,8 @@ pub fn serve_plugin(plugin: &impl Plugin, encoder: impl PluginEncoder + 'static) } } else { eprintln!( - "{}: This plugin must be run from within Nushell.", + "{}: This plugin must be run from within Nushell. See `plugin add --help` for details \ + on how to use plugins.", env::current_exe() .map(|path| path.display().to_string()) .unwrap_or_else(|_| "plugin".into()) diff --git a/crates/nu-protocol/Cargo.toml b/crates/nu-protocol/Cargo.toml index f954ae487d..3949f9174a 100644 --- a/crates/nu-protocol/Cargo.toml +++ b/crates/nu-protocol/Cargo.toml @@ -32,6 +32,10 @@ serde_json = { workspace = true, optional = true } thiserror = "1.0" typetag = "0.2" ecow = { version = "0.2.2", features = ["serde"] } +os_pipe = { workspace = true, features = ["io_safety"] } + +[target.'cfg(unix)'.dependencies] +nix = { workspace = true, default-features = false, features = ["signal"] } [features] plugin = [ @@ -48,6 +52,7 @@ nu-test-support = { path = "../nu-test-support", version = "0.93.1" } pretty_assertions = { workspace = true } rstest = { workspace = true } tempfile = { workspace = true } +os_pipe = { workspace = true } [package.metadata.docs.rs] all-features = true diff --git a/crates/nu-protocol/src/ast/call.rs b/crates/nu-protocol/src/ast/call.rs index 6689bc3a55..9aaf17d5db 100644 --- a/crates/nu-protocol/src/ast/call.rs +++ b/crates/nu-protocol/src/ast/call.rs @@ -154,30 +154,10 @@ impl Call { }) } - pub fn positional_iter_mut(&mut self) -> impl Iterator { - self.arguments - .iter_mut() - .take_while(|arg| match arg { - Argument::Spread(_) => false, // Don't include positional arguments given to rest parameter - _ => true, - }) - .filter_map(|arg| match arg { - Argument::Named(_) => None, - Argument::Positional(positional) => Some(positional), - Argument::Unknown(unknown) => Some(unknown), - Argument::Spread(_) => None, - }) - } - pub fn positional_nth(&self, i: usize) -> Option<&Expression> { self.positional_iter().nth(i) } - // TODO this method is never used. Delete? - pub fn positional_nth_mut(&mut self, i: usize) -> Option<&mut Expression> { - self.positional_iter_mut().nth(i) - } - pub fn positional_len(&self) -> usize { self.positional_iter().count() } diff --git a/crates/nu-protocol/src/ast/import_pattern.rs b/crates/nu-protocol/src/ast/import_pattern.rs index 2af08087a9..893dd9897b 100644 --- a/crates/nu-protocol/src/ast/import_pattern.rs +++ b/crates/nu-protocol/src/ast/import_pattern.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -use crate::{span, ModuleId, Span, VarId}; +use crate::{ModuleId, Span, VarId}; use std::collections::HashSet; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -12,17 +12,22 @@ pub enum ImportPatternMember { impl ImportPatternMember { pub fn span(&self) -> Span { - let mut spans = vec![]; match self { - ImportPatternMember::Glob { span } => spans.push(*span), - ImportPatternMember::Name { name: _, span } => spans.push(*span), + ImportPatternMember::Glob { span } | ImportPatternMember::Name { span, .. } => *span, ImportPatternMember::List { names } => { - for (_, span) in names { - spans.push(*span); - } + let first = names + .first() + .map(|&(_, span)| span) + .unwrap_or(Span::unknown()); + + let last = names + .last() + .map(|&(_, span)| span) + .unwrap_or(Span::unknown()); + + Span::append(first, last) } } - span(&spans) } } @@ -59,13 +64,13 @@ impl ImportPattern { } pub fn span(&self) -> Span { - let mut spans = vec![self.head.span]; - - for member in &self.members { - spans.push(member.span()); - } - - span(&spans) + Span::append( + self.head.span, + self.members + .last() + .map(ImportPatternMember::span) + .unwrap_or(self.head.span), + ) } pub fn with_hidden(self, hidden: HashSet>) -> Self { diff --git a/crates/nu-protocol/src/debugger/debugger_trait.rs b/crates/nu-protocol/src/debugger/debugger_trait.rs index 7a842f6c28..69d395fb98 100644 --- a/crates/nu-protocol/src/debugger/debugger_trait.rs +++ b/crates/nu-protocol/src/debugger/debugger_trait.rs @@ -44,7 +44,7 @@ pub trait DebugContext: Clone + Copy + Debug { fn leave_element( engine_state: &EngineState, element: &PipelineElement, - result: &Result<(PipelineData, bool), ShellError>, + result: &Result, ) { } } @@ -77,7 +77,7 @@ impl DebugContext for WithDebug { fn leave_element( engine_state: &EngineState, element: &PipelineElement, - result: &Result<(PipelineData, bool), ShellError>, + result: &Result, ) { if let Ok(mut debugger) = engine_state.debugger.lock() { debugger @@ -128,7 +128,7 @@ pub trait Debugger: Send + Debug { &mut self, engine_state: &EngineState, element: &PipelineElement, - result: &Result<(PipelineData, bool), ShellError>, + result: &Result, ) { } diff --git a/crates/nu-protocol/src/debugger/profiler.rs b/crates/nu-protocol/src/debugger/profiler.rs index 9d5bece0ab..53b9d0555a 100644 --- a/crates/nu-protocol/src/debugger/profiler.rs +++ b/crates/nu-protocol/src/debugger/profiler.rs @@ -158,7 +158,7 @@ impl Debugger for Profiler { &mut self, _engine_state: &EngineState, element: &PipelineElement, - result: &Result<(PipelineData, bool), ShellError>, + result: &Result, ) { if self.depth > self.max_depth { return; @@ -167,12 +167,10 @@ impl Debugger for Profiler { let element_span = element.expr.span; let out_opt = self.collect_values.then(|| match result { - Ok((pipeline_data, _not_sure_what_this_is)) => match pipeline_data { + Ok(pipeline_data) => match pipeline_data { PipelineData::Value(val, ..) => val.clone(), PipelineData::ListStream(..) => Value::string("list stream", element_span), - PipelineData::ExternalStream { .. } => { - Value::string("external stream", element_span) - } + PipelineData::ByteStream(..) => Value::string("byte stream", element_span), _ => Value::nothing(element_span), }, Err(e) => Value::error(e.clone(), element_span), diff --git a/crates/nu-protocol/src/engine/engine_state.rs b/crates/nu-protocol/src/engine/engine_state.rs index 1593b3341a..bea49b5d6c 100644 --- a/crates/nu-protocol/src/engine/engine_state.rs +++ b/crates/nu-protocol/src/engine/engine_state.rs @@ -6,6 +6,7 @@ use crate::{ CachedFile, Command, CommandType, EnvVars, OverlayFrame, ScopeFrame, Stack, StateDelta, Variable, Visibility, DEFAULT_OVERLAY_NAME, }, + eval_const::create_nu_constant, BlockId, Category, Config, DeclId, Example, FileId, HistoryConfig, Module, ModuleId, OverlayId, ShellError, Signature, Span, Type, Value, VarId, VirtualPathId, }; @@ -753,8 +754,8 @@ impl EngineState { var.const_val.as_ref() } - pub fn set_variable_const_val(&mut self, var_id: VarId, val: Value) { - self.vars[var_id].const_val = Some(val); + pub fn generate_nu_constant(&mut self) { + self.vars[NU_VARIABLE_ID].const_val = Some(create_nu_constant(self, Span::unknown())); } pub fn get_decl(&self, decl_id: DeclId) -> &dyn Command { @@ -930,13 +931,12 @@ impl EngineState { /// directory on the stack that have yet to be merged into the engine state. pub fn cwd(&self, stack: Option<&Stack>) -> Result { // Helper function to create a simple generic error. - // Its messages are not especially helpful, but these errors don't occur often, so it's probably fine. - fn error(msg: &str) -> Result { + fn error(msg: &str, cwd: impl AsRef) -> Result { Err(ShellError::GenericError { error: msg.into(), - msg: "".into(), + msg: format!("$env.PWD = {}", cwd.as_ref().display()), span: None, - help: None, + help: Some("Use `cd` to reset $env.PWD into a good state".into()), inner: vec![], }) } @@ -966,21 +966,21 @@ impl EngineState { // Technically, a root path counts as "having trailing slashes", but // for the purpose of PWD, a root path is acceptable. if !is_root(&path) && has_trailing_slash(&path) { - error("$env.PWD contains trailing slashes") + error("$env.PWD contains trailing slashes", path) } else if !path.is_absolute() { - error("$env.PWD is not an absolute path") + error("$env.PWD is not an absolute path", path) } else if !path.exists() { - error("$env.PWD points to a non-existent directory") + error("$env.PWD points to a non-existent directory", path) } else if !path.is_dir() { - error("$env.PWD points to a non-directory") + error("$env.PWD points to a non-directory", path) } else { Ok(path) } } else { - error("$env.PWD is not a string") + error("$env.PWD is not a string", format!("{pwd:?}")) } } else { - error("$env.PWD not found") + error("$env.PWD not found", "") } } diff --git a/crates/nu-protocol/src/engine/stack.rs b/crates/nu-protocol/src/engine/stack.rs index 2fa71f57fa..19726db9c0 100644 --- a/crates/nu-protocol/src/engine/stack.rs +++ b/crates/nu-protocol/src/engine/stack.rs @@ -7,6 +7,7 @@ use crate::{ }; use std::{ collections::{HashMap, HashSet}, + fs::File, sync::Arc, }; @@ -74,26 +75,10 @@ impl Stack { } } - /// Unwrap a uniquely-owned stack. - /// - /// In debug mode, this panics if there are multiple references. - /// In production this will instead clone the underlying stack. - pub fn unwrap_unique(stack_arc: Arc) -> Stack { - // If you hit an error here, it's likely that you created an extra - // Arc pointing to the stack somewhere. Make sure that it gets dropped before - // getting here! - Arc::try_unwrap(stack_arc).unwrap_or_else(|arc| { - // in release mode, we clone the stack, but this can lead to - // major performance issues, so we should avoid it - debug_assert!(false, "More than one stack reference remaining!"); - (*arc).clone() - }) - } - /// Create a new child stack from a parent. /// /// Changes from this child can be merged back into the parent with - /// Stack::with_changes_from_child + /// [`Stack::with_changes_from_child`] pub fn with_parent(parent: Arc) -> Stack { Stack { // here we are still cloning environment variable-related information @@ -108,19 +93,17 @@ impl Stack { } } - /// Take an Arc of a parent (assumed to be unique), and a child, and apply - /// all the changes from a child back to the parent. + /// Take an [`Arc`] parent, and a child, and apply all the changes from a child back to the parent. /// - /// Here it is assumed that child was created with a call to Stack::with_parent - /// with parent + /// Here it is assumed that `child` was created by a call to [`Stack::with_parent`] with `parent`. + /// + /// For this to be performant and not clone `parent`, `child` should be the only other + /// referencer of `parent`. pub fn with_changes_from_child(parent: Arc, child: Stack) -> Stack { // we're going to drop the link to the parent stack on our new stack // so that we can unwrap the Arc as a unique reference - // - // This makes the new_stack be in a bit of a weird state, so we shouldn't call - // any structs drop(child.parent_stack); - let mut unique_stack = Stack::unwrap_unique(parent); + let mut unique_stack = Arc::unwrap_or_clone(parent); unique_stack .vars @@ -592,6 +575,91 @@ impl Stack { self.out_dest.pipe_stderr = None; self } + + /// Replaces the default stdout of the stack with a given file. + /// + /// This method configures the default stdout to redirect to a specified file. + /// It is primarily useful for applications using `nu` as a language, where the stdout of + /// external commands that are not explicitly piped can be redirected to a file. + /// + /// # Using Pipes + /// + /// For use in third-party applications pipes might be very useful as they allow using the + /// stdout of external commands for different uses. + /// For example the [`os_pipe`](https://docs.rs/os_pipe) crate provides a elegant way to to + /// access the stdout. + /// + /// ``` + /// # use std::{fs::File, io::{self, Read}, thread, error}; + /// # use nu_protocol::engine::Stack; + /// # + /// let (mut reader, writer) = os_pipe::pipe().unwrap(); + /// // Use a thread to avoid blocking the execution of the called command. + /// let reader = thread::spawn(move || { + /// let mut buf: Vec = Vec::new(); + /// reader.read_to_end(&mut buf)?; + /// Ok::<_, io::Error>(buf) + /// }); + /// + /// #[cfg(windows)] + /// let file = std::os::windows::io::OwnedHandle::from(writer).into(); + /// #[cfg(unix)] + /// let file = std::os::unix::io::OwnedFd::from(writer).into(); + /// + /// let stack = Stack::new().stdout_file(file); + /// + /// // Execute some nu code. + /// + /// drop(stack); // drop the stack so that the writer will be dropped too + /// let buf = reader.join().unwrap().unwrap(); + /// // Do with your buffer whatever you want. + /// ``` + pub fn stdout_file(mut self, file: File) -> Self { + self.out_dest.stdout = OutDest::File(Arc::new(file)); + self + } + + /// Replaces the default stderr of the stack with a given file. + /// + /// For more info, see [`stdout_file`](Self::stdout_file). + pub fn stderr_file(mut self, file: File) -> Self { + self.out_dest.stderr = OutDest::File(Arc::new(file)); + self + } + + /// Set the PWD environment variable to `path`. + /// + /// This method accepts `path` with trailing slashes, but they're removed + /// before writing the value into PWD. + pub fn set_cwd(&mut self, path: impl AsRef) -> Result<(), ShellError> { + // Helper function to create a simple generic error. + // Its messages are not especially helpful, but these errors don't occur often, so it's probably fine. + fn error(msg: &str) -> Result<(), ShellError> { + Err(ShellError::GenericError { + error: msg.into(), + msg: "".into(), + span: None, + help: None, + inner: vec![], + }) + } + + let path = path.as_ref(); + + if !path.is_absolute() { + error("Cannot set $env.PWD to a non-absolute path") + } else if !path.exists() { + error("Cannot set $env.PWD to a non-existent directory") + } else if !path.is_dir() { + error("Cannot set $env.PWD to a non-directory") + } else { + // Strip trailing slashes, if any. + let path = nu_path::strip_trailing_slash(path); + let value = Value::string(path.to_string_lossy(), Span::unknown()); + self.add_env_var("PWD".into(), value); + Ok(()) + } + } } #[cfg(test)] diff --git a/crates/nu-protocol/src/errors/cli_error.rs b/crates/nu-protocol/src/errors/cli_error.rs index 6be1ebce40..003564f933 100644 --- a/crates/nu-protocol/src/errors/cli_error.rs +++ b/crates/nu-protocol/src/errors/cli_error.rs @@ -41,7 +41,6 @@ pub fn report_error_new( error: &(dyn miette::Diagnostic + Send + Sync + 'static), ) { let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, error); } diff --git a/crates/nu-protocol/src/errors/shell_error.rs b/crates/nu-protocol/src/errors/shell_error.rs index e1d7ade338..525f32e925 100644 --- a/crates/nu-protocol/src/errors/shell_error.rs +++ b/crates/nu-protocol/src/errors/shell_error.rs @@ -1,5 +1,6 @@ use miette::Diagnostic; use serde::{Deserialize, Serialize}; +use std::io; use thiserror::Error; use crate::{ @@ -1374,42 +1375,79 @@ impl ShellError { } } -impl From for ShellError { - fn from(input: std::io::Error) -> ShellError { - ShellError::IOError { - msg: format!("{input:?}"), +impl From for ShellError { + fn from(error: io::Error) -> ShellError { + if error.kind() == io::ErrorKind::Other { + match error.into_inner() { + Some(err) => match err.downcast() { + Ok(err) => *err, + Err(err) => Self::IOError { + msg: err.to_string(), + }, + }, + None => Self::IOError { + msg: "unknown error".into(), + }, + } + } else { + Self::IOError { + msg: error.to_string(), + } } } } -impl From> for ShellError { - fn from(error: Spanned) -> Self { - ShellError::IOErrorSpanned { - msg: error.item.to_string(), - span: error.span, +impl From> for ShellError { + fn from(error: Spanned) -> Self { + let Spanned { item: error, span } = error; + if error.kind() == io::ErrorKind::Other { + match error.into_inner() { + Some(err) => match err.downcast() { + Ok(err) => *err, + Err(err) => Self::IOErrorSpanned { + msg: err.to_string(), + span, + }, + }, + None => Self::IOErrorSpanned { + msg: "unknown error".into(), + span, + }, + } + } else { + Self::IOErrorSpanned { + msg: error.to_string(), + span, + } } } } -impl std::convert::From> for ShellError { - fn from(input: Box) -> ShellError { +impl From for io::Error { + fn from(error: ShellError) -> Self { + io::Error::new(io::ErrorKind::Other, error) + } +} + +impl From> for ShellError { + fn from(error: Box) -> ShellError { ShellError::IOError { - msg: input.to_string(), + msg: error.to_string(), } } } impl From> for ShellError { - fn from(input: Box) -> ShellError { + fn from(error: Box) -> ShellError { ShellError::IOError { - msg: format!("{input:?}"), + msg: format!("{error:?}"), } } } impl From for ShellError { - fn from(value: super::LabeledError) -> Self { - ShellError::LabeledError(Box::new(value)) + fn from(error: super::LabeledError) -> Self { + ShellError::LabeledError(Box::new(error)) } } diff --git a/crates/nu-protocol/src/eval_const.rs b/crates/nu-protocol/src/eval_const.rs index e45e3b7e4b..4cc7e25324 100644 --- a/crates/nu-protocol/src/eval_const.rs +++ b/crates/nu-protocol/src/eval_const.rs @@ -12,7 +12,7 @@ use std::{ }; /// Create a Value for `$nu`. -pub fn create_nu_constant(engine_state: &EngineState, span: Span) -> Result { +pub(crate) fn create_nu_constant(engine_state: &EngineState, span: Span) -> Value { fn canonicalize_path(engine_state: &EngineState, path: &Path) -> PathBuf { #[allow(deprecated)] let cwd = engine_state.current_work_dir(); @@ -200,7 +200,7 @@ pub fn create_nu_constant(engine_state: &EngineState, span: Span) -> Result Result { // TODO: Allow debugging const eval // TODO: eval.rs uses call.head for the span rather than expr.span - Ok(eval_const_call(working_set, call, PipelineData::empty())?.into_value(span)) + eval_const_call(working_set, call, PipelineData::empty())?.into_value(span) } fn eval_external_call( @@ -339,10 +339,7 @@ impl Eval for EvalConst { ) -> Result { // TODO: Allow debugging const eval let block = working_set.get_block(block_id); - Ok( - eval_const_subexpression(working_set, block, PipelineData::empty(), span)? - .into_value(span), - ) + eval_const_subexpression(working_set, block, PipelineData::empty(), span)?.into_value(span) } fn regex_match( diff --git a/crates/nu-protocol/src/lib.rs b/crates/nu-protocol/src/lib.rs index f5842b5b3a..d09186cf46 100644 --- a/crates/nu-protocol/src/lib.rs +++ b/crates/nu-protocol/src/lib.rs @@ -11,14 +11,14 @@ mod example; mod id; mod lev_distance; mod module; -mod pipeline_data; +mod pipeline; #[cfg(feature = "plugin")] mod plugin; +pub mod process; mod signature; pub mod span; mod syntax_shape; mod ty; -pub mod util; mod value; pub use alias::*; @@ -31,12 +31,11 @@ pub use example::*; pub use id::*; pub use lev_distance::levenshtein_distance; pub use module::*; -pub use pipeline_data::*; +pub use pipeline::*; #[cfg(feature = "plugin")] pub use plugin::*; pub use signature::*; pub use span::*; pub use syntax_shape::*; pub use ty::*; -pub use util::BufferedReader; pub use value::*; diff --git a/crates/nu-protocol/src/pipeline/byte_stream.rs b/crates/nu-protocol/src/pipeline/byte_stream.rs new file mode 100644 index 0000000000..64b566a625 --- /dev/null +++ b/crates/nu-protocol/src/pipeline/byte_stream.rs @@ -0,0 +1,833 @@ +use crate::{ + process::{ChildPipe, ChildProcess, ExitStatus}, + ErrSpan, IntoSpanned, OutDest, PipelineData, ShellError, Span, Value, +}; +#[cfg(unix)] +use std::os::fd::OwnedFd; +#[cfg(windows)] +use std::os::windows::io::OwnedHandle; +use std::{ + fmt::Debug, + fs::File, + io::{self, BufRead, BufReader, Cursor, ErrorKind, Read, Write}, + process::Stdio, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread, +}; + +/// The source of bytes for a [`ByteStream`]. +/// +/// Currently, there are only three possibilities: +/// 1. `Read` (any `dyn` type that implements [`Read`]) +/// 2. [`File`] +/// 3. [`ChildProcess`] +pub enum ByteStreamSource { + Read(Box), + File(File), + Child(Box), +} + +impl ByteStreamSource { + fn reader(self) -> Option { + match self { + ByteStreamSource::Read(read) => Some(SourceReader::Read(read)), + ByteStreamSource::File(file) => Some(SourceReader::File(file)), + ByteStreamSource::Child(mut child) => child.stdout.take().map(|stdout| match stdout { + ChildPipe::Pipe(pipe) => SourceReader::File(convert_file(pipe)), + ChildPipe::Tee(tee) => SourceReader::Read(tee), + }), + } + } +} + +enum SourceReader { + Read(Box), + File(File), +} + +impl Read for SourceReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + SourceReader::Read(reader) => reader.read(buf), + SourceReader::File(file) => file.read(buf), + } + } +} + +/// A potentially infinite, interruptible stream of bytes. +/// +/// To create a [`ByteStream`], you can use any of the following methods: +/// - [`read`](ByteStream::read): takes any type that implements [`Read`]. +/// - [`file`](ByteStream::file): takes a [`File`]. +/// - [`from_iter`](ByteStream::from_iter): takes an [`Iterator`] whose items implement `AsRef<[u8]>`. +/// - [`from_result_iter`](ByteStream::from_result_iter): same as [`from_iter`](ByteStream::from_iter), +/// but each item is a `Result`. +/// +/// The data of a [`ByteStream`] can be accessed using one of the following methods: +/// - [`reader`](ByteStream::reader): returns a [`Read`]-able type to get the raw bytes in the stream. +/// - [`lines`](ByteStream::lines): splits the bytes on lines and returns an [`Iterator`] +/// where each item is a `Result`. +/// - [`chunks`](ByteStream::chunks): returns an [`Iterator`] of [`Value`]s where each value is either a string or binary. +/// Try not to use this method if possible. Rather, please use [`reader`](ByteStream::reader) +/// (or [`lines`](ByteStream::lines) if it matches the situation). +/// +/// Additionally, there are few methods to collect a [`Bytestream`] into memory: +/// - [`into_bytes`](ByteStream::into_bytes): collects all bytes into a [`Vec`]. +/// - [`into_string`](ByteStream::into_string): collects all bytes into a [`String`], erroring if utf-8 decoding failed. +/// - [`into_value`](ByteStream::into_value): collects all bytes into a string [`Value`]. +/// If utf-8 decoding failed, then a binary [`Value`] is returned instead. +/// +/// There are also a few other methods to consume all the data of a [`Bytestream`]: +/// - [`drain`](ByteStream::drain): consumes all bytes and outputs nothing. +/// - [`write_to`](ByteStream::write_to): writes all bytes to the given [`Write`] destination. +/// - [`print`](ByteStream::print): a convenience wrapper around [`write_to`](ByteStream::write_to). +/// It prints all bytes to stdout or stderr. +/// +/// Internally, [`ByteStream`]s currently come in three flavors according to [`ByteStreamSource`]. +/// See its documentation for more information. +pub struct ByteStream { + stream: ByteStreamSource, + span: Span, + ctrlc: Option>, + known_size: Option, +} + +impl ByteStream { + /// Create a new [`ByteStream`] from a [`ByteStreamSource`]. + pub fn new(stream: ByteStreamSource, span: Span, interrupt: Option>) -> Self { + Self { + stream, + span, + ctrlc: interrupt, + known_size: None, + } + } + + /// Create a new [`ByteStream`] from a [`ByteStreamSource::Read`]. + pub fn read( + reader: impl Read + Send + 'static, + span: Span, + interrupt: Option>, + ) -> Self { + Self::new(ByteStreamSource::Read(Box::new(reader)), span, interrupt) + } + + /// Create a new [`ByteStream`] from a [`ByteStreamSource::File`]. + pub fn file(file: File, span: Span, interrupt: Option>) -> Self { + Self::new(ByteStreamSource::File(file), span, interrupt) + } + + /// Create a new [`ByteStream`] from a [`ByteStreamSource::Child`]. + pub fn child(child: ChildProcess, span: Span) -> Self { + Self::new(ByteStreamSource::Child(Box::new(child)), span, None) + } + + /// Create a new [`ByteStream`] that reads from stdin. + pub fn stdin(span: Span) -> Result { + let stdin = os_pipe::dup_stdin().err_span(span)?; + let source = ByteStreamSource::File(convert_file(stdin)); + Ok(Self::new(source, span, None)) + } + + /// Create a new [`ByteStream`] from an [`Iterator`] of bytes slices. + /// + /// The returned [`ByteStream`] will have a [`ByteStreamSource`] of `Read`. + pub fn from_iter(iter: I, span: Span, interrupt: Option>) -> Self + where + I: IntoIterator, + I::IntoIter: Send + 'static, + I::Item: AsRef<[u8]> + Default + Send + 'static, + { + let iter = iter.into_iter(); + let cursor = Some(Cursor::new(I::Item::default())); + Self::read(ReadIterator { iter, cursor }, span, interrupt) + } + + /// Create a new [`ByteStream`] from an [`Iterator`] of [`Result`] bytes slices. + /// + /// The returned [`ByteStream`] will have a [`ByteStreamSource`] of `Read`. + pub fn from_result_iter(iter: I, span: Span, interrupt: Option>) -> Self + where + I: IntoIterator>, + I::IntoIter: Send + 'static, + T: AsRef<[u8]> + Default + Send + 'static, + { + let iter = iter.into_iter(); + let cursor = Some(Cursor::new(T::default())); + Self::read(ReadResultIterator { iter, cursor }, span, interrupt) + } + + /// Set the known size, in number of bytes, of the [`ByteStream`]. + pub fn with_known_size(mut self, size: Option) -> Self { + self.known_size = size; + self + } + + /// Get a reference to the inner [`ByteStreamSource`] of the [`ByteStream`]. + pub fn source(&self) -> &ByteStreamSource { + &self.stream + } + + /// Get a mutable reference to the inner [`ByteStreamSource`] of the [`ByteStream`]. + pub fn source_mut(&mut self) -> &mut ByteStreamSource { + &mut self.stream + } + + /// Returns the [`Span`] associated with the [`ByteStream`]. + pub fn span(&self) -> Span { + self.span + } + + /// Returns the known size, in number of bytes, of the [`ByteStream`]. + pub fn known_size(&self) -> Option { + self.known_size + } + + /// Convert the [`ByteStream`] into its [`Reader`] which allows one to [`Read`] the raw bytes of the stream. + /// + /// [`Reader`] is buffered and also implements [`BufRead`]. + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`] and the child has no stdout, + /// then the stream is considered empty and `None` will be returned. + pub fn reader(self) -> Option { + let reader = self.stream.reader()?; + Some(Reader { + reader: BufReader::new(reader), + span: self.span, + ctrlc: self.ctrlc, + }) + } + + /// Convert the [`ByteStream`] into a [`Lines`] iterator where each element is a `Result`. + /// + /// There is no limit on how large each line will be. Ending new lines (`\n` or `\r\n`) are + /// stripped from each line. If a line fails to be decoded as utf-8, then it will become a [`ShellError`]. + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`] and the child has no stdout, + /// then the stream is considered empty and `None` will be returned. + pub fn lines(self) -> Option { + let reader = self.stream.reader()?; + Some(Lines { + reader: BufReader::new(reader), + span: self.span, + ctrlc: self.ctrlc, + }) + } + + /// Convert the [`ByteStream`] into a [`Chunks`] iterator where each element is a `Result`. + /// + /// Each call to [`next`](Iterator::next) reads the currently available data from the byte stream source, + /// up to a maximum size. If the chunk of bytes, or an expected portion of it, succeeds utf-8 decoding, + /// then it is returned as a [`Value::String`]. Otherwise, it is turned into a [`Value::Binary`]. + /// Any and all newlines are kept intact in each chunk. + /// + /// Where possible, prefer [`reader`](ByteStream::reader) or [`lines`](ByteStream::lines) over this method. + /// Those methods are more likely to be used in a semantically correct way + /// (and [`reader`](ByteStream::reader) is more efficient too). + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`] and the child has no stdout, + /// then the stream is considered empty and `None` will be returned. + pub fn chunks(self) -> Option { + let reader = self.stream.reader()?; + Some(Chunks { + reader: BufReader::new(reader), + span: self.span, + ctrlc: self.ctrlc, + leftover: Vec::new(), + }) + } + + /// Convert the [`ByteStream`] into its inner [`ByteStreamSource`]. + pub fn into_source(self) -> ByteStreamSource { + self.stream + } + + /// Attempt to convert the [`ByteStream`] into a [`Stdio`]. + /// + /// This will succeed if the [`ByteStreamSource`] of the [`ByteStream`] is either: + /// - [`File`](ByteStreamSource::File) + /// - [`Child`](ByteStreamSource::Child) and the child has a stdout that is `Some(ChildPipe::Pipe(..))`. + /// + /// All other cases return an `Err` with the original [`ByteStream`] in it. + pub fn into_stdio(mut self) -> Result { + match self.stream { + ByteStreamSource::Read(..) => Err(self), + ByteStreamSource::File(file) => Ok(file.into()), + ByteStreamSource::Child(child) => { + if let ChildProcess { + stdout: Some(ChildPipe::Pipe(stdout)), + stderr, + .. + } = *child + { + debug_assert!(stderr.is_none(), "stderr should not exist"); + Ok(stdout.into()) + } else { + self.stream = ByteStreamSource::Child(child); + Err(self) + } + } + } + } + + /// Attempt to convert the [`ByteStream`] into a [`ChildProcess`]. + /// + /// This will only succeed if the [`ByteStreamSource`] of the [`ByteStream`] is [`Child`](ByteStreamSource::Child). + /// All other cases return an `Err` with the original [`ByteStream`] in it. + pub fn into_child(self) -> Result { + if let ByteStreamSource::Child(child) = self.stream { + Ok(*child) + } else { + Err(self) + } + } + + /// Collect all the bytes of the [`ByteStream`] into a [`Vec`]. + /// + /// Any trailing new lines are kept in the returned [`Vec`]. + pub fn into_bytes(self) -> Result, ShellError> { + // todo!() ctrlc + match self.stream { + ByteStreamSource::Read(mut read) => { + let mut buf = Vec::new(); + read.read_to_end(&mut buf).err_span(self.span)?; + Ok(buf) + } + ByteStreamSource::File(mut file) => { + let mut buf = Vec::new(); + file.read_to_end(&mut buf).err_span(self.span)?; + Ok(buf) + } + ByteStreamSource::Child(child) => child.into_bytes(), + } + } + + /// Collect all the bytes of the [`ByteStream`] into a [`String`]. + /// + /// The trailing new line (`\n` or `\r\n`), if any, is removed from the [`String`] prior to being returned. + /// + /// If utf-8 decoding fails, an error is returned. + pub fn into_string(self) -> Result { + let span = self.span; + let bytes = self.into_bytes()?; + let mut string = String::from_utf8(bytes).map_err(|_| ShellError::NonUtf8 { span })?; + trim_end_newline(&mut string); + Ok(string) + } + + /// Collect all the bytes of the [`ByteStream`] into a [`Value`]. + /// + /// If the collected bytes are successfully decoded as utf-8, then a [`Value::String`] is returned. + /// The trailing new line (`\n` or `\r\n`), if any, is removed from the [`String`] prior to being returned. + /// Otherwise, a [`Value::Binary`] is returned with any trailing new lines preserved. + pub fn into_value(self) -> Result { + let span = self.span; + let bytes = self.into_bytes()?; + let value = match String::from_utf8(bytes) { + Ok(mut str) => { + trim_end_newline(&mut str); + Value::string(str, span) + } + Err(err) => Value::binary(err.into_bytes(), span), + }; + Ok(value) + } + + /// Consume and drop all bytes of the [`ByteStream`]. + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`], + /// then the [`ExitStatus`] of the [`ChildProcess`] is returned. + pub fn drain(self) -> Result, ShellError> { + match self.stream { + ByteStreamSource::Read(mut read) => { + copy_with_interrupt(&mut read, &mut io::sink(), self.span, self.ctrlc.as_deref())?; + Ok(None) + } + ByteStreamSource::File(_) => Ok(None), + ByteStreamSource::Child(child) => Ok(Some(child.wait()?)), + } + } + + /// Print all bytes of the [`ByteStream`] to stdout or stderr. + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`], + /// then the [`ExitStatus`] of the [`ChildProcess`] is returned. + pub fn print(self, to_stderr: bool) -> Result, ShellError> { + if to_stderr { + self.write_to(&mut io::stderr()) + } else { + self.write_to(&mut io::stdout()) + } + } + + /// Write all bytes of the [`ByteStream`] to `dest`. + /// + /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`], + /// then the [`ExitStatus`] of the [`ChildProcess`] is returned. + pub fn write_to(self, dest: &mut impl Write) -> Result, ShellError> { + let span = self.span; + let ctrlc = self.ctrlc.as_deref(); + match self.stream { + ByteStreamSource::Read(mut read) => { + copy_with_interrupt(&mut read, dest, span, ctrlc)?; + Ok(None) + } + ByteStreamSource::File(mut file) => { + copy_with_interrupt(&mut file, dest, span, ctrlc)?; + Ok(None) + } + ByteStreamSource::Child(mut child) => { + // All `OutDest`s except `OutDest::Capture` will cause `stderr` to be `None`. + // Only `save`, `tee`, and `complete` set the stderr `OutDest` to `OutDest::Capture`, + // and those commands have proper simultaneous handling of stdout and stderr. + debug_assert!(child.stderr.is_none(), "stderr should not exist"); + + if let Some(stdout) = child.stdout.take() { + match stdout { + ChildPipe::Pipe(mut pipe) => { + copy_with_interrupt(&mut pipe, dest, span, ctrlc)?; + } + ChildPipe::Tee(mut tee) => { + copy_with_interrupt(&mut tee, dest, span, ctrlc)?; + } + } + } + Ok(Some(child.wait()?)) + } + } + } + + pub(crate) fn write_to_out_dests( + self, + stdout: &OutDest, + stderr: &OutDest, + ) -> Result, ShellError> { + let span = self.span; + let ctrlc = self.ctrlc.as_deref(); + + match self.stream { + ByteStreamSource::Read(read) => { + write_to_out_dest(read, stdout, true, span, ctrlc)?; + Ok(None) + } + ByteStreamSource::File(mut file) => { + match stdout { + OutDest::Pipe | OutDest::Capture | OutDest::Null => {} + OutDest::Inherit => { + copy_with_interrupt(&mut file, &mut io::stdout(), span, ctrlc)?; + } + OutDest::File(f) => { + copy_with_interrupt(&mut file, &mut f.as_ref(), span, ctrlc)?; + } + } + Ok(None) + } + ByteStreamSource::Child(mut child) => { + match (child.stdout.take(), child.stderr.take()) { + (Some(out), Some(err)) => { + // To avoid deadlocks, we must spawn a separate thread to wait on stderr. + thread::scope(|s| { + let err_thread = thread::Builder::new() + .name("stderr writer".into()) + .spawn_scoped(s, || match err { + ChildPipe::Pipe(pipe) => { + write_to_out_dest(pipe, stderr, false, span, ctrlc) + } + ChildPipe::Tee(tee) => { + write_to_out_dest(tee, stderr, false, span, ctrlc) + } + }) + .err_span(span); + + match out { + ChildPipe::Pipe(pipe) => { + write_to_out_dest(pipe, stdout, true, span, ctrlc) + } + ChildPipe::Tee(tee) => { + write_to_out_dest(tee, stdout, true, span, ctrlc) + } + }?; + + if let Ok(result) = err_thread?.join() { + result?; + } else { + // thread panicked, which should not happen + debug_assert!(false) + } + + Ok::<_, ShellError>(()) + })?; + } + (Some(out), None) => { + // single output stream, we can consume directly + write_to_out_dest(out, stdout, true, span, ctrlc)?; + } + (None, Some(err)) => { + // single output stream, we can consume directly + write_to_out_dest(err, stderr, false, span, ctrlc)?; + } + (None, None) => {} + } + Ok(Some(child.wait()?)) + } + } + } +} + +impl Debug for ByteStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ByteStream").finish() + } +} + +impl From for PipelineData { + fn from(stream: ByteStream) -> Self { + Self::ByteStream(stream, None) + } +} + +struct ReadIterator +where + I: Iterator, + I::Item: AsRef<[u8]>, +{ + iter: I, + cursor: Option>, +} + +impl Read for ReadIterator +where + I: Iterator, + I::Item: AsRef<[u8]>, +{ + fn read(&mut self, buf: &mut [u8]) -> io::Result { + while let Some(cursor) = self.cursor.as_mut() { + let read = cursor.read(buf)?; + if read == 0 { + self.cursor = self.iter.next().map(Cursor::new); + } else { + return Ok(read); + } + } + Ok(0) + } +} + +struct ReadResultIterator +where + I: Iterator>, + T: AsRef<[u8]>, +{ + iter: I, + cursor: Option>, +} + +impl Read for ReadResultIterator +where + I: Iterator>, + T: AsRef<[u8]>, +{ + fn read(&mut self, buf: &mut [u8]) -> io::Result { + while let Some(cursor) = self.cursor.as_mut() { + let read = cursor.read(buf)?; + if read == 0 { + self.cursor = self.iter.next().transpose()?.map(Cursor::new); + } else { + return Ok(read); + } + } + Ok(0) + } +} + +pub struct Reader { + reader: BufReader, + span: Span, + ctrlc: Option>, +} + +impl Reader { + pub fn span(&self) -> Span { + self.span + } +} + +impl Read for Reader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { + Err(ShellError::InterruptedByUser { + span: Some(self.span), + } + .into()) + } else { + self.reader.read(buf) + } + } +} + +impl BufRead for Reader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.reader.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.reader.consume(amt) + } +} + +pub struct Lines { + reader: BufReader, + span: Span, + ctrlc: Option>, +} + +impl Lines { + pub fn span(&self) -> Span { + self.span + } +} + +impl Iterator for Lines { + type Item = Result; + + fn next(&mut self) -> Option { + if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { + None + } else { + let mut buf = Vec::new(); + match self.reader.read_until(b'\n', &mut buf) { + Ok(0) => None, + Ok(_) => { + let Ok(mut string) = String::from_utf8(buf) else { + return Some(Err(ShellError::NonUtf8 { span: self.span })); + }; + trim_end_newline(&mut string); + Some(Ok(string)) + } + Err(e) => Some(Err(e.into_spanned(self.span).into())), + } + } + } +} + +pub struct Chunks { + reader: BufReader, + span: Span, + ctrlc: Option>, + leftover: Vec, +} + +impl Chunks { + pub fn span(&self) -> Span { + self.span + } +} + +impl Iterator for Chunks { + type Item = Result; + + fn next(&mut self) -> Option { + if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { + None + } else { + loop { + match self.reader.fill_buf() { + Ok(buf) => { + self.leftover.extend_from_slice(buf); + let len = buf.len(); + self.reader.consume(len); + break; + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(err) => return Some(Err(err.into_spanned(self.span).into())), + }; + } + + if self.leftover.is_empty() { + return None; + } + + match String::from_utf8(std::mem::take(&mut self.leftover)) { + Ok(str) => Some(Ok(Value::string(str, self.span))), + Err(err) => { + if err.utf8_error().error_len().is_some() { + Some(Ok(Value::binary(err.into_bytes(), self.span))) + } else { + let i = err.utf8_error().valid_up_to(); + let mut bytes = err.into_bytes(); + self.leftover = bytes.split_off(i); + let str = String::from_utf8(bytes).expect("valid utf8"); + Some(Ok(Value::string(str, self.span))) + } + } + } + } + } +} + +fn trim_end_newline(string: &mut String) { + if string.ends_with('\n') { + string.pop(); + if string.ends_with('\r') { + string.pop(); + } + } +} + +fn write_to_out_dest( + mut read: impl Read, + stream: &OutDest, + stdout: bool, + span: Span, + ctrlc: Option<&AtomicBool>, +) -> Result<(), ShellError> { + match stream { + OutDest::Pipe | OutDest::Capture => return Ok(()), + OutDest::Null => copy_with_interrupt(&mut read, &mut io::sink(), span, ctrlc), + OutDest::Inherit if stdout => { + copy_with_interrupt(&mut read, &mut io::stdout(), span, ctrlc) + } + OutDest::Inherit => copy_with_interrupt(&mut read, &mut io::stderr(), span, ctrlc), + OutDest::File(file) => copy_with_interrupt(&mut read, &mut file.as_ref(), span, ctrlc), + }?; + Ok(()) +} + +#[cfg(unix)] +pub(crate) fn convert_file>(file: impl Into) -> T { + file.into().into() +} + +#[cfg(windows)] +pub(crate) fn convert_file>(file: impl Into) -> T { + file.into().into() +} + +const DEFAULT_BUF_SIZE: usize = 8192; + +pub fn copy_with_interrupt( + reader: &mut R, + writer: &mut W, + span: Span, + interrupt: Option<&AtomicBool>, +) -> Result +where + R: Read, + W: Write, +{ + if let Some(interrupt) = interrupt { + // #[cfg(any(target_os = "linux", target_os = "android"))] + // { + // return crate::sys::kernel_copy::copy_spec(reader, writer); + // } + match generic_copy(reader, writer, span, interrupt) { + Ok(len) => { + writer.flush().err_span(span)?; + Ok(len) + } + Err(err) => { + let _ = writer.flush(); + Err(err) + } + } + } else { + match io::copy(reader, writer) { + Ok(n) => { + writer.flush().err_span(span)?; + Ok(n) + } + Err(err) => { + let _ = writer.flush(); + Err(err.into_spanned(span).into()) + } + } + } +} + +// Copied from [`std::io::copy`] +fn generic_copy( + reader: &mut R, + writer: &mut W, + span: Span, + interrupt: &AtomicBool, +) -> Result +where + R: Read, + W: Write, +{ + let buf = &mut [0; DEFAULT_BUF_SIZE]; + let mut len = 0; + loop { + if interrupt.load(Ordering::Relaxed) { + return Err(ShellError::InterruptedByUser { span: Some(span) }); + } + let n = match reader.read(buf) { + Ok(0) => break, + Ok(n) => n, + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) => return Err(e.into_spanned(span).into()), + }; + len += n; + writer.write_all(&buf[..n]).err_span(span)?; + } + Ok(len as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_chunks(data: Vec) -> Chunks + where + T: AsRef<[u8]> + Default + Send + 'static, + { + let reader = ReadIterator { + iter: data.into_iter(), + cursor: Some(Cursor::new(T::default())), + }; + Chunks { + reader: BufReader::new(SourceReader::Read(Box::new(reader))), + span: Span::test_data(), + ctrlc: None, + leftover: Vec::new(), + } + } + + #[test] + fn chunks_read_string() { + let data = vec!["Nushell", "が好きです"]; + let chunks = test_chunks(data.clone()); + let actual = chunks.collect::, _>>().unwrap(); + let expected = data.into_iter().map(Value::test_string).collect::>(); + assert_eq!(expected, actual); + } + + #[test] + fn chunks_read_string_split_utf8() { + let expected = "Nushell最高!"; + let chunks = test_chunks(vec![&b"Nushell\xe6"[..], b"\x9c\x80\xe9", b"\xab\x98!"]); + + let actual = chunks + .into_iter() + .map(|value| value.and_then(Value::into_string)) + .collect::>() + .unwrap(); + + assert_eq!(expected, actual); + } + + #[test] + fn chunks_returns_string_or_binary() { + let chunks = test_chunks(vec![b"Nushell".as_slice(), b"\x9c\x80\xe9abcd", b"efgh"]); + let actual = chunks.collect::, _>>().unwrap(); + let expected = vec![ + Value::test_string("Nushell"), + Value::test_binary(b"\x9c\x80\xe9abcd"), + Value::test_string("efgh"), + ]; + assert_eq!(actual, expected) + } +} diff --git a/crates/nu-protocol/src/pipeline_data/list_stream.rs b/crates/nu-protocol/src/pipeline/list_stream.rs similarity index 100% rename from crates/nu-protocol/src/pipeline_data/list_stream.rs rename to crates/nu-protocol/src/pipeline/list_stream.rs diff --git a/crates/nu-protocol/src/pipeline_data/metadata.rs b/crates/nu-protocol/src/pipeline/metadata.rs similarity index 100% rename from crates/nu-protocol/src/pipeline_data/metadata.rs rename to crates/nu-protocol/src/pipeline/metadata.rs diff --git a/crates/nu-protocol/src/pipeline/mod.rs b/crates/nu-protocol/src/pipeline/mod.rs new file mode 100644 index 0000000000..a018a084ed --- /dev/null +++ b/crates/nu-protocol/src/pipeline/mod.rs @@ -0,0 +1,11 @@ +pub mod byte_stream; +pub mod list_stream; +mod metadata; +mod out_dest; +mod pipeline_data; + +pub use byte_stream::*; +pub use list_stream::*; +pub use metadata::*; +pub use out_dest::*; +pub use pipeline_data::*; diff --git a/crates/nu-protocol/src/pipeline_data/out_dest.rs b/crates/nu-protocol/src/pipeline/out_dest.rs similarity index 81% rename from crates/nu-protocol/src/pipeline_data/out_dest.rs rename to crates/nu-protocol/src/pipeline/out_dest.rs index 976123e883..69955e6b0b 100644 --- a/crates/nu-protocol/src/pipeline_data/out_dest.rs +++ b/crates/nu-protocol/src/pipeline/out_dest.rs @@ -5,17 +5,17 @@ use std::{fs::File, io, process::Stdio, sync::Arc}; pub enum OutDest { /// Redirect the stdout and/or stderr of one command as the input for the next command in the pipeline. /// - /// The output pipe will be available as the `stdout` of `PipelineData::ExternalStream`. + /// The output pipe will be available as the `stdout` of [`ChildProcess`](crate::process::ChildProcess). /// /// If stdout and stderr are both set to `Pipe`, - /// then they will combined into the `stdout` of `PipelineData::ExternalStream`. + /// then they will combined into the `stdout` of [`ChildProcess`](crate::process::ChildProcess). Pipe, /// Capture output to later be collected into a [`Value`](crate::Value), `Vec`, or used in some other way. /// - /// The output stream(s) will be available in the `stdout` or `stderr` of `PipelineData::ExternalStream`. + /// The output stream(s) will be available in the `stdout` or `stderr` of [`ChildProcess`](crate::process::ChildProcess). /// /// This is similar to `Pipe` but will never combine stdout and stderr - /// or place an external command's stderr into `stdout` of `PipelineData::ExternalStream`. + /// or place an external command's stderr into `stdout` of [`ChildProcess`](crate::process::ChildProcess). Capture, /// Ignore output. /// diff --git a/crates/nu-protocol/src/pipeline/pipeline_data.rs b/crates/nu-protocol/src/pipeline/pipeline_data.rs new file mode 100644 index 0000000000..d7e58e63a3 --- /dev/null +++ b/crates/nu-protocol/src/pipeline/pipeline_data.rs @@ -0,0 +1,725 @@ +use crate::{ + ast::{Call, PathMember}, + engine::{EngineState, Stack}, + process::{ChildPipe, ChildProcess, ExitStatus}, + ByteStream, Config, ErrSpan, ListStream, OutDest, PipelineMetadata, Range, ShellError, Span, + Value, +}; +use nu_utils::{stderr_write_all_and_flush, stdout_write_all_and_flush}; +use std::{ + io::{Cursor, Read, Write}, + sync::{atomic::AtomicBool, Arc}, +}; + +const LINE_ENDING_PATTERN: &[char] = &['\r', '\n']; + +/// The foundational abstraction for input and output to commands +/// +/// This represents either a single Value or a stream of values coming into the command or leaving a command. +/// +/// A note on implementation: +/// +/// We've tried a few variations of this structure. Listing these below so we have a record. +/// +/// * We tried always assuming a stream in Nushell. This was a great 80% solution, but it had some rough edges. +/// Namely, how do you know the difference between a single string and a list of one string. How do you know +/// when to flatten the data given to you from a data source into the stream or to keep it as an unflattened +/// list? +/// +/// * We tried putting the stream into Value. This had some interesting properties as now commands "just worked +/// on values", but lead to a few unfortunate issues. +/// +/// The first is that you can't easily clone Values in a way that felt largely immutable. For example, if +/// you cloned a Value which contained a stream, and in one variable drained some part of it, then the second +/// variable would see different values based on what you did to the first. +/// +/// To make this kind of mutation thread-safe, we would have had to produce a lock for the stream, which in +/// practice would have meant always locking the stream before reading from it. But more fundamentally, it +/// felt wrong in practice that observation of a value at runtime could affect other values which happen to +/// alias the same stream. By separating these, we don't have this effect. Instead, variables could get +/// concrete list values rather than streams, and be able to view them without non-local effects. +/// +/// * A balance of the two approaches is what we've landed on: Values are thread-safe to pass, and we can stream +/// them into any sources. Streams are still available to model the infinite streams approach of original +/// Nushell. +#[derive(Debug)] +pub enum PipelineData { + Empty, + Value(Value, Option), + ListStream(ListStream, Option), + ByteStream(ByteStream, Option), +} + +impl PipelineData { + pub fn empty() -> PipelineData { + PipelineData::Empty + } + + /// create a `PipelineData::ByteStream` with proper exit_code + /// + /// It's useful to break running without raising error at user level. + pub fn new_external_stream_with_only_exit_code(exit_code: i32) -> PipelineData { + let span = Span::unknown(); + let mut child = ChildProcess::from_raw(None, None, None, span); + child.set_exit_code(exit_code); + PipelineData::ByteStream(ByteStream::child(child, span), None) + } + + pub fn metadata(&self) -> Option { + match self { + PipelineData::Empty => None, + PipelineData::Value(_, meta) + | PipelineData::ListStream(_, meta) + | PipelineData::ByteStream(_, meta) => meta.clone(), + } + } + + pub fn set_metadata(mut self, metadata: Option) -> Self { + match &mut self { + PipelineData::Empty => {} + PipelineData::Value(_, meta) + | PipelineData::ListStream(_, meta) + | PipelineData::ByteStream(_, meta) => *meta = metadata, + } + self + } + + pub fn is_nothing(&self) -> bool { + matches!(self, PipelineData::Value(Value::Nothing { .. }, ..)) + || matches!(self, PipelineData::Empty) + } + + /// PipelineData doesn't always have a Span, but we can try! + pub fn span(&self) -> Option { + match self { + PipelineData::Empty => None, + PipelineData::Value(value, ..) => Some(value.span()), + PipelineData::ListStream(stream, ..) => Some(stream.span()), + PipelineData::ByteStream(stream, ..) => Some(stream.span()), + } + } + + pub fn into_value(self, span: Span) -> Result { + match self { + PipelineData::Empty => Ok(Value::nothing(span)), + PipelineData::Value(value, ..) => Ok(value.with_span(span)), + PipelineData::ListStream(stream, ..) => Ok(stream.into_value()), + PipelineData::ByteStream(stream, ..) => stream.into_value(), + } + } + + /// Writes all values or redirects all output to the current [`OutDest`]s in `stack`. + /// + /// For [`OutDest::Pipe`] and [`OutDest::Capture`], this will return the `PipelineData` as is + /// without consuming input and without writing anything. + /// + /// For the other [`OutDest`]s, the given `PipelineData` will be completely consumed + /// and `PipelineData::Empty` will be returned. + pub fn write_to_out_dests( + self, + engine_state: &EngineState, + stack: &mut Stack, + ) -> Result { + match (self, stack.stdout()) { + (PipelineData::ByteStream(stream, ..), stdout) => { + stream.write_to_out_dests(stdout, stack.stderr())?; + } + (data, OutDest::Pipe | OutDest::Capture) => return Ok(data), + (PipelineData::Empty, ..) => {} + (PipelineData::Value(..), OutDest::Null) => {} + (PipelineData::ListStream(stream, ..), OutDest::Null) => { + // we need to drain the stream in case there are external commands in the pipeline + stream.drain()?; + } + (PipelineData::Value(value, ..), OutDest::File(file)) => { + let bytes = value_to_bytes(value)?; + let mut file = file.as_ref(); + file.write_all(&bytes)?; + file.flush()?; + } + (PipelineData::ListStream(stream, ..), OutDest::File(file)) => { + let mut file = file.as_ref(); + // use BufWriter here? + for value in stream { + let bytes = value_to_bytes(value)?; + file.write_all(&bytes)?; + file.write_all(b"\n")?; + } + file.flush()?; + } + (data @ (PipelineData::Value(..) | PipelineData::ListStream(..)), OutDest::Inherit) => { + data.print(engine_state, stack, false, false)?; + } + } + Ok(PipelineData::Empty) + } + + pub fn drain(self) -> Result, ShellError> { + match self { + PipelineData::Empty => Ok(None), + PipelineData::Value(Value::Error { error, .. }, ..) => Err(*error), + PipelineData::Value(..) => Ok(None), + PipelineData::ListStream(stream, ..) => { + stream.drain()?; + Ok(None) + } + PipelineData::ByteStream(stream, ..) => stream.drain(), + } + } + + /// Try convert from self into iterator + /// + /// It returns Err if the `self` cannot be converted to an iterator. + pub fn into_iter_strict(self, span: Span) -> Result { + Ok(PipelineIterator(match self { + PipelineData::Value(value, ..) => { + let val_span = value.span(); + match value { + Value::List { vals, .. } => PipelineIteratorInner::ListStream( + ListStream::new(vals.into_iter(), val_span, None).into_iter(), + ), + Value::Binary { val, .. } => PipelineIteratorInner::ListStream( + ListStream::new( + val.into_iter().map(move |x| Value::int(x as i64, val_span)), + val_span, + None, + ) + .into_iter(), + ), + Value::Range { val, .. } => PipelineIteratorInner::ListStream( + ListStream::new(val.into_range_iter(val_span, None), val_span, None) + .into_iter(), + ), + // Propagate errors by explicitly matching them before the final case. + Value::Error { error, .. } => return Err(*error), + other => { + return Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary, range, or byte stream".into(), + wrong_type: other.get_type().to_string(), + dst_span: span, + src_span: val_span, + }) + } + } + } + PipelineData::ListStream(stream, ..) => { + PipelineIteratorInner::ListStream(stream.into_iter()) + } + PipelineData::Empty => { + return Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary, range, or byte stream".into(), + wrong_type: "null".into(), + dst_span: span, + src_span: span, + }) + } + PipelineData::ByteStream(stream, ..) => { + if let Some(chunks) = stream.chunks() { + PipelineIteratorInner::ByteStream(chunks) + } else { + PipelineIteratorInner::Empty + } + } + })) + } + + pub fn collect_string(self, separator: &str, config: &Config) -> Result { + match self { + PipelineData::Empty => Ok(String::new()), + PipelineData::Value(value, ..) => Ok(value.to_expanded_string(separator, config)), + PipelineData::ListStream(stream, ..) => Ok(stream.into_string(separator, config)), + PipelineData::ByteStream(stream, ..) => stream.into_string(), + } + } + + /// Retrieves string from pipeline data. + /// + /// As opposed to `collect_string` this raises error rather than converting non-string values. + /// The `span` will be used if `ListStream` is encountered since it doesn't carry a span. + pub fn collect_string_strict( + self, + span: Span, + ) -> Result<(String, Span, Option), ShellError> { + match self { + PipelineData::Empty => Ok((String::new(), span, None)), + PipelineData::Value(Value::String { val, .. }, metadata) => Ok((val, span, metadata)), + PipelineData::Value(val, ..) => Err(ShellError::TypeMismatch { + err_message: "string".into(), + span: val.span(), + }), + PipelineData::ListStream(..) => Err(ShellError::TypeMismatch { + err_message: "string".into(), + span, + }), + PipelineData::ByteStream(stream, metadata) => { + let span = stream.span(); + Ok((stream.into_string()?, span, metadata)) + } + } + } + + pub fn follow_cell_path( + self, + cell_path: &[PathMember], + head: Span, + insensitive: bool, + ) -> Result { + match self { + // FIXME: there are probably better ways of doing this + PipelineData::ListStream(stream, ..) => Value::list(stream.into_iter().collect(), head) + .follow_cell_path(cell_path, insensitive), + PipelineData::Value(v, ..) => v.follow_cell_path(cell_path, insensitive), + PipelineData::Empty => Err(ShellError::IncompatiblePathAccess { + type_name: "empty pipeline".to_string(), + span: head, + }), + PipelineData::ByteStream(stream, ..) => Err(ShellError::IncompatiblePathAccess { + type_name: "byte stream".to_string(), + span: stream.span(), + }), + } + } + + /// Simplified mapper to help with simple values also. For full iterator support use `.into_iter()` instead + pub fn map( + self, + mut f: F, + ctrlc: Option>, + ) -> Result + where + Self: Sized, + F: FnMut(Value) -> Value + 'static + Send, + { + match self { + PipelineData::Value(value, ..) => { + let span = value.span(); + match value { + Value::List { vals, .. } => { + Ok(vals.into_iter().map(f).into_pipeline_data(span, ctrlc)) + } + Value::Range { val, .. } => Ok(val + .into_range_iter(span, ctrlc.clone()) + .map(f) + .into_pipeline_data(span, ctrlc)), + value => match f(value) { + Value::Error { error, .. } => Err(*error), + v => Ok(v.into_pipeline_data()), + }, + } + } + PipelineData::Empty => Ok(PipelineData::Empty), + PipelineData::ListStream(stream, ..) => { + Ok(PipelineData::ListStream(stream.map(f), None)) + } + PipelineData::ByteStream(stream, ..) => { + // TODO: is this behavior desired / correct ? + let span = stream.span(); + match String::from_utf8(stream.into_bytes()?) { + Ok(mut str) => { + str.truncate(str.trim_end_matches(LINE_ENDING_PATTERN).len()); + Ok(f(Value::string(str, span)).into_pipeline_data()) + } + Err(err) => Ok(f(Value::binary(err.into_bytes(), span)).into_pipeline_data()), + } + } + } + } + + /// Simplified flatmapper. For full iterator support use `.into_iter()` instead + pub fn flat_map( + self, + mut f: F, + ctrlc: Option>, + ) -> Result + where + Self: Sized, + U: IntoIterator + 'static, + ::IntoIter: 'static + Send, + F: FnMut(Value) -> U + 'static + Send, + { + match self { + PipelineData::Empty => Ok(PipelineData::Empty), + PipelineData::Value(value, ..) => { + let span = value.span(); + match value { + Value::List { vals, .. } => { + Ok(vals.into_iter().flat_map(f).into_pipeline_data(span, ctrlc)) + } + Value::Range { val, .. } => Ok(val + .into_range_iter(span, ctrlc.clone()) + .flat_map(f) + .into_pipeline_data(span, ctrlc)), + value => Ok(f(value).into_iter().into_pipeline_data(span, ctrlc)), + } + } + PipelineData::ListStream(stream, ..) => { + Ok(stream.modify(|iter| iter.flat_map(f)).into()) + } + PipelineData::ByteStream(stream, ..) => { + // TODO: is this behavior desired / correct ? + let span = stream.span(); + match String::from_utf8(stream.into_bytes()?) { + Ok(mut str) => { + str.truncate(str.trim_end_matches(LINE_ENDING_PATTERN).len()); + Ok(f(Value::string(str, span)) + .into_iter() + .into_pipeline_data(span, ctrlc)) + } + Err(err) => Ok(f(Value::binary(err.into_bytes(), span)) + .into_iter() + .into_pipeline_data(span, ctrlc)), + } + } + } + } + + pub fn filter( + self, + mut f: F, + ctrlc: Option>, + ) -> Result + where + Self: Sized, + F: FnMut(&Value) -> bool + 'static + Send, + { + match self { + PipelineData::Empty => Ok(PipelineData::Empty), + PipelineData::Value(value, ..) => { + let span = value.span(); + match value { + Value::List { vals, .. } => { + Ok(vals.into_iter().filter(f).into_pipeline_data(span, ctrlc)) + } + Value::Range { val, .. } => Ok(val + .into_range_iter(span, ctrlc.clone()) + .filter(f) + .into_pipeline_data(span, ctrlc)), + value => { + if f(&value) { + Ok(value.into_pipeline_data()) + } else { + Ok(Value::nothing(span).into_pipeline_data()) + } + } + } + } + PipelineData::ListStream(stream, ..) => Ok(stream.modify(|iter| iter.filter(f)).into()), + PipelineData::ByteStream(stream, ..) => { + // TODO: is this behavior desired / correct ? + let span = stream.span(); + let value = match String::from_utf8(stream.into_bytes()?) { + Ok(mut str) => { + str.truncate(str.trim_end_matches(LINE_ENDING_PATTERN).len()); + Value::string(str, span) + } + Err(err) => Value::binary(err.into_bytes(), span), + }; + if f(&value) { + Ok(value.into_pipeline_data()) + } else { + Ok(Value::nothing(span).into_pipeline_data()) + } + } + } + } + + /// Try to catch the external command exit status and detect if it failed. + /// + /// This is useful for external commands with semicolon, we can detect errors early to avoid + /// commands after the semicolon running. + /// + /// Returns `self` and a flag that indicates if the external command run failed. If `self` is + /// not [`PipelineData::ByteStream`], the flag will be `false`. + /// + /// Currently this will consume an external command to completion. + pub fn check_external_failed(self) -> Result<(Self, bool), ShellError> { + if let PipelineData::ByteStream(stream, metadata) = self { + let span = stream.span(); + match stream.into_child() { + Ok(mut child) => { + // Only check children without stdout. This means that nothing + // later in the pipeline can possibly consume output from this external command. + if child.stdout.is_none() { + // Note: + // In run-external's implementation detail, the result sender thread + // send out stderr message first, then stdout message, then exit_code. + // + // In this clause, we already make sure that `stdout` is None + // But not the case of `stderr`, so if `stderr` is not None + // We need to consume stderr message before reading external commands' exit code. + // + // Or we'll never have a chance to read exit_code if stderr producer produce too much stderr message. + // So we consume stderr stream and rebuild it. + let stderr = child + .stderr + .take() + .map(|mut stderr| { + let mut buf = Vec::new(); + stderr.read_to_end(&mut buf).err_span(span)?; + Ok::<_, ShellError>(buf) + }) + .transpose()?; + + let code = child.wait()?.code(); + let mut child = ChildProcess::from_raw(None, None, None, span); + if let Some(stderr) = stderr { + child.stderr = Some(ChildPipe::Tee(Box::new(Cursor::new(stderr)))); + } + child.set_exit_code(code); + let stream = ByteStream::child(child, span); + Ok((PipelineData::ByteStream(stream, metadata), code != 0)) + } else { + let stream = ByteStream::child(child, span); + Ok((PipelineData::ByteStream(stream, metadata), false)) + } + } + Err(stream) => Ok((PipelineData::ByteStream(stream, metadata), false)), + } + } else { + Ok((self, false)) + } + } + + /// Try to convert Value from Value::Range to Value::List. + /// This is useful to expand Value::Range into array notation, specifically when + /// converting `to json` or `to nuon`. + /// `1..3 | to XX -> [1,2,3]` + pub fn try_expand_range(self) -> Result { + match self { + PipelineData::Value(v, metadata) => { + let span = v.span(); + match v { + Value::Range { val, .. } => { + match *val { + Range::IntRange(range) => { + if range.is_unbounded() { + return Err(ShellError::GenericError { + error: "Cannot create range".into(), + msg: "Unbounded ranges are not allowed when converting to this format".into(), + span: Some(span), + help: Some("Consider using ranges with valid start and end point.".into()), + inner: vec![], + }); + } + } + Range::FloatRange(range) => { + if range.is_unbounded() { + return Err(ShellError::GenericError { + error: "Cannot create range".into(), + msg: "Unbounded ranges are not allowed when converting to this format".into(), + span: Some(span), + help: Some("Consider using ranges with valid start and end point.".into()), + inner: vec![], + }); + } + } + } + let range_values: Vec = val.into_range_iter(span, None).collect(); + Ok(PipelineData::Value(Value::list(range_values, span), None)) + } + x => Ok(PipelineData::Value(x, metadata)), + } + } + _ => Ok(self), + } + } + + /// Consume and print self data immediately. + /// + /// `no_newline` controls if we need to attach newline character to output. + /// `to_stderr` controls if data is output to stderr, when the value is false, the data is output to stdout. + pub fn print( + self, + engine_state: &EngineState, + stack: &mut Stack, + no_newline: bool, + to_stderr: bool, + ) -> Result, ShellError> { + if let PipelineData::ByteStream(stream, ..) = self { + stream.print(to_stderr) + } else { + // If the table function is in the declarations, then we can use it + // to create the table value that will be printed in the terminal + if let Some(decl_id) = engine_state.table_decl_id { + let command = engine_state.get_decl(decl_id); + if command.get_block_id().is_some() { + self.write_all_and_flush(engine_state, no_newline, to_stderr)?; + } else { + let call = Call::new(Span::new(0, 0)); + let table = command.run(engine_state, stack, &call, self)?; + table.write_all_and_flush(engine_state, no_newline, to_stderr)?; + } + } else { + self.write_all_and_flush(engine_state, no_newline, to_stderr)?; + } + Ok(None) + } + } + + fn write_all_and_flush( + self, + engine_state: &EngineState, + no_newline: bool, + to_stderr: bool, + ) -> Result<(), ShellError> { + let config = engine_state.get_config(); + for item in self { + let mut out = if let Value::Error { error, .. } = item { + return Err(*error); + } else { + item.to_expanded_string("\n", config) + }; + + if !no_newline { + out.push('\n'); + } + + if to_stderr { + stderr_write_all_and_flush(out)? + } else { + stdout_write_all_and_flush(out)? + } + } + + Ok(()) + } +} + +enum PipelineIteratorInner { + Empty, + Value(Value), + ListStream(crate::list_stream::IntoIter), + ByteStream(crate::byte_stream::Chunks), +} + +pub struct PipelineIterator(PipelineIteratorInner); + +impl IntoIterator for PipelineData { + type Item = Value; + + type IntoIter = PipelineIterator; + + fn into_iter(self) -> Self::IntoIter { + PipelineIterator(match self { + PipelineData::Empty => PipelineIteratorInner::Empty, + PipelineData::Value(value, ..) => { + let span = value.span(); + match value { + Value::List { vals, .. } => PipelineIteratorInner::ListStream( + ListStream::new(vals.into_iter(), span, None).into_iter(), + ), + Value::Range { val, .. } => PipelineIteratorInner::ListStream( + ListStream::new(val.into_range_iter(span, None), span, None).into_iter(), + ), + x => PipelineIteratorInner::Value(x), + } + } + PipelineData::ListStream(stream, ..) => { + PipelineIteratorInner::ListStream(stream.into_iter()) + } + PipelineData::ByteStream(stream, ..) => stream.chunks().map_or( + PipelineIteratorInner::Empty, + PipelineIteratorInner::ByteStream, + ), + }) + } +} + +impl Iterator for PipelineIterator { + type Item = Value; + + fn next(&mut self) -> Option { + match &mut self.0 { + PipelineIteratorInner::Empty => None, + PipelineIteratorInner::Value(Value::Nothing { .. }, ..) => None, + PipelineIteratorInner::Value(v, ..) => Some(std::mem::take(v)), + PipelineIteratorInner::ListStream(stream, ..) => stream.next(), + PipelineIteratorInner::ByteStream(stream) => stream.next().map(|x| match x { + Ok(x) => x, + Err(err) => Value::error( + err, + Span::unknown(), //FIXME: unclear where this span should come from + ), + }), + } + } +} + +pub trait IntoPipelineData { + fn into_pipeline_data(self) -> PipelineData; + + fn into_pipeline_data_with_metadata( + self, + metadata: impl Into>, + ) -> PipelineData; +} + +impl IntoPipelineData for V +where + V: Into, +{ + fn into_pipeline_data(self) -> PipelineData { + PipelineData::Value(self.into(), None) + } + + fn into_pipeline_data_with_metadata( + self, + metadata: impl Into>, + ) -> PipelineData { + PipelineData::Value(self.into(), metadata.into()) + } +} + +pub trait IntoInterruptiblePipelineData { + fn into_pipeline_data(self, span: Span, ctrlc: Option>) -> PipelineData; + fn into_pipeline_data_with_metadata( + self, + span: Span, + ctrlc: Option>, + metadata: impl Into>, + ) -> PipelineData; +} + +impl IntoInterruptiblePipelineData for I +where + I: IntoIterator + Send + 'static, + I::IntoIter: Send + 'static, + ::Item: Into, +{ + fn into_pipeline_data(self, span: Span, ctrlc: Option>) -> PipelineData { + ListStream::new(self.into_iter().map(Into::into), span, ctrlc).into() + } + + fn into_pipeline_data_with_metadata( + self, + span: Span, + ctrlc: Option>, + metadata: impl Into>, + ) -> PipelineData { + PipelineData::ListStream( + ListStream::new(self.into_iter().map(Into::into), span, ctrlc), + metadata.into(), + ) + } +} + +fn value_to_bytes(value: Value) -> Result, ShellError> { + let bytes = match value { + Value::String { val, .. } => val.into_bytes(), + Value::Binary { val, .. } => val, + Value::List { vals, .. } => { + let val = vals + .into_iter() + .map(Value::coerce_into_string) + .collect::, ShellError>>()? + .join("\n") + + "\n"; + + val.into_bytes() + } + // Propagate errors by explicitly matching them before the final case. + Value::Error { error, .. } => return Err(*error), + value => value.coerce_into_string()?.into_bytes(), + }; + Ok(bytes) +} diff --git a/crates/nu-protocol/src/pipeline_data/mod.rs b/crates/nu-protocol/src/pipeline_data/mod.rs deleted file mode 100644 index 71c667fa70..0000000000 --- a/crates/nu-protocol/src/pipeline_data/mod.rs +++ /dev/null @@ -1,1211 +0,0 @@ -pub mod list_stream; -mod metadata; -mod out_dest; -mod raw_stream; - -pub use list_stream::{ListStream, ValueIterator}; -pub use metadata::*; -pub use out_dest::*; -pub use raw_stream::*; - -use crate::{ - ast::{Call, PathMember}, - engine::{EngineState, Stack, StateWorkingSet}, - format_error, Config, Range, ShellError, Span, Value, -}; -use nu_utils::{stderr_write_all_and_flush, stdout_write_all_and_flush}; -use std::{ - io::{self, Cursor, Read, Write}, - sync::{atomic::AtomicBool, Arc}, - thread, -}; - -const LINE_ENDING_PATTERN: &[char] = &['\r', '\n']; - -/// The foundational abstraction for input and output to commands -/// -/// This represents either a single Value or a stream of values coming into the command or leaving a command. -/// -/// A note on implementation: -/// -/// We've tried a few variations of this structure. Listing these below so we have a record. -/// -/// * We tried always assuming a stream in Nushell. This was a great 80% solution, but it had some rough edges. -/// Namely, how do you know the difference between a single string and a list of one string. How do you know -/// when to flatten the data given to you from a data source into the stream or to keep it as an unflattened -/// list? -/// -/// * We tried putting the stream into Value. This had some interesting properties as now commands "just worked -/// on values", but lead to a few unfortunate issues. -/// -/// The first is that you can't easily clone Values in a way that felt largely immutable. For example, if -/// you cloned a Value which contained a stream, and in one variable drained some part of it, then the second -/// variable would see different values based on what you did to the first. -/// -/// To make this kind of mutation thread-safe, we would have had to produce a lock for the stream, which in -/// practice would have meant always locking the stream before reading from it. But more fundamentally, it -/// felt wrong in practice that observation of a value at runtime could affect other values which happen to -/// alias the same stream. By separating these, we don't have this effect. Instead, variables could get -/// concrete list values rather than streams, and be able to view them without non-local effects. -/// -/// * A balance of the two approaches is what we've landed on: Values are thread-safe to pass, and we can stream -/// them into any sources. Streams are still available to model the infinite streams approach of original -/// Nushell. -#[derive(Debug)] -pub enum PipelineData { - Value(Value, Option), - ListStream(ListStream, Option), - ExternalStream { - stdout: Option, - stderr: Option, - exit_code: Option, - span: Span, - metadata: Option, - trim_end_newline: bool, - }, - Empty, -} - -impl PipelineData { - pub fn new_with_metadata(metadata: Option, span: Span) -> PipelineData { - PipelineData::Value(Value::nothing(span), metadata) - } - - /// create a `PipelineData::ExternalStream` with proper exit_code - /// - /// It's useful to break running without raising error at user level. - pub fn new_external_stream_with_only_exit_code(exit_code: i64) -> PipelineData { - PipelineData::ExternalStream { - stdout: None, - stderr: None, - exit_code: Some(ListStream::new( - [Value::int(exit_code, Span::unknown())].into_iter(), - Span::unknown(), - None, - )), - span: Span::unknown(), - metadata: None, - trim_end_newline: false, - } - } - - pub fn empty() -> PipelineData { - PipelineData::Empty - } - - pub fn metadata(&self) -> Option { - match self { - PipelineData::ListStream(_, x) => x.clone(), - PipelineData::ExternalStream { metadata: x, .. } => x.clone(), - PipelineData::Value(_, x) => x.clone(), - PipelineData::Empty => None, - } - } - - pub fn set_metadata(mut self, metadata: Option) -> Self { - match &mut self { - PipelineData::ListStream(_, x) => *x = metadata, - PipelineData::ExternalStream { metadata: x, .. } => *x = metadata, - PipelineData::Value(_, x) => *x = metadata, - PipelineData::Empty => {} - } - - self - } - - pub fn is_nothing(&self) -> bool { - matches!(self, PipelineData::Value(Value::Nothing { .. }, ..)) - || matches!(self, PipelineData::Empty) - } - - /// PipelineData doesn't always have a Span, but we can try! - pub fn span(&self) -> Option { - match self { - PipelineData::ListStream(stream, ..) => Some(stream.span()), - PipelineData::ExternalStream { span, .. } => Some(*span), - PipelineData::Value(v, _) => Some(v.span()), - PipelineData::Empty => None, - } - } - - pub fn into_value(self, span: Span) -> Value { - match self { - PipelineData::Empty => Value::nothing(span), - PipelineData::Value(Value::Nothing { .. }, ..) => Value::nothing(span), - PipelineData::Value(v, ..) => v.with_span(span), - PipelineData::ListStream(s, ..) => Value::list( - s.into_iter().collect(), - span, // FIXME? - ), - PipelineData::ExternalStream { - stdout: None, - exit_code, - .. - } => { - // Make sure everything has finished - if let Some(exit_code) = exit_code { - let _: Vec<_> = exit_code.into_iter().collect(); - } - Value::nothing(span) - } - PipelineData::ExternalStream { - stdout: Some(mut s), - exit_code, - trim_end_newline, - .. - } => { - let mut items = vec![]; - - for val in &mut s { - match val { - Ok(val) => { - items.push(val); - } - Err(e) => { - return Value::error(e, span); - } - } - } - - // Make sure everything has finished - if let Some(exit_code) = exit_code { - let _: Vec<_> = exit_code.into_iter().collect(); - } - - // NOTE: currently trim-end-newline only handles for string output. - // For binary, user might need origin data. - if s.is_binary { - let mut output = vec![]; - for item in items { - match item.coerce_into_binary() { - Ok(item) => { - output.extend(item); - } - Err(err) => { - return Value::error(err, span); - } - } - } - - Value::binary( - output, span, // FIXME? - ) - } else { - let mut output = String::new(); - for item in items { - match item.coerce_into_string() { - Ok(s) => output.push_str(&s), - Err(err) => { - return Value::error(err, span); - } - } - } - if trim_end_newline { - output.truncate(output.trim_end_matches(LINE_ENDING_PATTERN).len()) - } - Value::string( - output, span, // FIXME? - ) - } - } - } - } - - /// Writes all values or redirects all output to the current [`OutDest`]s in `stack`. - /// - /// For [`OutDest::Pipe`] and [`OutDest::Capture`], this will return the `PipelineData` as is - /// without consuming input and without writing anything. - /// - /// For the other [`OutDest`]s, the given `PipelineData` will be completely consumed - /// and `PipelineData::Empty` will be returned. - pub fn write_to_out_dests( - self, - engine_state: &EngineState, - stack: &mut Stack, - ) -> Result { - match (self, stack.stdout()) { - ( - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - }, - _, - ) => { - fn needs_redirect( - stream: Option, - out_dest: &OutDest, - ) -> Result> { - match (stream, out_dest) { - (Some(stream), OutDest::Pipe | OutDest::Capture) => Err(Some(stream)), - (Some(stream), _) => Ok(stream), - (None, _) => Err(None), - } - } - - let (stdout, stderr) = match ( - needs_redirect(stdout, stack.stdout()), - needs_redirect(stderr, stack.stderr()), - ) { - (Ok(stdout), Ok(stderr)) => { - // We need to redirect both stdout and stderr - - // To avoid deadlocks, we must spawn a separate thread to wait on stderr. - let err_thread = { - let err = stack.stderr().clone(); - std::thread::Builder::new() - .spawn(move || consume_child_output(stderr, &err)) - }; - - consume_child_output(stdout, stack.stdout())?; - - match err_thread?.join() { - Ok(result) => result?, - Err(err) => { - return Err(ShellError::GenericError { - error: "Error consuming external command stderr".into(), - msg: format! {"{err:?}"}, - span: Some(span), - help: None, - inner: Vec::new(), - }) - } - } - - (None, None) - } - (Ok(stdout), Err(stderr)) => { - // single output stream, we can consume directly - consume_child_output(stdout, stack.stdout())?; - (None, stderr) - } - (Err(stdout), Ok(stderr)) => { - // single output stream, we can consume directly - consume_child_output(stderr, stack.stderr())?; - (stdout, None) - } - (Err(stdout), Err(stderr)) => (stdout, stderr), - }; - - Ok(PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - span, - metadata, - trim_end_newline, - }) - } - (data, OutDest::Pipe | OutDest::Capture) => Ok(data), - (PipelineData::Empty, _) => Ok(PipelineData::Empty), - (PipelineData::Value(_, _), OutDest::Null) => Ok(PipelineData::Empty), - (PipelineData::ListStream(stream, _), OutDest::Null) => { - // we need to drain the stream in case there are external commands in the pipeline - stream.drain()?; - Ok(PipelineData::Empty) - } - (PipelineData::Value(value, _), OutDest::File(file)) => { - let bytes = value_to_bytes(value)?; - let mut file = file.try_clone()?; - file.write_all(&bytes)?; - file.flush()?; - Ok(PipelineData::Empty) - } - (PipelineData::ListStream(stream, _), OutDest::File(file)) => { - let mut file = file.try_clone()?; - // use BufWriter here? - for value in stream { - let bytes = value_to_bytes(value)?; - file.write_all(&bytes)?; - file.write_all(b"\n")?; - } - file.flush()?; - Ok(PipelineData::Empty) - } - ( - data @ (PipelineData::Value(_, _) | PipelineData::ListStream(_, _)), - OutDest::Inherit, - ) => { - let config = engine_state.get_config(); - - if let Some(decl_id) = engine_state.table_decl_id { - let command = engine_state.get_decl(decl_id); - if command.get_block_id().is_some() { - data.write_all_and_flush(engine_state, config, false, false)?; - } else { - let call = Call::new(Span::unknown()); - let stack = &mut stack.start_capture(); - let table = command.run(engine_state, stack, &call, data)?; - table.write_all_and_flush(engine_state, config, false, false)?; - } - } else { - data.write_all_and_flush(engine_state, config, false, false)?; - }; - Ok(PipelineData::Empty) - } - } - } - - pub fn drain(self) -> Result<(), ShellError> { - match self { - PipelineData::Value(Value::Error { error, .. }, _) => Err(*error), - PipelineData::Value(_, _) => Ok(()), - PipelineData::ListStream(stream, _) => stream.drain(), - PipelineData::ExternalStream { stdout, stderr, .. } => { - if let Some(stdout) = stdout { - stdout.drain()?; - } - - if let Some(stderr) = stderr { - stderr.drain()?; - } - - Ok(()) - } - PipelineData::Empty => Ok(()), - } - } - - pub fn drain_with_exit_code(self) -> Result { - match self { - PipelineData::Value(Value::Error { error, .. }, _) => Err(*error), - PipelineData::Value(_, _) => Ok(0), - PipelineData::ListStream(stream, _) => { - stream.drain()?; - Ok(0) - } - PipelineData::ExternalStream { - stdout, - stderr, - exit_code, - .. - } => { - if let Some(stdout) = stdout { - stdout.drain()?; - } - - if let Some(stderr) = stderr { - stderr.drain()?; - } - - if let Some(exit_code) = exit_code { - let result = drain_exit_code(exit_code)?; - Ok(result) - } else { - Ok(0) - } - } - PipelineData::Empty => Ok(0), - } - } - - /// Try convert from self into iterator - /// - /// It returns Err if the `self` cannot be converted to an iterator. - pub fn into_iter_strict(self, span: Span) -> Result { - Ok(PipelineIterator(match self { - PipelineData::Value(value, ..) => { - let val_span = value.span(); - match value { - Value::List { vals, .. } => PipelineIteratorInner::ListStream( - ListStream::new(vals.into_iter(), val_span, None).into_iter(), - ), - Value::Binary { val, .. } => PipelineIteratorInner::ListStream( - ListStream::new( - val.into_iter().map(move |x| Value::int(x as i64, val_span)), - val_span, - None, - ) - .into_iter(), - ), - Value::Range { val, .. } => PipelineIteratorInner::ListStream( - ListStream::new(val.into_range_iter(value.span(), None), val_span, None) - .into_iter(), - ), - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => return Err(*error), - other => { - return Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary, raw data or range".into(), - wrong_type: other.get_type().to_string(), - dst_span: span, - src_span: val_span, - }) - } - } - } - PipelineData::ListStream(stream, ..) => { - PipelineIteratorInner::ListStream(stream.into_iter()) - } - PipelineData::Empty => { - return Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary, raw data or range".into(), - wrong_type: "null".into(), - dst_span: span, - src_span: span, - }) - } - PipelineData::ExternalStream { - stdout: Some(stdout), - .. - } => PipelineIteratorInner::ExternalStream(stdout), - PipelineData::ExternalStream { stdout: None, .. } => PipelineIteratorInner::Empty, - })) - } - - pub fn collect_string(self, separator: &str, config: &Config) -> Result { - match self { - PipelineData::Empty => Ok(String::new()), - PipelineData::Value(v, ..) => Ok(v.to_expanded_string(separator, config)), - PipelineData::ListStream(s, ..) => Ok(s.into_string(separator, config)), - PipelineData::ExternalStream { stdout: None, .. } => Ok(String::new()), - PipelineData::ExternalStream { - stdout: Some(s), - trim_end_newline, - .. - } => { - let mut output = String::new(); - - for val in s { - output.push_str(&val?.coerce_into_string()?); - } - if trim_end_newline { - output.truncate(output.trim_end_matches(LINE_ENDING_PATTERN).len()); - } - Ok(output) - } - } - } - - /// Retrieves string from pipeline data. - /// - /// As opposed to `collect_string` this raises error rather than converting non-string values. - /// The `span` will be used if `ListStream` is encountered since it doesn't carry a span. - pub fn collect_string_strict( - self, - span: Span, - ) -> Result<(String, Span, Option), ShellError> { - match self { - PipelineData::Empty => Ok((String::new(), span, None)), - PipelineData::Value(Value::String { val, .. }, metadata) => Ok((val, span, metadata)), - PipelineData::Value(val, _) => Err(ShellError::TypeMismatch { - err_message: "string".into(), - span: val.span(), - }), - PipelineData::ListStream(_, _) => Err(ShellError::TypeMismatch { - err_message: "string".into(), - span, - }), - PipelineData::ExternalStream { - stdout: None, - metadata, - span, - .. - } => Ok((String::new(), span, metadata)), - PipelineData::ExternalStream { - stdout: Some(stdout), - metadata, - span, - .. - } => Ok((stdout.into_string()?.item, span, metadata)), - } - } - - pub fn follow_cell_path( - self, - cell_path: &[PathMember], - head: Span, - insensitive: bool, - ) -> Result { - match self { - // FIXME: there are probably better ways of doing this - PipelineData::ListStream(stream, ..) => Value::list(stream.into_iter().collect(), head) - .follow_cell_path(cell_path, insensitive), - PipelineData::Value(v, ..) => v.follow_cell_path(cell_path, insensitive), - PipelineData::Empty => Err(ShellError::IncompatiblePathAccess { - type_name: "empty pipeline".to_string(), - span: head, - }), - PipelineData::ExternalStream { span, .. } => Err(ShellError::IncompatiblePathAccess { - type_name: "external stream".to_string(), - span, - }), - } - } - - /// Simplified mapper to help with simple values also. For full iterator support use `.into_iter()` instead - pub fn map( - self, - mut f: F, - ctrlc: Option>, - ) -> Result - where - Self: Sized, - F: FnMut(Value) -> Value + 'static + Send, - { - match self { - PipelineData::Value(value, ..) => { - let span = value.span(); - match value { - Value::List { vals, .. } => { - Ok(vals.into_iter().map(f).into_pipeline_data(span, ctrlc)) - } - Value::Range { val, .. } => Ok(val - .into_range_iter(span, ctrlc.clone()) - .map(f) - .into_pipeline_data(span, ctrlc)), - value => match f(value) { - Value::Error { error, .. } => Err(*error), - v => Ok(v.into_pipeline_data()), - }, - } - } - PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::ListStream(stream, ..) => { - Ok(PipelineData::ListStream(stream.map(f), None)) - } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::empty()), - PipelineData::ExternalStream { - stdout: Some(stream), - trim_end_newline, - .. - } => { - let collected = stream.into_bytes()?; - - if let Ok(mut st) = String::from_utf8(collected.clone().item) { - if trim_end_newline { - st.truncate(st.trim_end_matches(LINE_ENDING_PATTERN).len()); - } - Ok(f(Value::string(st, collected.span)).into_pipeline_data()) - } else { - Ok(f(Value::binary(collected.item, collected.span)).into_pipeline_data()) - } - } - } - } - - /// Simplified flatmapper. For full iterator support use `.into_iter()` instead - pub fn flat_map( - self, - mut f: F, - ctrlc: Option>, - ) -> Result - where - Self: Sized, - U: IntoIterator + 'static, - ::IntoIter: 'static + Send, - F: FnMut(Value) -> U + 'static + Send, - { - match self { - PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::Value(value, ..) => { - let span = value.span(); - match value { - Value::List { vals, .. } => { - Ok(vals.into_iter().flat_map(f).into_pipeline_data(span, ctrlc)) - } - Value::Range { val, .. } => Ok(val - .into_range_iter(span, ctrlc.clone()) - .flat_map(f) - .into_pipeline_data(span, ctrlc)), - value => Ok(f(value).into_iter().into_pipeline_data(span, ctrlc)), - } - } - PipelineData::ListStream(stream, ..) => { - Ok(stream.modify(|iter| iter.flat_map(f)).into()) - } - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::Empty), - PipelineData::ExternalStream { - stdout: Some(stream), - span, - trim_end_newline, - .. - } => { - let collected = stream.into_bytes()?; - - if let Ok(mut st) = String::from_utf8(collected.clone().item) { - if trim_end_newline { - st.truncate(st.trim_end_matches(LINE_ENDING_PATTERN).len()) - } - Ok(f(Value::string(st, collected.span)) - .into_iter() - .into_pipeline_data(span, ctrlc)) - } else { - Ok(f(Value::binary(collected.item, collected.span)) - .into_iter() - .into_pipeline_data(span, ctrlc)) - } - } - } - } - - pub fn filter( - self, - mut f: F, - ctrlc: Option>, - ) -> Result - where - Self: Sized, - F: FnMut(&Value) -> bool + 'static + Send, - { - match self { - PipelineData::Empty => Ok(PipelineData::Empty), - PipelineData::Value(value, ..) => { - let span = value.span(); - match value { - Value::List { vals, .. } => { - Ok(vals.into_iter().filter(f).into_pipeline_data(span, ctrlc)) - } - Value::Range { val, .. } => Ok(val - .into_range_iter(span, ctrlc.clone()) - .filter(f) - .into_pipeline_data(span, ctrlc)), - value => { - if f(&value) { - Ok(value.into_pipeline_data()) - } else { - Ok(Value::nothing(span).into_pipeline_data()) - } - } - } - } - PipelineData::ListStream(stream, ..) => Ok(stream.modify(|iter| iter.filter(f)).into()), - PipelineData::ExternalStream { stdout: None, .. } => Ok(PipelineData::Empty), - PipelineData::ExternalStream { - stdout: Some(stream), - trim_end_newline, - .. - } => { - let collected = stream.into_bytes()?; - - if let Ok(mut st) = String::from_utf8(collected.clone().item) { - if trim_end_newline { - st.truncate(st.trim_end_matches(LINE_ENDING_PATTERN).len()) - } - let v = Value::string(st, collected.span); - - if f(&v) { - Ok(v.into_pipeline_data()) - } else { - Ok(PipelineData::new_with_metadata(None, collected.span)) - } - } else { - let v = Value::binary(collected.item, collected.span); - - if f(&v) { - Ok(v.into_pipeline_data()) - } else { - Ok(PipelineData::new_with_metadata(None, collected.span)) - } - } - } - } - } - - /// Try to catch the external stream exit status and detect if it failed. - /// - /// This is useful for external commands with semicolon, we can detect errors early to avoid - /// commands after the semicolon running. - /// - /// Returns `self` and a flag that indicates if the external stream run failed. If `self` is - /// not [`PipelineData::ExternalStream`], the flag will be `false`. - /// - /// Currently this will consume an external stream to completion. - pub fn check_external_failed(self) -> (Self, bool) { - let mut failed_to_run = false; - // Only need ExternalStream without redirecting output. - // It indicates we have no more commands to execute currently. - if let PipelineData::ExternalStream { - stdout: None, - stderr, - mut exit_code, - span, - metadata, - trim_end_newline, - } = self - { - let exit_code = exit_code.take(); - - // Note: - // In run-external's implementation detail, the result sender thread - // send out stderr message first, then stdout message, then exit_code. - // - // In this clause, we already make sure that `stdout` is None - // But not the case of `stderr`, so if `stderr` is not None - // We need to consume stderr message before reading external commands' exit code. - // - // Or we'll never have a chance to read exit_code if stderr producer produce too much stderr message. - // So we consume stderr stream and rebuild it. - let stderr = stderr.map(|stderr_stream| { - let stderr_ctrlc = stderr_stream.ctrlc.clone(); - let stderr_span = stderr_stream.span; - let stderr_bytes = stderr_stream - .into_bytes() - .map(|bytes| bytes.item) - .unwrap_or_default(); - RawStream::new( - Box::new(std::iter::once(Ok(stderr_bytes))), - stderr_ctrlc, - stderr_span, - None, - ) - }); - - match exit_code { - Some(exit_code_stream) => { - let exit_code: Vec = exit_code_stream.into_iter().collect(); - if let Some(Value::Int { val: code, .. }) = exit_code.last() { - // if exit_code is not 0, it indicates error occurred, return back Err. - if *code != 0 { - failed_to_run = true; - } - } - ( - PipelineData::ExternalStream { - stdout: None, - stderr, - exit_code: Some(ListStream::new(exit_code.into_iter(), span, None)), - span, - metadata, - trim_end_newline, - }, - failed_to_run, - ) - } - None => ( - PipelineData::ExternalStream { - stdout: None, - stderr, - exit_code: None, - span, - metadata, - trim_end_newline, - }, - failed_to_run, - ), - } - } else { - (self, false) - } - } - /// Try to convert Value from Value::Range to Value::List. - /// This is useful to expand Value::Range into array notation, specifically when - /// converting `to json` or `to nuon`. - /// `1..3 | to XX -> [1,2,3]` - pub fn try_expand_range(self) -> Result { - match self { - PipelineData::Value(v, metadata) => { - let span = v.span(); - match v { - Value::Range { val, .. } => { - match val { - Range::IntRange(range) => { - if range.is_unbounded() { - return Err(ShellError::GenericError { - error: "Cannot create range".into(), - msg: "Unbounded ranges are not allowed when converting to this format".into(), - span: Some(span), - help: Some("Consider using ranges with valid start and end point.".into()), - inner: vec![], - }); - } - } - Range::FloatRange(range) => { - if range.is_unbounded() { - return Err(ShellError::GenericError { - error: "Cannot create range".into(), - msg: "Unbounded ranges are not allowed when converting to this format".into(), - span: Some(span), - help: Some("Consider using ranges with valid start and end point.".into()), - inner: vec![], - }); - } - } - } - let range_values: Vec = val.into_range_iter(span, None).collect(); - Ok(PipelineData::Value(Value::list(range_values, span), None)) - } - x => Ok(PipelineData::Value(x, metadata)), - } - } - _ => Ok(self), - } - } - - /// Consume and print self data immediately. - /// - /// `no_newline` controls if we need to attach newline character to output. - /// `to_stderr` controls if data is output to stderr, when the value is false, the data is output to stdout. - pub fn print( - self, - engine_state: &EngineState, - stack: &mut Stack, - no_newline: bool, - to_stderr: bool, - ) -> Result { - // If the table function is in the declarations, then we can use it - // to create the table value that will be printed in the terminal - - let config = engine_state.get_config(); - - if let PipelineData::ExternalStream { - stdout: stream, - stderr: stderr_stream, - exit_code, - .. - } = self - { - return print_if_stream(stream, stderr_stream, to_stderr, exit_code); - } - - if let Some(decl_id) = engine_state.table_decl_id { - let command = engine_state.get_decl(decl_id); - if command.get_block_id().is_some() { - return self.write_all_and_flush(engine_state, config, no_newline, to_stderr); - } - - let call = Call::new(Span::new(0, 0)); - let table = command.run(engine_state, stack, &call, self)?; - table.write_all_and_flush(engine_state, config, no_newline, to_stderr)?; - } else { - self.write_all_and_flush(engine_state, config, no_newline, to_stderr)?; - }; - - Ok(0) - } - - /// Consume and print self data immediately. - /// - /// Unlike [`.print()`] does not call `table` to format data and just prints it - /// one element on a line - /// * `no_newline` controls if we need to attach newline character to output. - /// * `to_stderr` controls if data is output to stderr, when the value is false, the data is output to stdout. - pub fn print_not_formatted( - self, - engine_state: &EngineState, - no_newline: bool, - to_stderr: bool, - ) -> Result { - if let PipelineData::ExternalStream { - stdout: stream, - stderr: stderr_stream, - exit_code, - .. - } = self - { - print_if_stream(stream, stderr_stream, to_stderr, exit_code) - } else { - let config = engine_state.get_config(); - self.write_all_and_flush(engine_state, config, no_newline, to_stderr) - } - } - - fn write_all_and_flush( - self, - engine_state: &EngineState, - config: &Config, - no_newline: bool, - to_stderr: bool, - ) -> Result { - for item in self { - let mut is_err = false; - let mut out = if let Value::Error { error, .. } = item { - let working_set = StateWorkingSet::new(engine_state); - // Value::Errors must always go to stderr, not stdout. - is_err = true; - format_error(&working_set, &*error) - } else if no_newline { - item.to_expanded_string("", config) - } else { - item.to_expanded_string("\n", config) - }; - - if !no_newline { - out.push('\n'); - } - - if !to_stderr && !is_err { - stdout_write_all_and_flush(out)? - } else { - stderr_write_all_and_flush(out)? - } - } - - Ok(0) - } -} - -enum PipelineIteratorInner { - Empty, - Value(Value), - ListStream(list_stream::IntoIter), - ExternalStream(RawStream), -} - -pub struct PipelineIterator(PipelineIteratorInner); - -impl IntoIterator for PipelineData { - type Item = Value; - - type IntoIter = PipelineIterator; - - fn into_iter(self) -> Self::IntoIter { - PipelineIterator(match self { - PipelineData::Value(value, ..) => { - let span = value.span(); - match value { - Value::List { vals, .. } => PipelineIteratorInner::ListStream( - ListStream::new(vals.into_iter(), span, None).into_iter(), - ), - Value::Range { val, .. } => PipelineIteratorInner::ListStream( - ListStream::new(val.into_range_iter(span, None), span, None).into_iter(), - ), - x => PipelineIteratorInner::Value(x), - } - } - PipelineData::ListStream(stream, ..) => { - PipelineIteratorInner::ListStream(stream.into_iter()) - } - PipelineData::ExternalStream { - stdout: Some(stdout), - .. - } => PipelineIteratorInner::ExternalStream(stdout), - PipelineData::ExternalStream { stdout: None, .. } => PipelineIteratorInner::Empty, - PipelineData::Empty => PipelineIteratorInner::Empty, - }) - } -} - -pub fn print_if_stream( - stream: Option, - stderr_stream: Option, - to_stderr: bool, - exit_code: Option, -) -> Result { - if let Some(stderr_stream) = stderr_stream { - thread::Builder::new() - .name("stderr consumer".to_string()) - .spawn(move || { - let RawStream { - stream, - leftover, - ctrlc, - .. - } = stderr_stream; - let mut stderr = std::io::stderr(); - let _ = stderr.write_all(&leftover); - drop(leftover); - for bytes in stream { - if nu_utils::ctrl_c::was_pressed(&ctrlc) { - break; - } - match bytes { - Ok(bytes) => { - let _ = stderr.write_all(&bytes); - } - Err(err) => { - // we don't have access to EngineState, but maybe logging the debug - // impl is better than nothing - eprintln!("Error in stderr stream: {err:?}"); - break; - } - } - } - })?; - } - - if let Some(stream) = stream { - for s in stream { - let s_live = s?; - let bin_output = s_live.coerce_into_binary()?; - - if !to_stderr { - stdout_write_all_and_flush(&bin_output)? - } else { - stderr_write_all_and_flush(&bin_output)? - } - } - } - - // Make sure everything has finished - if let Some(exit_code) = exit_code { - return drain_exit_code(exit_code); - } - - Ok(0) -} - -fn drain_exit_code(exit_code: ListStream) -> Result { - let mut exit_codes: Vec<_> = exit_code.into_iter().collect(); - match exit_codes.pop() { - #[cfg(unix)] - Some(Value::Error { error, .. }) => Err(*error), - Some(Value::Int { val, .. }) => Ok(val), - _ => Ok(0), - } -} - -/// Only call this if `output_stream` is not `OutDest::Pipe` or `OutDest::Capture`. -fn consume_child_output(child_output: RawStream, output_stream: &OutDest) -> io::Result<()> { - let mut output = ReadRawStream::new(child_output); - match output_stream { - OutDest::Pipe | OutDest::Capture => { - // The point of `consume_child_output` is to redirect output *right now*, - // but OutDest::Pipe means to redirect output - // into an OS pipe for *future use* (as input for another command). - // So, this branch makes no sense, and will simply drop `output` instead of draining it. - // This could trigger a `SIGPIPE` for the external command, - // since there will be no reader for its pipe. - debug_assert!(false) - } - OutDest::Null => { - io::copy(&mut output, &mut io::sink())?; - } - OutDest::Inherit => { - io::copy(&mut output, &mut io::stdout())?; - } - OutDest::File(file) => { - io::copy(&mut output, &mut file.try_clone()?)?; - } - } - Ok(()) -} - -impl Iterator for PipelineIterator { - type Item = Value; - - fn next(&mut self) -> Option { - match &mut self.0 { - PipelineIteratorInner::Empty => None, - PipelineIteratorInner::Value(Value::Nothing { .. }, ..) => None, - PipelineIteratorInner::Value(v, ..) => Some(std::mem::take(v)), - PipelineIteratorInner::ListStream(stream, ..) => stream.next(), - PipelineIteratorInner::ExternalStream(stream) => stream.next().map(|x| match x { - Ok(x) => x, - Err(err) => Value::error( - err, - Span::unknown(), //FIXME: unclear where this span should come from - ), - }), - } - } -} - -pub trait IntoPipelineData { - fn into_pipeline_data(self) -> PipelineData; - - fn into_pipeline_data_with_metadata( - self, - metadata: impl Into>, - ) -> PipelineData; -} - -impl IntoPipelineData for V -where - V: Into, -{ - fn into_pipeline_data(self) -> PipelineData { - PipelineData::Value(self.into(), None) - } - - fn into_pipeline_data_with_metadata( - self, - metadata: impl Into>, - ) -> PipelineData { - PipelineData::Value(self.into(), metadata.into()) - } -} - -pub trait IntoInterruptiblePipelineData { - fn into_pipeline_data(self, span: Span, ctrlc: Option>) -> PipelineData; - fn into_pipeline_data_with_metadata( - self, - span: Span, - ctrlc: Option>, - metadata: impl Into>, - ) -> PipelineData; -} - -impl IntoInterruptiblePipelineData for I -where - I: IntoIterator + Send + 'static, - I::IntoIter: Send + 'static, - ::Item: Into, -{ - fn into_pipeline_data(self, span: Span, ctrlc: Option>) -> PipelineData { - ListStream::new(self.into_iter().map(Into::into), span, ctrlc).into() - } - - fn into_pipeline_data_with_metadata( - self, - span: Span, - ctrlc: Option>, - metadata: impl Into>, - ) -> PipelineData { - PipelineData::ListStream( - ListStream::new(self.into_iter().map(Into::into), span, ctrlc), - metadata.into(), - ) - } -} - -fn value_to_bytes(value: Value) -> Result, ShellError> { - let bytes = match value { - Value::String { val, .. } => val.into_bytes(), - Value::Binary { val, .. } => val, - Value::List { vals, .. } => { - let val = vals - .into_iter() - .map(Value::coerce_into_string) - .collect::, ShellError>>()? - .join("\n") - + "\n"; - - val.into_bytes() - } - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => return Err(*error), - value => value.coerce_into_string()?.into_bytes(), - }; - Ok(bytes) -} - -struct ReadRawStream { - iter: Box, ShellError>>>, - cursor: Option>>, -} - -impl ReadRawStream { - fn new(stream: RawStream) -> Self { - debug_assert!(stream.leftover.is_empty()); - Self { - iter: stream.stream, - cursor: Some(Cursor::new(Vec::new())), - } - } -} - -impl Read for ReadRawStream { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - while let Some(cursor) = self.cursor.as_mut() { - let read = cursor.read(buf)?; - if read > 0 { - return Ok(read); - } else { - match self.iter.next().transpose() { - Ok(next) => { - self.cursor = next.map(Cursor::new); - } - Err(err) => { - // temporary hack - return Err(io::Error::new(io::ErrorKind::Other, err)); - } - } - } - } - Ok(0) - } -} diff --git a/crates/nu-protocol/src/pipeline_data/raw_stream.rs b/crates/nu-protocol/src/pipeline_data/raw_stream.rs deleted file mode 100644 index 846cdd772b..0000000000 --- a/crates/nu-protocol/src/pipeline_data/raw_stream.rs +++ /dev/null @@ -1,176 +0,0 @@ -use crate::*; -use std::{ - fmt::Debug, - sync::{atomic::AtomicBool, Arc}, -}; - -pub struct RawStream { - pub stream: Box, ShellError>> + Send + 'static>, - pub leftover: Vec, - pub ctrlc: Option>, - pub is_binary: bool, - pub span: Span, - pub known_size: Option, // (bytes) -} - -impl RawStream { - pub fn new( - stream: Box, ShellError>> + Send + 'static>, - ctrlc: Option>, - span: Span, - known_size: Option, - ) -> Self { - Self { - stream, - leftover: vec![], - ctrlc, - is_binary: false, - span, - known_size, - } - } - - pub fn into_bytes(self) -> Result>, ShellError> { - let mut output = vec![]; - - for item in self.stream { - if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { - break; - } - output.extend(item?); - } - - Ok(Spanned { - item: output, - span: self.span, - }) - } - - pub fn into_string(self) -> Result, ShellError> { - let mut output = String::new(); - let span = self.span; - let ctrlc = &self.ctrlc.clone(); - - for item in self { - if nu_utils::ctrl_c::was_pressed(ctrlc) { - break; - } - output.push_str(&item?.coerce_into_string()?); - } - - Ok(Spanned { item: output, span }) - } - - pub fn chain(self, stream: RawStream) -> RawStream { - RawStream { - stream: Box::new(self.stream.chain(stream.stream)), - leftover: self.leftover.into_iter().chain(stream.leftover).collect(), - ctrlc: self.ctrlc, - is_binary: self.is_binary, - span: self.span, - known_size: self.known_size, - } - } - - pub fn drain(self) -> Result<(), ShellError> { - for next in self { - match next { - Ok(val) => { - if let Value::Error { error, .. } = val { - return Err(*error); - } - } - Err(err) => return Err(err), - } - } - Ok(()) - } -} -impl Debug for RawStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RawStream").finish() - } -} -impl Iterator for RawStream { - type Item = Result; - - fn next(&mut self) -> Option { - if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { - return None; - } - - // If we know we're already binary, just output that - if self.is_binary { - self.stream.next().map(|buffer| { - buffer.map(|mut v| { - if !self.leftover.is_empty() { - for b in self.leftover.drain(..).rev() { - v.insert(0, b); - } - } - Value::binary(v, self.span) - }) - }) - } else { - // We *may* be text. We're only going to try utf-8. Other decodings - // needs to be taken as binary first, then passed through `decode`. - if let Some(buffer) = self.stream.next() { - match buffer { - Ok(mut v) => { - if !self.leftover.is_empty() { - while let Some(b) = self.leftover.pop() { - v.insert(0, b); - } - } - - match String::from_utf8(v.clone()) { - Ok(s) => { - // Great, we have a complete string, let's output it - Some(Ok(Value::string(s, self.span))) - } - Err(err) => { - // Okay, we *might* have a string but we've also got some errors - if v.is_empty() { - // We can just end here - None - } else if v.len() > 3 - && (v.len() - err.utf8_error().valid_up_to() > 3) - { - // As UTF-8 characters are max 4 bytes, if we have more than that in error we know - // that it's not just a character spanning two frames. - // We now know we are definitely binary, so switch to binary and stay there. - self.is_binary = true; - Some(Ok(Value::binary(v, self.span))) - } else { - // Okay, we have a tiny bit of error at the end of the buffer. This could very well be - // a character that spans two frames. Since this is the case, remove the error from - // the current frame an dput it in the leftover buffer. - self.leftover = v[err.utf8_error().valid_up_to()..].to_vec(); - - let buf = v[0..err.utf8_error().valid_up_to()].to_vec(); - - match String::from_utf8(buf) { - Ok(s) => Some(Ok(Value::string(s, self.span))), - Err(_) => { - // Something is definitely wrong. Switch to binary, and stay there - self.is_binary = true; - Some(Ok(Value::binary(v, self.span))) - } - } - } - } - } - } - Err(e) => Some(Err(e)), - } - } else if !self.leftover.is_empty() { - let output = Ok(Value::binary(self.leftover.clone(), self.span)); - self.leftover.clear(); - - Some(output) - } else { - None - } - } - } -} diff --git a/crates/nu-protocol/src/process/child.rs b/crates/nu-protocol/src/process/child.rs new file mode 100644 index 0000000000..cc74b40fc1 --- /dev/null +++ b/crates/nu-protocol/src/process/child.rs @@ -0,0 +1,294 @@ +use crate::{ + byte_stream::convert_file, process::ExitStatus, ErrSpan, IntoSpanned, ShellError, Span, +}; +use nu_system::ForegroundChild; +use os_pipe::PipeReader; +use std::{ + fmt::Debug, + io::{self, Read}, + sync::mpsc::{self, Receiver, RecvError, TryRecvError}, + thread, +}; + +#[derive(Debug)] +enum ExitStatusFuture { + Finished(Result>), + Running(Receiver>), +} + +impl ExitStatusFuture { + fn wait(&mut self, span: Span) -> Result { + match self { + ExitStatusFuture::Finished(Ok(status)) => Ok(*status), + ExitStatusFuture::Finished(Err(err)) => Err(err.as_ref().clone()), + ExitStatusFuture::Running(receiver) => { + let code = match receiver.recv() { + Ok(Ok(status)) => Ok(status), + Ok(Err(err)) => Err(ShellError::IOErrorSpanned { + msg: format!("failed to get exit code: {err:?}"), + span, + }), + Err(RecvError) => Err(ShellError::IOErrorSpanned { + msg: "failed to get exit code".into(), + span, + }), + }; + + *self = ExitStatusFuture::Finished(code.clone().map_err(Box::new)); + + code + } + } + } + + fn try_wait(&mut self, span: Span) -> Result, ShellError> { + match self { + ExitStatusFuture::Finished(Ok(code)) => Ok(Some(*code)), + ExitStatusFuture::Finished(Err(err)) => Err(err.as_ref().clone()), + ExitStatusFuture::Running(receiver) => { + let code = match receiver.try_recv() { + Ok(Ok(status)) => Ok(Some(status)), + Ok(Err(err)) => Err(ShellError::IOErrorSpanned { + msg: format!("failed to get exit code: {err:?}"), + span, + }), + Err(TryRecvError::Disconnected) => Err(ShellError::IOErrorSpanned { + msg: "failed to get exit code".into(), + span, + }), + Err(TryRecvError::Empty) => Ok(None), + }; + + if let Some(code) = code.clone().transpose() { + *self = ExitStatusFuture::Finished(code.map_err(Box::new)); + } + + code + } + } + } +} + +pub enum ChildPipe { + Pipe(PipeReader), + Tee(Box), +} + +impl Debug for ChildPipe { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ChildPipe").finish() + } +} + +impl From for ChildPipe { + fn from(pipe: PipeReader) -> Self { + Self::Pipe(pipe) + } +} + +impl Read for ChildPipe { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + ChildPipe::Pipe(pipe) => pipe.read(buf), + ChildPipe::Tee(tee) => tee.read(buf), + } + } +} + +#[derive(Debug)] +pub struct ChildProcess { + pub stdout: Option, + pub stderr: Option, + exit_status: ExitStatusFuture, + span: Span, +} + +impl ChildProcess { + pub fn new( + mut child: ForegroundChild, + reader: Option, + swap: bool, + span: Span, + ) -> Result { + let (stdout, stderr) = if let Some(combined) = reader { + (Some(combined), None) + } else { + let stdout = child.as_mut().stdout.take().map(convert_file); + let stderr = child.as_mut().stderr.take().map(convert_file); + + if swap { + (stderr, stdout) + } else { + (stdout, stderr) + } + }; + + // Create a thread to wait for the exit status. + let (exit_status_sender, exit_status) = mpsc::channel(); + + thread::Builder::new() + .name("exit status waiter".into()) + .spawn(move || exit_status_sender.send(child.wait().map(Into::into))) + .err_span(span)?; + + Ok(Self::from_raw(stdout, stderr, Some(exit_status), span)) + } + + pub fn from_raw( + stdout: Option, + stderr: Option, + exit_status: Option>>, + span: Span, + ) -> Self { + Self { + stdout: stdout.map(Into::into), + stderr: stderr.map(Into::into), + exit_status: exit_status + .map(ExitStatusFuture::Running) + .unwrap_or(ExitStatusFuture::Finished(Ok(ExitStatus::Exited(0)))), + span, + } + } + + pub fn set_exit_code(&mut self, exit_code: i32) { + self.exit_status = ExitStatusFuture::Finished(Ok(ExitStatus::Exited(exit_code))); + } + + pub fn span(&self) -> Span { + self.span + } + + pub fn into_bytes(mut self) -> Result, ShellError> { + if self.stderr.is_some() { + debug_assert!(false, "stderr should not exist"); + return Err(ShellError::IOErrorSpanned { + msg: "internal error".into(), + span: self.span, + }); + } + + let bytes = if let Some(stdout) = self.stdout { + collect_bytes(stdout).err_span(self.span)? + } else { + Vec::new() + }; + + // TODO: check exit_status + self.exit_status.wait(self.span)?; + + Ok(bytes) + } + + pub fn wait(mut self) -> Result { + if let Some(stdout) = self.stdout.take() { + let stderr = self + .stderr + .take() + .map(|stderr| { + thread::Builder::new() + .name("stderr consumer".into()) + .spawn(move || consume_pipe(stderr)) + }) + .transpose() + .err_span(self.span)?; + + let res = consume_pipe(stdout); + + if let Some(handle) = stderr { + handle + .join() + .map_err(|e| match e.downcast::() { + Ok(io) => ShellError::from((*io).into_spanned(self.span)), + Err(err) => ShellError::GenericError { + error: "Unknown error".into(), + msg: format!("{err:?}"), + span: Some(self.span), + help: None, + inner: Vec::new(), + }, + })? + .err_span(self.span)?; + } + + res.err_span(self.span)?; + } else if let Some(stderr) = self.stderr.take() { + consume_pipe(stderr).err_span(self.span)?; + } + + self.exit_status.wait(self.span) + } + + pub fn try_wait(&mut self) -> Result, ShellError> { + self.exit_status.try_wait(self.span) + } + + pub fn wait_with_output(mut self) -> Result { + let (stdout, stderr) = if let Some(stdout) = self.stdout { + let stderr = self + .stderr + .map(|stderr| thread::Builder::new().spawn(move || collect_bytes(stderr))) + .transpose() + .err_span(self.span)?; + + let stdout = collect_bytes(stdout).err_span(self.span)?; + + let stderr = stderr + .map(|handle| { + handle.join().map_err(|e| match e.downcast::() { + Ok(io) => ShellError::from((*io).into_spanned(self.span)), + Err(err) => ShellError::GenericError { + error: "Unknown error".into(), + msg: format!("{err:?}"), + span: Some(self.span), + help: None, + inner: Vec::new(), + }, + }) + }) + .transpose()? + .transpose() + .err_span(self.span)?; + + (Some(stdout), stderr) + } else { + let stderr = self + .stderr + .map(collect_bytes) + .transpose() + .err_span(self.span)?; + + (None, stderr) + }; + + let exit_status = self.exit_status.wait(self.span)?; + + Ok(ProcessOutput { + stdout, + stderr, + exit_status, + }) + } +} + +fn collect_bytes(pipe: ChildPipe) -> io::Result> { + let mut buf = Vec::new(); + match pipe { + ChildPipe::Pipe(mut pipe) => pipe.read_to_end(&mut buf), + ChildPipe::Tee(mut tee) => tee.read_to_end(&mut buf), + }?; + Ok(buf) +} + +fn consume_pipe(pipe: ChildPipe) -> io::Result<()> { + match pipe { + ChildPipe::Pipe(mut pipe) => io::copy(&mut pipe, &mut io::sink()), + ChildPipe::Tee(mut tee) => io::copy(&mut tee, &mut io::sink()), + }?; + Ok(()) +} + +pub struct ProcessOutput { + pub stdout: Option>, + pub stderr: Option>, + pub exit_status: ExitStatus, +} diff --git a/crates/nu-protocol/src/process/exit_status.rs b/crates/nu-protocol/src/process/exit_status.rs new file mode 100644 index 0000000000..8f3794c44f --- /dev/null +++ b/crates/nu-protocol/src/process/exit_status.rs @@ -0,0 +1,64 @@ +use std::process; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExitStatus { + Exited(i32), + #[cfg(unix)] + Signaled { + signal: i32, + core_dumped: bool, + }, +} + +impl ExitStatus { + pub fn code(self) -> i32 { + match self { + ExitStatus::Exited(code) => code, + #[cfg(unix)] + ExitStatus::Signaled { signal, .. } => -signal, + } + } +} + +#[cfg(unix)] +impl From for ExitStatus { + fn from(status: process::ExitStatus) -> Self { + use std::os::unix::process::ExitStatusExt; + + match (status.code(), status.signal()) { + (Some(code), None) => Self::Exited(code), + (None, Some(signal)) => Self::Signaled { + signal, + core_dumped: status.core_dumped(), + }, + (None, None) => { + debug_assert!(false, "ExitStatus should have either a code or a signal"); + Self::Exited(-1) + } + (Some(code), Some(signal)) => { + // Should be unreachable, as `code()` will be `None` if `signal()` is `Some` + // according to the docs for `ExitStatus::code`. + debug_assert!( + false, + "ExitStatus cannot have both a code ({code}) and a signal ({signal})" + ); + Self::Signaled { + signal, + core_dumped: status.core_dumped(), + } + } + } + } +} + +#[cfg(not(unix))] +impl From for ExitStatus { + fn from(status: process::ExitStatus) -> Self { + let code = status.code(); + debug_assert!( + code.is_some(), + "`ExitStatus::code` cannot return `None` on windows" + ); + Self::Exited(code.unwrap_or(-1)) + } +} diff --git a/crates/nu-protocol/src/process/mod.rs b/crates/nu-protocol/src/process/mod.rs new file mode 100644 index 0000000000..2fcf65f56e --- /dev/null +++ b/crates/nu-protocol/src/process/mod.rs @@ -0,0 +1,5 @@ +mod child; +mod exit_status; + +pub use child::*; +pub use exit_status::ExitStatus; diff --git a/crates/nu-protocol/src/span.rs b/crates/nu-protocol/src/span.rs index 7bc13997a1..3d32aa4ddf 100644 --- a/crates/nu-protocol/src/span.rs +++ b/crates/nu-protocol/src/span.rs @@ -1,7 +1,6 @@ -use std::ops::Deref; - use miette::SourceSpan; use serde::{Deserialize, Serialize}; +use std::ops::Deref; /// A spanned area of interest, generic over what kind of thing is of interest #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -74,77 +73,123 @@ impl IntoSpanned for T { /// Spans are a global offset across all seen files, which are cached in the engine's state. The start and /// end offset together make the inclusive start/exclusive end pair for where to underline to highlight /// a given point of interest. -#[non_exhaustive] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub struct Span { pub start: usize, pub end: usize, } -impl From for SourceSpan { - fn from(s: Span) -> Self { - Self::new(s.start.into(), s.end - s.start) - } -} - impl Span { - pub fn new(start: usize, end: usize) -> Span { + pub fn new(start: usize, end: usize) -> Self { debug_assert!( end >= start, "Can't create a Span whose end < start, start={start}, end={end}" ); - Span { start, end } + Self { start, end } } - pub const fn unknown() -> Span { - Span { start: 0, end: 0 } + pub const fn unknown() -> Self { + Self { start: 0, end: 0 } } /// Note: Only use this for test data, *not* live data, as it will point into unknown source /// when used in errors. - pub const fn test_data() -> Span { + pub const fn test_data() -> Self { Self::unknown() } - pub fn offset(&self, offset: usize) -> Span { - Span::new(self.start - offset, self.end - offset) + pub fn offset(&self, offset: usize) -> Self { + Self::new(self.start - offset, self.end - offset) } pub fn contains(&self, pos: usize) -> bool { - pos >= self.start && pos < self.end + self.start <= pos && pos < self.end } - pub fn contains_span(&self, span: Span) -> bool { - span.start >= self.start && span.end <= self.end + pub fn contains_span(&self, span: Self) -> bool { + self.start <= span.start && span.end <= self.end } - /// Point to the space just past this span, useful for missing - /// values - pub fn past(&self) -> Span { - Span { + /// Point to the space just past this span, useful for missing values + pub fn past(&self) -> Self { + Self { start: self.end, end: self.end, } } + + /// Returns the minimal [`Span`] that encompasses both of the given spans. + /// + /// The two `Spans` can overlap in the middle, + /// but must otherwise be in order by satisfying: + /// - `self.start <= after.start` + /// - `self.end <= after.end` + /// + /// If this is not guaranteed to be the case, use [`Span::merge`] instead. + pub fn append(self, after: Self) -> Self { + debug_assert!( + self.start <= after.start && self.end <= after.end, + "Can't merge two Spans that are not in order" + ); + Self { + start: self.start, + end: after.end, + } + } + + /// Returns the minimal [`Span`] that encompasses both of the given spans. + /// + /// The spans need not be in order or have any relationship. + /// + /// [`Span::append`] is slightly more efficient if the spans are known to be in order. + pub fn merge(self, other: Self) -> Self { + Self { + start: usize::min(self.start, other.start), + end: usize::max(self.end, other.end), + } + } + + /// Returns the minimal [`Span`] that encompasses all of the spans in the given slice. + /// + /// The spans are assumed to be in order, that is, all consecutive spans must satisfy: + /// - `spans[i].start <= spans[i + 1].start` + /// - `spans[i].end <= spans[i + 1].end` + /// + /// (Two consecutive spans can overlap as long as the above is true.) + /// + /// Use [`Span::merge_many`] if the spans are not known to be in order. + pub fn concat(spans: &[Self]) -> Self { + // TODO: enable assert below + // debug_assert!(!spans.is_empty()); + debug_assert!(spans.windows(2).all(|spans| { + let &[a, b] = spans else { + return false; + }; + a.start <= b.start && a.end <= b.end + })); + Self { + start: spans.first().map(|s| s.start).unwrap_or(0), + end: spans.last().map(|s| s.end).unwrap_or(0), + } + } + + /// Returns the minimal [`Span`] that encompasses all of the spans in the given iterator. + /// + /// The spans need not be in order or have any relationship. + /// + /// [`Span::concat`] is more efficient if the spans are known to be in order. + pub fn merge_many(spans: impl IntoIterator) -> Self { + spans + .into_iter() + .reduce(Self::merge) + .unwrap_or(Self::unknown()) + } } -/// Used when you have a slice of spans of at least size 1 -pub fn span(spans: &[Span]) -> Span { - let length = spans.len(); - - //TODO debug_assert!(length > 0, "expect spans > 0"); - if length == 0 { - Span::unknown() - } else if length == 1 { - spans[0] - } else { - let end = spans - .iter() - .map(|s| s.end) - .max() - .expect("Must be an end. Length > 0"); - Span::new(spans[0].start, end) +impl From for SourceSpan { + fn from(s: Span) -> Self { + Self::new(s.start.into(), s.end - s.start) } } diff --git a/crates/nu-protocol/src/util.rs b/crates/nu-protocol/src/util.rs deleted file mode 100644 index 1c17c49e4c..0000000000 --- a/crates/nu-protocol/src/util.rs +++ /dev/null @@ -1,52 +0,0 @@ -use crate::ShellError; -use std::io::{BufRead, BufReader, Read}; - -pub struct BufferedReader { - input: BufReader, - error: bool, -} - -impl BufferedReader { - pub fn new(input: BufReader) -> Self { - Self { - input, - error: false, - } - } - - pub fn into_inner(self) -> BufReader { - self.input - } -} - -impl Iterator for BufferedReader { - type Item = Result, ShellError>; - - fn next(&mut self) -> Option { - // Don't try to read more data if an error occurs - if self.error { - return None; - } - - let buffer = self.input.fill_buf(); - match buffer { - Ok(s) => { - let result = s.to_vec(); - - let buffer_len = s.len(); - - if buffer_len == 0 { - None - } else { - self.input.consume(buffer_len); - - Some(Ok(result)) - } - } - Err(e) => { - self.error = true; - Some(Err(ShellError::IOError { msg: e.to_string() })) - } - } - } -} diff --git a/crates/nu-protocol/src/value/from_value.rs b/crates/nu-protocol/src/value/from_value.rs index f2f669fa4c..af776662f4 100644 --- a/crates/nu-protocol/src/value/from_value.rs +++ b/crates/nu-protocol/src/value/from_value.rs @@ -442,7 +442,7 @@ impl FromValue for Spanned> { impl FromValue for Range { fn from_value(v: Value) -> Result { match v { - Value::Range { val, .. } => Ok(val), + Value::Range { val, .. } => Ok(*val), v => Err(ShellError::CantConvert { to_type: "range".into(), from_type: v.get_type().to_string(), @@ -457,7 +457,7 @@ impl FromValue for Spanned { fn from_value(v: Value) -> Result { let span = v.span(); match v { - Value::Range { val, .. } => Ok(Spanned { item: val, span }), + Value::Range { val, .. } => Ok(Spanned { item: *val, span }), v => Err(ShellError::CantConvert { to_type: "range".into(), from_type: v.get_type().to_string(), @@ -552,7 +552,7 @@ impl FromValue for Record { impl FromValue for Closure { fn from_value(v: Value) -> Result { match v { - Value::Closure { val, .. } => Ok(val), + Value::Closure { val, .. } => Ok(*val), v => Err(ShellError::CantConvert { to_type: "Closure".into(), from_type: v.get_type().to_string(), @@ -567,7 +567,7 @@ impl FromValue for Spanned { fn from_value(v: Value) -> Result { let span = v.span(); match v { - Value::Closure { val, .. } => Ok(Spanned { item: val, span }), + Value::Closure { val, .. } => Ok(Spanned { item: *val, span }), v => Err(ShellError::CantConvert { to_type: "Closure".into(), from_type: v.get_type().to_string(), diff --git a/crates/nu-protocol/src/value/mod.rs b/crates/nu-protocol/src/value/mod.rs index 348e4b598f..56826e65d3 100644 --- a/crates/nu-protocol/src/value/mod.rs +++ b/crates/nu-protocol/src/value/mod.rs @@ -86,7 +86,7 @@ pub enum Value { internal_span: Span, }, Range { - val: Range, + val: Box, // note: spans are being refactored out of Value // please use .span() instead of matching this span value #[serde(rename = "span")] @@ -122,7 +122,7 @@ pub enum Value { internal_span: Span, }, Closure { - val: Closure, + val: Box, // note: spans are being refactored out of Value // please use .span() instead of matching this span value #[serde(rename = "span")] @@ -182,7 +182,7 @@ impl Clone for Value { internal_span: *internal_span, }, Value::Range { val, internal_span } => Value::Range { - val: *val, + val: val.clone(), internal_span: *internal_span, }, Value::Float { val, internal_span } => Value::float(*val, *internal_span), @@ -332,7 +332,7 @@ impl Value { /// Returns a reference to the inner [`Range`] value or an error if this `Value` is not a range pub fn as_range(&self) -> Result { if let Value::Range { val, .. } = self { - Ok(*val) + Ok(**val) } else { self.cant_convert_to("range") } @@ -341,7 +341,7 @@ impl Value { /// Unwraps the inner [`Range`] value or returns an error if this `Value` is not a range pub fn into_range(self) -> Result { if let Value::Range { val, .. } = self { - Ok(val) + Ok(*val) } else { self.cant_convert_to("range") } @@ -558,7 +558,7 @@ impl Value { /// Unwraps the inner [`Closure`] value or returns an error if this `Value` is not a closure pub fn into_closure(self) -> Result { if let Value::Closure { val, .. } = self { - Ok(val) + Ok(*val) } else { self.cant_convert_to("closure") } @@ -1017,7 +1017,7 @@ impl Value { }); } } - Value::Range { val, .. } => { + Value::Range { ref val, .. } => { if let Some(item) = val.into_range_iter(current.span(), None).nth(*count) { @@ -1828,7 +1828,7 @@ impl Value { pub fn range(val: Range, span: Span) -> Value { Value::Range { - val, + val: val.into(), internal_span: span, } } @@ -1864,7 +1864,7 @@ impl Value { pub fn closure(val: Closure, span: Span) -> Value { Value::Closure { - val, + val: val.into(), internal_span: span, } } diff --git a/crates/nu-protocol/tests/test_pipeline_data.rs b/crates/nu-protocol/tests/test_pipeline_data.rs index 6675f6a04a..95941285ad 100644 --- a/crates/nu-protocol/tests/test_pipeline_data.rs +++ b/crates/nu-protocol/tests/test_pipeline_data.rs @@ -11,5 +11,5 @@ fn test_convert_pipeline_data_to_value() { let new_span = Span::new(5, 6); let converted_value = pipeline_data.into_value(new_span); - assert_eq!(converted_value, Value::int(value_val, new_span)); + assert_eq!(converted_value, Ok(Value::int(value_val, new_span))); } diff --git a/crates/nu-std/testing.nu b/crates/nu-std/testing.nu index ddae1d371e..489430e462 100644 --- a/crates/nu-std/testing.nu +++ b/crates/nu-std/testing.nu @@ -287,7 +287,7 @@ export def run-tests [ --list, # list the selected tests without running them. --threads: int@"nu-complete threads", # Amount of threads to use for parallel execution. Default: All threads are utilized ] { - let available_threads = (sys | get cpu | length) + let available_threads = (sys cpu | length) # Can't use pattern matching here due to https://github.com/nushell/nushell/issues/9198 let threads = (if $threads == null { diff --git a/crates/nu-system/src/foreground.rs b/crates/nu-system/src/foreground.rs index d54cab1f19..2fe3c4fb29 100644 --- a/crates/nu-system/src/foreground.rs +++ b/crates/nu-system/src/foreground.rs @@ -1,6 +1,6 @@ use std::{ io, - process::{Child, Command}, + process::{Child, Command, ExitStatus}, sync::{atomic::AtomicU32, Arc}, }; @@ -72,6 +72,10 @@ impl ForegroundChild { }) } } + + pub fn wait(&mut self) -> io::Result { + self.as_mut().wait() + } } impl AsMut for ForegroundChild { diff --git a/crates/nu_plugin_example/src/commands/collect_external.rs b/crates/nu_plugin_example/src/commands/collect_bytes.rs similarity index 56% rename from crates/nu_plugin_example/src/commands/collect_external.rs rename to crates/nu_plugin_example/src/commands/collect_bytes.rs index e5c8c61f2e..51ca1d4222 100644 --- a/crates/nu_plugin_example/src/commands/collect_external.rs +++ b/crates/nu_plugin_example/src/commands/collect_bytes.rs @@ -1,22 +1,22 @@ use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand}; use nu_protocol::{ - Category, Example, LabeledError, PipelineData, RawStream, Signature, Type, Value, + ByteStream, Category, Example, LabeledError, PipelineData, Signature, Type, Value, }; use crate::ExamplePlugin; -/// `> | example collect-external` -pub struct CollectExternal; +/// `> | example collect-bytes` +pub struct CollectBytes; -impl PluginCommand for CollectExternal { +impl PluginCommand for CollectBytes { type Plugin = ExamplePlugin; fn name(&self) -> &str { - "example collect-external" + "example collect-bytes" } fn usage(&self) -> &str { - "Example transformer to raw external stream" + "Example transformer to byte stream" } fn search_terms(&self) -> Vec<&str> { @@ -34,7 +34,7 @@ impl PluginCommand for CollectExternal { fn examples(&self) -> Vec { vec![Example { - example: "[a b] | example collect-external", + example: "[a b] | example collect-bytes", description: "collect strings into one stream", result: Some(Value::test_string("ab")), }] @@ -47,26 +47,19 @@ impl PluginCommand for CollectExternal { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let stream = input.into_iter().map(|value| { - value - .as_str() - .map(|str| str.as_bytes()) - .or_else(|_| value.as_binary()) - .map(|bin| bin.to_vec()) - }); - Ok(PipelineData::ExternalStream { - stdout: Some(RawStream::new(Box::new(stream), None, call.head, None)), - stderr: None, - exit_code: None, - span: call.head, - metadata: None, - trim_end_newline: false, - }) + Ok(PipelineData::ByteStream( + ByteStream::from_result_iter( + input.into_iter().map(Value::coerce_into_binary), + call.head, + None, + ), + None, + )) } } #[test] fn test_examples() -> Result<(), nu_protocol::ShellError> { use nu_plugin_test_support::PluginTest; - PluginTest::new("example", ExamplePlugin.into())?.test_command_examples(&CollectExternal) + PluginTest::new("example", ExamplePlugin.into())?.test_command_examples(&CollectBytes) } diff --git a/crates/nu_plugin_example/src/commands/echo.rs b/crates/nu_plugin_example/src/commands/echo.rs new file mode 100644 index 0000000000..e60ccf8107 --- /dev/null +++ b/crates/nu_plugin_example/src/commands/echo.rs @@ -0,0 +1,55 @@ +use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand}; +use nu_protocol::{Category, Example, LabeledError, PipelineData, Signature, Type, Value}; + +use crate::ExamplePlugin; + +/// ` | example echo` +pub struct Echo; + +impl PluginCommand for Echo { + type Plugin = ExamplePlugin; + + fn name(&self) -> &str { + "example echo" + } + + fn usage(&self) -> &str { + "Example stream consumer that outputs the received input" + } + + fn signature(&self) -> Signature { + Signature::build(self.name()) + .input_output_types(vec![(Type::Any, Type::Any)]) + .category(Category::Experimental) + } + + fn search_terms(&self) -> Vec<&str> { + vec!["example"] + } + + fn examples(&self) -> Vec { + vec![Example { + example: "example seq 1 5 | example echo", + description: "echos the values from 1 to 5", + result: Some(Value::test_list( + (1..=5).map(Value::test_int).collect::>(), + )), + }] + } + + fn run( + &self, + _plugin: &ExamplePlugin, + _engine: &EngineInterface, + _call: &EvaluatedCall, + input: PipelineData, + ) -> Result { + Ok(input) + } +} + +#[test] +fn test_examples() -> Result<(), nu_protocol::ShellError> { + use nu_plugin_test_support::PluginTest; + PluginTest::new("example", ExamplePlugin.into())?.test_command_examples(&Echo) +} diff --git a/crates/nu_plugin_example/src/commands/mod.rs b/crates/nu_plugin_example/src/commands/mod.rs index 2d7ef4274a..dd808616a9 100644 --- a/crates/nu_plugin_example/src/commands/mod.rs +++ b/crates/nu_plugin_example/src/commands/mod.rs @@ -24,13 +24,15 @@ pub use env::Env; pub use view_span::ViewSpan; // Stream demos -mod collect_external; +mod collect_bytes; +mod echo; mod for_each; mod generate; mod seq; mod sum; -pub use collect_external::CollectExternal; +pub use collect_bytes::CollectBytes; +pub use echo::Echo; pub use for_each::ForEach; pub use generate::Generate; pub use seq::Seq; diff --git a/crates/nu_plugin_example/src/lib.rs b/crates/nu_plugin_example/src/lib.rs index 0c394c78aa..182bc85121 100644 --- a/crates/nu_plugin_example/src/lib.rs +++ b/crates/nu_plugin_example/src/lib.rs @@ -24,7 +24,8 @@ impl Plugin for ExamplePlugin { Box::new(ViewSpan), Box::new(DisableGc), // Stream demos - Box::new(CollectExternal), + Box::new(CollectBytes), + Box::new(Echo), Box::new(ForEach), Box::new(Generate), Box::new(Seq), diff --git a/crates/nu_plugin_polars/src/cache/rm.rs b/crates/nu_plugin_polars/src/cache/rm.rs index b8b814ba60..5918209f32 100644 --- a/crates/nu_plugin_polars/src/cache/rm.rs +++ b/crates/nu_plugin_polars/src/cache/rm.rs @@ -94,7 +94,7 @@ mod test { .add_decl(Box::new(First))? .add_decl(Box::new(Get))? .eval("let df = ([[a b];[1 2] [3 4]] | polars into-df); polars store-ls | get key | first | polars store-rm $in")?; - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let msg = value .as_list()? .first() diff --git a/crates/nu_plugin_polars/src/dataframe/eager/summary.rs b/crates/nu_plugin_polars/src/dataframe/eager/summary.rs index 8e5151837d..2b22d73bfb 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/summary.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/summary.rs @@ -38,7 +38,7 @@ impl PluginCommand for Summary { ) .named( "quantiles", - SyntaxShape::Table(vec![]), + SyntaxShape::List(Box::new(SyntaxShape::Float)), "provide optional quantiles", Some('q'), ) diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_arrow.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_arrow.rs index 8dad0d195f..dfb331ac46 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_arrow.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_arrow.rs @@ -124,7 +124,7 @@ pub mod test { assert!(tmp_file.exists()); - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let list = value.as_list()?; assert_eq!(list.len(), 1); let msg = list.first().expect("should have a value").as_str()?; diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_avro.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_avro.rs index 7a7197e47a..3a5dc317e7 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_avro.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_avro.rs @@ -153,7 +153,7 @@ pub mod test { assert!(tmp_file.exists()); - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let list = value.as_list()?; assert_eq!(list.len(), 1); let msg = list.first().expect("should have a value").as_str()?; diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_csv.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_csv.rs index ace95d08bb..d55a53f1fc 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_csv.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_csv.rs @@ -171,7 +171,7 @@ pub mod test { assert!(tmp_file.exists()); - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let list = value.as_list()?; assert_eq!(list.len(), 1); let msg = list.first().expect("should have a value").as_str()?; diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_json_lines.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_json_lines.rs index 4140ca199b..88b4a61bbf 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_json_lines.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_json_lines.rs @@ -125,7 +125,7 @@ pub mod test { assert!(tmp_file.exists()); - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let list = value.as_list()?; assert_eq!(list.len(), 1); let msg = list.first().expect("should have a value").as_str()?; diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_nu.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_nu.rs index 9acac7355c..8e3cdffa24 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_nu.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_nu.rs @@ -89,7 +89,7 @@ impl PluginCommand for ToNu { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) { dataframe_command(plugin, call, value) } else { diff --git a/crates/nu_plugin_polars/src/dataframe/eager/to_parquet.rs b/crates/nu_plugin_polars/src/dataframe/eager/to_parquet.rs index e53a4ac41d..4a8208ae12 100644 --- a/crates/nu_plugin_polars/src/dataframe/eager/to_parquet.rs +++ b/crates/nu_plugin_polars/src/dataframe/eager/to_parquet.rs @@ -124,7 +124,7 @@ pub mod test { assert!(tmp_file.exists()); - let value = pipeline_data.into_value(Span::test_data()); + let value = pipeline_data.into_value(Span::test_data())?; let list = value.as_list()?; assert_eq!(list.len(), 1); let msg = list.first().expect("should have a value").as_str()?; diff --git a/crates/nu_plugin_polars/src/dataframe/expressions/expressions_macro.rs b/crates/nu_plugin_polars/src/dataframe/expressions/expressions_macro.rs index 577524123c..feb559aba5 100644 --- a/crates/nu_plugin_polars/src/dataframe/expressions/expressions_macro.rs +++ b/crates/nu_plugin_polars/src/dataframe/expressions/expressions_macro.rs @@ -159,7 +159,7 @@ macro_rules! lazy_expr_command { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) { let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value) .map_err(LabeledError::from)?; @@ -239,7 +239,7 @@ macro_rules! lazy_expr_command { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) { let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value) .map_err(LabeledError::from)?; diff --git a/crates/nu_plugin_polars/src/dataframe/expressions/is_in.rs b/crates/nu_plugin_polars/src/dataframe/expressions/is_in.rs index ed4b567983..47be15e2f3 100644 --- a/crates/nu_plugin_polars/src/dataframe/expressions/is_in.rs +++ b/crates/nu_plugin_polars/src/dataframe/expressions/is_in.rs @@ -114,8 +114,7 @@ impl PluginCommand for ExprIsIn { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => command_df(plugin, engine, call, df), PolarsPluginObject::NuLazyFrame(lazy) => { diff --git a/crates/nu_plugin_polars/src/dataframe/expressions/otherwise.rs b/crates/nu_plugin_polars/src/dataframe/expressions/otherwise.rs index 2bdbfefb35..0e84d8fe96 100644 --- a/crates/nu_plugin_polars/src/dataframe/expressions/otherwise.rs +++ b/crates/nu_plugin_polars/src/dataframe/expressions/otherwise.rs @@ -99,7 +99,7 @@ impl PluginCommand for ExprOtherwise { let otherwise_predicate: Value = call.req(0)?; let otherwise_predicate = NuExpression::try_from_value(plugin, &otherwise_predicate)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let complete: NuExpression = match NuWhen::try_from_value(plugin, &value)?.when_type { NuWhenType::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(), NuWhenType::ChainedThen(chained_when) => chained_when diff --git a/crates/nu_plugin_polars/src/dataframe/expressions/when.rs b/crates/nu_plugin_polars/src/dataframe/expressions/when.rs index 158b2ac757..3c1b0eb481 100644 --- a/crates/nu_plugin_polars/src/dataframe/expressions/when.rs +++ b/crates/nu_plugin_polars/src/dataframe/expressions/when.rs @@ -111,7 +111,7 @@ impl PluginCommand for ExprWhen { let then_predicate: Value = call.req(1)?; let then_predicate = NuExpression::try_from_value(plugin, &then_predicate)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let when_then: NuWhen = match value { Value::Nothing { .. } => when(when_predicate.into_polars()) .then(then_predicate.into_polars()) diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/cast.rs b/crates/nu_plugin_polars/src/dataframe/lazy/cast.rs index 559ca27658..9348a9ec82 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/cast.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/cast.rs @@ -90,7 +90,7 @@ impl PluginCommand for CastDF { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuLazyFrame(lazy) => { let (dtype, column_nm) = df_args(call)?; diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/collect.rs b/crates/nu_plugin_polars/src/dataframe/lazy/collect.rs index db62426e83..b6d8909e31 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/collect.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/collect.rs @@ -61,7 +61,7 @@ impl PluginCommand for LazyCollect { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuLazyFrame(lazy) => { let eager = lazy.collect(call.head)?; diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/explode.rs b/crates/nu_plugin_polars/src/dataframe/lazy/explode.rs index b0609d7a3c..787f07fd46 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/explode.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/explode.rs @@ -50,7 +50,7 @@ impl PluginCommand for LazyExplode { result: Some( NuDataFrame::try_from_columns(vec![ Column::new( - "id".to_string(), + "id".to_string(), vec![ Value::test_int(1), Value::test_int(1), @@ -58,7 +58,7 @@ impl PluginCommand for LazyExplode { Value::test_int(2), ]), Column::new( - "name".to_string(), + "name".to_string(), vec![ Value::test_string("Mercy"), Value::test_string("Mercy"), @@ -66,7 +66,7 @@ impl PluginCommand for LazyExplode { Value::test_string("Bob"), ]), Column::new( - "hobbies".to_string(), + "hobbies".to_string(), vec![ Value::test_string("Cycling"), Value::test_string("Knitting"), @@ -84,7 +84,7 @@ impl PluginCommand for LazyExplode { result: Some( NuDataFrame::try_from_columns(vec![ Column::new( - "hobbies".to_string(), + "hobbies".to_string(), vec![ Value::test_string("Cycling"), Value::test_string("Knitting"), @@ -116,8 +116,7 @@ pub(crate) fn explode( call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => { let lazy = df.lazy(); diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/fetch.rs b/crates/nu_plugin_polars/src/dataframe/lazy/fetch.rs index 49d917a393..8fee4cd159 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/fetch.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/fetch.rs @@ -67,7 +67,7 @@ impl PluginCommand for LazyFetch { input: PipelineData, ) -> Result { let rows: i64 = call.req(0)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; let eager: NuDataFrame = lazy diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/fill_nan.rs b/crates/nu_plugin_polars/src/dataframe/lazy/fill_nan.rs index 851be588f9..baeb9da01b 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/fill_nan.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/fill_nan.rs @@ -92,7 +92,7 @@ impl PluginCommand for LazyFillNA { input: PipelineData, ) -> Result { let fill: Value = call.req(0)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => { diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/fill_null.rs b/crates/nu_plugin_polars/src/dataframe/lazy/fill_null.rs index 64e6fd0d3f..c5fb67cd8a 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/fill_null.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/fill_null.rs @@ -69,7 +69,7 @@ impl PluginCommand for LazyFillNull { input: PipelineData, ) -> Result { let fill: Value = call.req(0)?; - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => cmd_lazy(plugin, engine, call, df.lazy(), fill), diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/filter.rs b/crates/nu_plugin_polars/src/dataframe/lazy/filter.rs index 6adabb967a..f8d400ddf4 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/filter.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/filter.rs @@ -72,7 +72,7 @@ impl PluginCommand for LazyFilter { ) -> Result { let expr_value: Value = call.req(0)?; let filter_expr = NuExpression::try_from_value(plugin, &expr_value)?; - let pipeline_value = input.into_value(call.head); + let pipeline_value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &pipeline_value)?; command(plugin, engine, call, lazy, filter_expr).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/filter_with.rs b/crates/nu_plugin_polars/src/dataframe/lazy/filter_with.rs index cd23a3b370..12ccfbc376 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/filter_with.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/filter_with.rs @@ -67,7 +67,7 @@ impl PluginCommand for FilterWith { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/first.rs b/crates/nu_plugin_polars/src/dataframe/lazy/first.rs index 7f32dbf71d..4692a933b0 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/first.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/first.rs @@ -97,7 +97,7 @@ impl PluginCommand for FirstDF { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuLazyFrame::can_downcast(&value) || NuDataFrame::can_downcast(&value) { let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command(plugin, engine, call, lazy).map_err(LabeledError::from) diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/groupby.rs b/crates/nu_plugin_polars/src/dataframe/lazy/groupby.rs index 2bc7f578c8..7aaccfead9 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/groupby.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/groupby.rs @@ -138,7 +138,7 @@ impl PluginCommand for ToLazyGroupBy { })?; } - let pipeline_value = input.into_value(call.head); + let pipeline_value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &pipeline_value)?; command(plugin, engine, call, lazy, expressions).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/join.rs b/crates/nu_plugin_polars/src/dataframe/lazy/join.rs index feea8cf308..6db0269403 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/join.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/join.rs @@ -228,7 +228,7 @@ impl PluginCommand for LazyJoin { let suffix: Option = call.get_flag("suffix")?; let suffix = suffix.unwrap_or_else(|| "_x".into()); - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; let lazy = lazy.to_polars(); diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/last.rs b/crates/nu_plugin_polars/src/dataframe/lazy/last.rs index 44095ac44f..0453c71d1e 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/last.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/last.rs @@ -72,7 +72,7 @@ impl PluginCommand for LastDF { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) { let df = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command(plugin, engine, call, df).map_err(|e| e.into()) diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/median.rs b/crates/nu_plugin_polars/src/dataframe/lazy/median.rs index abd55c77c1..ffd69d14e4 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/median.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/median.rs @@ -89,7 +89,7 @@ impl PluginCommand for LazyMedian { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => command(plugin, engine, call, df.lazy()), PolarsPluginObject::NuLazyFrame(lazy) => command(plugin, engine, call, lazy), diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/quantile.rs b/crates/nu_plugin_polars/src/dataframe/lazy/quantile.rs index 46339cc9fc..f6217ff89b 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/quantile.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/quantile.rs @@ -97,7 +97,7 @@ impl PluginCommand for LazyQuantile { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let quantile: f64 = call.req(0)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => { diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/rename.rs b/crates/nu_plugin_polars/src/dataframe/lazy/rename.rs index b678824584..c32b8d9451 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/rename.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/rename.rs @@ -120,7 +120,7 @@ impl PluginCommand for RenameDF { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/select.rs b/crates/nu_plugin_polars/src/dataframe/lazy/select.rs index e49aa8e654..75b3f8f804 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/select.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/select.rs @@ -65,7 +65,7 @@ impl PluginCommand for LazySelect { let expr_value = Value::list(vals, call.head); let expressions = NuExpression::extract_exprs(plugin, expr_value)?; - let pipeline_value = input.into_value(call.head); + let pipeline_value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &pipeline_value)?; let lazy = NuLazyFrame::new(lazy.to_polars().select(&expressions)); lazy.to_pipeline_data(plugin, engine, call.head) diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/sort_by_expr.rs b/crates/nu_plugin_polars/src/dataframe/lazy/sort_by_expr.rs index 4a975afe97..2beba4424c 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/sort_by_expr.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/sort_by_expr.rs @@ -145,7 +145,7 @@ impl PluginCommand for LazySortBy { maintain_order, }; - let pipeline_value = input.into_value(call.head); + let pipeline_value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &pipeline_value)?; let lazy = NuLazyFrame::new(lazy.to_polars().sort_by_exprs(&expressions, sort_options)); lazy.to_pipeline_data(plugin, engine, call.head) diff --git a/crates/nu_plugin_polars/src/dataframe/lazy/with_column.rs b/crates/nu_plugin_polars/src/dataframe/lazy/with_column.rs index e8092231d8..d2f953b068 100644 --- a/crates/nu_plugin_polars/src/dataframe/lazy/with_column.rs +++ b/crates/nu_plugin_polars/src/dataframe/lazy/with_column.rs @@ -83,7 +83,7 @@ impl PluginCommand for WithColumn { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/series/masks/is_not_null.rs b/crates/nu_plugin_polars/src/dataframe/series/masks/is_not_null.rs index 218cd116b4..b7e506a67c 100644 --- a/crates/nu_plugin_polars/src/dataframe/series/masks/is_not_null.rs +++ b/crates/nu_plugin_polars/src/dataframe/series/masks/is_not_null.rs @@ -78,8 +78,7 @@ impl PluginCommand for IsNotNull { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => command(plugin, engine, call, df), PolarsPluginObject::NuLazyFrame(lazy) => { diff --git a/crates/nu_plugin_polars/src/dataframe/series/masks/is_null.rs b/crates/nu_plugin_polars/src/dataframe/series/masks/is_null.rs index beb3793661..bc04e7fb76 100644 --- a/crates/nu_plugin_polars/src/dataframe/series/masks/is_null.rs +++ b/crates/nu_plugin_polars/src/dataframe/series/masks/is_null.rs @@ -80,8 +80,7 @@ impl PluginCommand for IsNull { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => command(plugin, engine, call, df), PolarsPluginObject::NuLazyFrame(lazy) => { diff --git a/crates/nu_plugin_polars/src/dataframe/series/n_unique.rs b/crates/nu_plugin_polars/src/dataframe/series/n_unique.rs index 5426ef6d1d..51c6e1bdb3 100644 --- a/crates/nu_plugin_polars/src/dataframe/series/n_unique.rs +++ b/crates/nu_plugin_polars/src/dataframe/series/n_unique.rs @@ -70,8 +70,7 @@ impl PluginCommand for NUnique { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); - + let value = input.into_value(call.head)?; match PolarsPluginObject::try_from_value(plugin, &value)? { PolarsPluginObject::NuDataFrame(df) => command(plugin, engine, call, df), PolarsPluginObject::NuLazyFrame(lazy) => { diff --git a/crates/nu_plugin_polars/src/dataframe/series/shift.rs b/crates/nu_plugin_polars/src/dataframe/series/shift.rs index 556b3361c1..c37ba2f2e9 100644 --- a/crates/nu_plugin_polars/src/dataframe/series/shift.rs +++ b/crates/nu_plugin_polars/src/dataframe/series/shift.rs @@ -92,7 +92,7 @@ impl PluginCommand for Shift { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/series/unique.rs b/crates/nu_plugin_polars/src/dataframe/series/unique.rs index 47efd880a6..2475ad026a 100644 --- a/crates/nu_plugin_polars/src/dataframe/series/unique.rs +++ b/crates/nu_plugin_polars/src/dataframe/series/unique.rs @@ -134,7 +134,7 @@ impl PluginCommand for Unique { call: &EvaluatedCall, input: PipelineData, ) -> Result { - let value = input.into_value(call.head); + let value = input.into_value(call.head)?; let df = NuLazyFrame::try_from_value_coerce(plugin, &value)?; command_lazy(plugin, engine, call, df).map_err(LabeledError::from) } diff --git a/crates/nu_plugin_polars/src/dataframe/values/mod.rs b/crates/nu_plugin_polars/src/dataframe/values/mod.rs index a43c8f2412..179c85bf36 100644 --- a/crates/nu_plugin_polars/src/dataframe/values/mod.rs +++ b/crates/nu_plugin_polars/src/dataframe/values/mod.rs @@ -84,7 +84,7 @@ impl PolarsPluginObject { input: PipelineData, span: Span, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(plugin, &value) } @@ -242,7 +242,7 @@ pub trait PolarsPluginCustomValue: CustomValue { /// Handles the ability for a PolarsObjectType implementations to convert between /// their respective CustValue type. /// PolarsPluginObjectType's (NuDataFrame, NuLazyFrame) should -/// implement this trait. +/// implement this trait. pub trait CustomValueSupport: Cacheable { type CV: PolarsPluginCustomValue + CustomValue + 'static; @@ -301,7 +301,7 @@ pub trait CustomValueSupport: Cacheable { input: PipelineData, span: Span, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value(plugin, &value) } diff --git a/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/between_values.rs b/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/between_values.rs index df0854ffee..a47197bde8 100644 --- a/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/between_values.rs +++ b/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/between_values.rs @@ -1,7 +1,7 @@ use super::{operations::Axis, NuDataFrame}; use nu_protocol::{ ast::{Boolean, Comparison, Math, Operator}, - span, ShellError, Span, Spanned, Value, + ShellError, Span, Spanned, Value, }; use num::Zero; use polars::prelude::{ @@ -17,9 +17,10 @@ pub(super) fn between_dataframes( right: &Value, rhs: &NuDataFrame, ) -> Result { - let operation_span = span(&[left.span(), right.span()]); match operator.item { - Operator::Math(Math::Plus) => lhs.append_df(rhs, Axis::Row, operation_span), + Operator::Math(Math::Plus) => { + lhs.append_df(rhs, Axis::Row, Span::merge(left.span(), right.span())) + } _ => Err(ShellError::OperatorMismatch { op_span: operator.span, lhs_ty: left.get_type().to_string(), @@ -37,7 +38,7 @@ pub(super) fn compute_between_series( right: &Value, rhs: &Series, ) -> Result { - let operation_span = span(&[left.span(), right.span()]); + let operation_span = Span::merge(left.span(), right.span()); match operator.item { Operator::Math(Math::Plus) => { let mut res = lhs + rhs; diff --git a/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/mod.rs b/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/mod.rs index b9c4c54f47..371ff20d13 100644 --- a/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/mod.rs +++ b/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/mod.rs @@ -517,7 +517,7 @@ impl NuDataFrame { input: PipelineData, span: Span, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value_coerce(plugin, &value, span) } } diff --git a/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/operations.rs b/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/operations.rs index 42d803b0e4..db5409aff2 100644 --- a/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/operations.rs +++ b/crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/operations.rs @@ -27,7 +27,7 @@ impl NuDataFrame { let rhs_span = right.span(); match right { Value::Custom { .. } => { - let rhs = NuDataFrame::try_from_value(plugin, right)?; + let rhs = NuDataFrame::try_from_value_coerce(plugin, right, rhs_span)?; match (self.is_series(), rhs.is_series()) { (true, true) => { diff --git a/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/custom_value.rs b/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/custom_value.rs index 731d210dd8..b3cd9500dd 100644 --- a/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/custom_value.rs +++ b/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/custom_value.rs @@ -7,7 +7,7 @@ use uuid::Uuid; use crate::{ values::{CustomValueSupport, NuDataFrame, PolarsPluginCustomValue}, - PolarsPlugin, + Cacheable, PolarsPlugin, }; use super::NuLazyFrame; @@ -76,11 +76,49 @@ impl PolarsPluginCustomValue for NuLazyFrameCustomValue { _engine: &EngineInterface, other_value: Value, ) -> Result, ShellError> { - // to compare, we need to convert to NuDataframe - let df = NuLazyFrame::try_from_custom_value(plugin, self)?; - let df = df.collect(other_value.span())?; + let eager = NuLazyFrame::try_from_custom_value(plugin, self)?.collect(Span::unknown())?; let other = NuDataFrame::try_from_value_coerce(plugin, &other_value, other_value.span())?; - let res = df.is_equal(&other); + let res = eager.is_equal(&other); Ok(res) } + + fn custom_value_operation( + &self, + plugin: &PolarsPlugin, + engine: &EngineInterface, + lhs_span: Span, + operator: nu_protocol::Spanned, + right: Value, + ) -> Result { + let eager = NuLazyFrame::try_from_custom_value(plugin, self)?.collect(Span::unknown())?; + Ok(eager + .compute_with_value(plugin, lhs_span, operator.item, operator.span, &right)? + .cache(plugin, engine, lhs_span)? + .into_value(lhs_span)) + } + + fn custom_value_follow_path_int( + &self, + plugin: &PolarsPlugin, + _engine: &EngineInterface, + _self_span: Span, + index: nu_protocol::Spanned, + ) -> Result { + let eager = NuLazyFrame::try_from_custom_value(plugin, self)?.collect(Span::unknown())?; + eager.get_value(index.item, index.span) + } + + fn custom_value_follow_path_string( + &self, + plugin: &PolarsPlugin, + engine: &EngineInterface, + self_span: Span, + column_name: nu_protocol::Spanned, + ) -> Result { + let eager = NuLazyFrame::try_from_custom_value(plugin, self)?.collect(Span::unknown())?; + let column = eager.column(&column_name.item, self_span)?; + Ok(column + .cache(plugin, engine, self_span)? + .into_value(self_span)) + } } diff --git a/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/mod.rs b/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/mod.rs index f3c969b03d..48e296e95e 100644 --- a/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/mod.rs +++ b/crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/mod.rs @@ -109,7 +109,7 @@ impl NuLazyFrame { input: PipelineData, span: Span, ) -> Result { - let value = input.into_value(span); + let value = input.into_value(span)?; Self::try_from_value_coerce(plugin, &value) } } diff --git a/crates/nu_plugin_polars/src/dataframe/values/utils.rs b/crates/nu_plugin_polars/src/dataframe/values/utils.rs index f77870114b..88ce8a4656 100644 --- a/crates/nu_plugin_polars/src/dataframe/values/utils.rs +++ b/crates/nu_plugin_polars/src/dataframe/values/utils.rs @@ -1,4 +1,4 @@ -use nu_protocol::{span as span_join, ShellError, Span, Spanned, Value}; +use nu_protocol::{ShellError, Span, Spanned, Value}; // Default value used when selecting rows from dataframe pub const DEFAULT_ROWS: usize = 5; @@ -20,8 +20,8 @@ pub(crate) fn convert_columns( span: Some(span), help: None, inner: vec![], - }) - .map(|v| v.span())?; + })? + .span(); let res = columns .into_iter() @@ -29,7 +29,7 @@ pub(crate) fn convert_columns( let span = value.span(); match value { Value::String { val, .. } => { - col_span = span_join(&[col_span, span]); + col_span = col_span.merge(span); Ok(Spanned { item: val, span }) } _ => Err(ShellError::GenericError { @@ -70,7 +70,7 @@ pub(crate) fn convert_columns_string( let span = value.span(); match value { Value::String { val, .. } => { - col_span = span_join(&[col_span, span]); + col_span = col_span.merge(span); Ok(val) } _ => Err(ShellError::GenericError { diff --git a/crates/nuon/src/to.rs b/crates/nuon/src/to.rs index 1a5306a135..c35159b92b 100644 --- a/crates/nuon/src/to.rs +++ b/crates/nuon/src/to.rs @@ -176,7 +176,7 @@ fn value_to_string( } } Value::Nothing { .. } => Ok("null".to_string()), - Value::Range { val, .. } => match val { + Value::Range { val, .. } => match **val { Range::IntRange(range) => Ok(range.to_string()), Range::FloatRange(range) => { let start = diff --git a/src/command.rs b/src/command.rs index 0d9aaf925f..ab7a74884e 100644 --- a/src/command.rs +++ b/src/command.rs @@ -67,7 +67,6 @@ pub(crate) fn parse_commandline_args( let output = parse(&mut working_set, None, commandline_args.as_bytes(), false); if let Some(err) = working_set.parse_errors.first() { report_error(&working_set, err); - std::process::exit(1); } diff --git a/src/config_files.rs b/src/config_files.rs index 6b2e5bb16d..ec4511860f 100644 --- a/src/config_files.rs +++ b/src/config_files.rs @@ -1,11 +1,11 @@ -use log::{info, trace}; +use log::warn; #[cfg(feature = "plugin")] use nu_cli::read_plugin_file; use nu_cli::{eval_config_contents, eval_source}; use nu_path::canonicalize_with; use nu_protocol::{ engine::{EngineState, Stack, StateWorkingSet}, - report_error, Config, ParseError, PipelineData, Spanned, + report_error, report_error_new, Config, ParseError, PipelineData, Spanned, }; use nu_utils::{get_default_config, get_default_env}; use std::{ @@ -27,7 +27,10 @@ pub(crate) fn read_config_file( config_file: Option>, is_env_config: bool, ) { - trace!("read_config_file {:?}", &config_file); + warn!( + "read_config_file() config_file_specified: {:?}, is_env_config: {is_env_config}", + &config_file + ); // Load config startup file if let Some(file) = config_file { let working_set = StateWorkingSet::new(engine_state); @@ -122,21 +125,27 @@ pub(crate) fn read_config_file( } pub(crate) fn read_loginshell_file(engine_state: &mut EngineState, stack: &mut Stack) { + warn!( + "read_loginshell_file() {}:{}:{}", + file!(), + line!(), + column!() + ); + // read and execute loginshell file if exists if let Some(mut config_path) = nu_path::config_dir() { config_path.push(NUSHELL_FOLDER); config_path.push(LOGINSHELL_FILE); + warn!("loginshell_file: {}", config_path.display()); + if config_path.exists() { eval_config_contents(config_path, engine_state, stack); } } - - info!("read_loginshell_file {}:{}:{}", file!(), line!(), column!()); } pub(crate) fn read_default_env_file(engine_state: &mut EngineState, stack: &mut Stack) { - trace!("read_default_env_file"); let config_file = get_default_env(); eval_source( engine_state, @@ -147,18 +156,22 @@ pub(crate) fn read_default_env_file(engine_state: &mut EngineState, stack: &mut false, ); - info!("read_config_file {}:{}:{}", file!(), line!(), column!()); + warn!( + "read_default_env_file() env_file_contents: {config_file} {}:{}:{}", + file!(), + line!(), + column!() + ); + // Merge the environment in case env vars changed in the config match engine_state.cwd(Some(stack)) { Ok(cwd) => { if let Err(e) = engine_state.merge_env(stack, cwd) { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &e); + report_error_new(engine_state, &e); } } Err(e) => { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &e); + report_error_new(engine_state, &e); } } } @@ -169,10 +182,9 @@ fn eval_default_config( config_file: &str, is_env_config: bool, ) { - trace!( - "eval_default_config: config_file: {:?}, is_env_config: {}", - &config_file, - is_env_config + warn!( + "eval_default_config() config_file_specified: {:?}, is_env_config: {}", + &config_file, is_env_config ); println!("Continuing without config file"); // Just use the contents of "default_config.nu" or "default_env.nu" @@ -193,13 +205,11 @@ fn eval_default_config( match engine_state.cwd(Some(stack)) { Ok(cwd) => { if let Err(e) = engine_state.merge_env(stack, cwd) { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &e); + report_error_new(engine_state, &e); } } Err(e) => { - let working_set = StateWorkingSet::new(engine_state); - report_error(&working_set, &e); + report_error_new(engine_state, &e); } } } @@ -212,11 +222,9 @@ pub(crate) fn setup_config( env_file: Option>, is_login_shell: bool, ) { - trace!( - "setup_config: config: {:?}, env: {:?}, login: {}", - &config_file, - &env_file, - is_login_shell + warn!( + "setup_config() config_file_specified: {:?}, env_file_specified: {:?}, login: {}", + &config_file, &env_file, is_login_shell ); let result = catch_unwind(AssertUnwindSafe(|| { #[cfg(feature = "plugin")] @@ -244,12 +252,9 @@ pub(crate) fn set_config_path( key: &str, config_file: Option<&Spanned>, ) { - trace!( - "set_config_path: cwd: {:?}, default_config: {}, key: {}, config_file: {:?}", - &cwd, - &default_config_name, - &key, - &config_file + warn!( + "set_config_path() cwd: {:?}, default_config: {}, key: {}, config_file_specified: {:?}", + &cwd, &default_config_name, &key, &config_file ); let config_path = match config_file { Some(s) => canonicalize_with(&s.item, cwd).ok(), diff --git a/src/ide.rs b/src/ide.rs index 2b39dda946..a3474fe3b6 100644 --- a/src/ide.rs +++ b/src/ide.rs @@ -3,8 +3,7 @@ use nu_cli::NuCompleter; use nu_parser::{flatten_block, parse, FlatShape}; use nu_protocol::{ engine::{EngineState, Stack, StateWorkingSet}, - eval_const::create_nu_constant, - report_error, DeclId, ShellError, Span, Value, VarId, NU_VARIABLE_ID, + report_error_new, DeclId, ShellError, Span, Value, VarId, }; use reedline::Completer; use serde_json::{json, Value as JsonValue}; @@ -56,9 +55,8 @@ fn read_in_file<'a>( let file = std::fs::read(file_path) .into_diagnostic() .unwrap_or_else(|e| { - let working_set = StateWorkingSet::new(engine_state); - report_error( - &working_set, + report_error_new( + engine_state, &ShellError::FileNotFoundCustom { msg: format!("Could not read file '{}': {:?}", file_path, e.to_string()), span: Span::unknown(), @@ -77,16 +75,7 @@ fn read_in_file<'a>( pub fn check(engine_state: &mut EngineState, file_path: &str, max_errors: &Value) { let cwd = std::env::current_dir().expect("Could not get current working directory."); engine_state.add_env_var("PWD".into(), Value::test_string(cwd.to_string_lossy())); - let working_set = StateWorkingSet::new(engine_state); - - let nu_const = match create_nu_constant(engine_state, Span::unknown()) { - Ok(nu_const) => nu_const, - Err(err) => { - report_error(&working_set, &err); - std::process::exit(1); - } - }; - engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const); + engine_state.generate_nu_constant(); let mut working_set = StateWorkingSet::new(engine_state); let file = std::fs::read(file_path); @@ -606,8 +595,7 @@ pub fn hover(engine_state: &mut EngineState, file_path: &str, location: &Value) } pub fn complete(engine_reference: Arc, file_path: &str, location: &Value) { - let stack = Stack::new(); - let mut completer = NuCompleter::new(engine_reference, stack); + let mut completer = NuCompleter::new(engine_reference, Arc::new(Stack::new())); let file = std::fs::read(file_path) .into_diagnostic() diff --git a/src/main.rs b/src/main.rs index b7e70f1ed4..d0fc023b68 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,8 +7,6 @@ mod signals; #[cfg(unix)] mod terminal; mod test_bins; -#[cfg(test)] -mod tests; #[cfg(feature = "mimalloc")] #[global_allocator] @@ -27,15 +25,13 @@ use nu_cmd_base::util::get_init_cwd; use nu_lsp::LanguageServer; use nu_path::canonicalize_with; use nu_protocol::{ - engine::EngineState, eval_const::create_nu_constant, report_error_new, util::BufferedReader, - PipelineData, RawStream, ShellError, Span, Value, NU_VARIABLE_ID, + engine::EngineState, report_error_new, ByteStream, PipelineData, ShellError, Span, Value, }; use nu_std::load_standard_library; use nu_utils::utils::perf; use run::{run_commands, run_file, run_repl}; use signals::ctrlc_protection; use std::{ - io::BufReader, path::PathBuf, str::FromStr, sync::{atomic::AtomicBool, Arc}, @@ -347,22 +343,7 @@ fn main() -> Result<()> { start_time = std::time::Instant::now(); let input = if let Some(redirect_stdin) = &parsed_nu_cli_args.redirect_stdin { trace!("redirecting stdin"); - let stdin = std::io::stdin(); - let buf_reader = BufReader::new(stdin); - - PipelineData::ExternalStream { - stdout: Some(RawStream::new( - Box::new(BufferedReader::new(buf_reader)), - Some(ctrlc.clone()), - redirect_stdin.span, - None, - )), - stderr: None, - exit_code: None, - span: redirect_stdin.span, - metadata: None, - trim_end_newline: false, - } + PipelineData::ByteStream(ByteStream::stdin(redirect_stdin.span)?, None) } else { trace!("not redirecting stdin"); PipelineData::empty() @@ -378,8 +359,7 @@ fn main() -> Result<()> { start_time = std::time::Instant::now(); // Set up the $nu constant before evaluating config files (need to have $nu available in them) - let nu_const = create_nu_constant(&engine_state, input.span().unwrap_or_else(Span::unknown))?; - engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const); + engine_state.generate_nu_constant(); perf( "create_nu_constant", start_time, @@ -453,7 +433,7 @@ fn main() -> Result<()> { ); } - LanguageServer::initialize_stdio_connection()?.serve_requests(engine_state, ctrlc) + LanguageServer::initialize_stdio_connection()?.serve_requests(engine_state, ctrlc)? } else if let Some(commands) = parsed_nu_cli_args.commands.clone() { run_commands( &mut engine_state, @@ -462,7 +442,7 @@ fn main() -> Result<()> { &commands, input, entire_start_time, - ) + ); } else if !script_name.is_empty() { run_file( &mut engine_state, @@ -471,8 +451,10 @@ fn main() -> Result<()> { script_name, args_to_script, input, - ) + ); } else { - run_repl(&mut engine_state, parsed_nu_cli_args, entire_start_time) + run_repl(&mut engine_state, parsed_nu_cli_args, entire_start_time)? } + + Ok(()) } diff --git a/src/run.rs b/src/run.rs index 81274df3de..2996bc76f8 100644 --- a/src/run.rs +++ b/src/run.rs @@ -8,19 +8,22 @@ use log::trace; #[cfg(feature = "plugin")] use nu_cli::read_plugin_file; use nu_cli::{evaluate_commands, evaluate_file, evaluate_repl}; -use nu_protocol::{eval_const::create_nu_constant, PipelineData, Span, NU_VARIABLE_ID}; +use nu_protocol::{ + engine::{EngineState, Stack}, + report_error_new, PipelineData, Spanned, +}; use nu_utils::utils::perf; pub(crate) fn run_commands( - engine_state: &mut nu_protocol::engine::EngineState, + engine_state: &mut EngineState, parsed_nu_cli_args: command::NushellCliArgs, use_color: bool, - commands: &nu_protocol::Spanned, + commands: &Spanned, input: PipelineData, entire_start_time: std::time::Instant, -) -> Result<(), miette::ErrReport> { +) { trace!("run_commands"); - let mut stack = nu_protocol::engine::Stack::new(); + let mut stack = Stack::new(); let start_time = std::time::Instant::now(); // if the --no-config-file(-n) option is NOT passed, load the plugin file, @@ -103,18 +106,20 @@ pub(crate) fn run_commands( engine_state.set_startup_time(entire_start_time.elapsed().as_nanos() as i64); // Regenerate the $nu constant to contain the startup time and any other potential updates - let nu_const = create_nu_constant(engine_state, commands.span)?; - engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const); + engine_state.generate_nu_constant(); let start_time = std::time::Instant::now(); - let ret_val = evaluate_commands( + if let Err(err) = evaluate_commands( commands, engine_state, &mut stack, input, parsed_nu_cli_args.table_mode, parsed_nu_cli_args.no_newline.is_some(), - ); + ) { + report_error_new(engine_state, &err); + std::process::exit(1); + } perf( "evaluate_commands", start_time, @@ -123,24 +128,18 @@ pub(crate) fn run_commands( column!(), use_color, ); - - match ret_val { - Ok(Some(exit_code)) => std::process::exit(exit_code as i32), - Ok(None) => Ok(()), - Err(e) => Err(e), - } } pub(crate) fn run_file( - engine_state: &mut nu_protocol::engine::EngineState, + engine_state: &mut EngineState, parsed_nu_cli_args: command::NushellCliArgs, use_color: bool, script_name: String, args_to_script: Vec, input: PipelineData, -) -> Result<(), miette::ErrReport> { +) { trace!("run_file"); - let mut stack = nu_protocol::engine::Stack::new(); + let mut stack = Stack::new(); // if the --no-config-file(-n) option is NOT passed, load the plugin file, // load the default env file or custom (depending on parsed_nu_cli_args.env_file), @@ -201,17 +200,19 @@ pub(crate) fn run_file( } // Regenerate the $nu constant to contain the startup time and any other potential updates - let nu_const = create_nu_constant(engine_state, input.span().unwrap_or_else(Span::unknown))?; - engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const); + engine_state.generate_nu_constant(); let start_time = std::time::Instant::now(); - let ret_val = evaluate_file( + if let Err(err) = evaluate_file( script_name, &args_to_script, engine_state, &mut stack, input, - ); + ) { + report_error_new(engine_state, &err); + std::process::exit(1); + } perf( "evaluate_file", start_time, @@ -239,17 +240,15 @@ pub(crate) fn run_file( column!(), use_color, ); - - ret_val } pub(crate) fn run_repl( - engine_state: &mut nu_protocol::engine::EngineState, + engine_state: &mut EngineState, parsed_nu_cli_args: command::NushellCliArgs, entire_start_time: std::time::Instant, ) -> Result<(), miette::ErrReport> { trace!("run_repl"); - let mut stack = nu_protocol::engine::Stack::new(); + let mut stack = Stack::new(); let start_time = std::time::Instant::now(); if parsed_nu_cli_args.no_config_file.is_none() { diff --git a/tests/main.rs b/tests/main.rs index db44c4e019..6824f26f13 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -11,5 +11,6 @@ mod path; mod plugin_persistence; #[cfg(feature = "plugin")] mod plugins; +mod repl; mod scope; mod shell; diff --git a/tests/plugins/stream.rs b/tests/plugins/stream.rs index ee62703017..b8771580f7 100644 --- a/tests/plugins/stream.rs +++ b/tests/plugins/stream.rs @@ -1,3 +1,5 @@ +use rstest::rstest; + use nu_test_support::nu_with_plugins; use pretty_assertions::assert_eq; @@ -117,40 +119,40 @@ fn sum_big_stream() { } #[test] -fn collect_external_accepts_list_of_string() { +fn collect_bytes_accepts_list_of_string() { let actual = nu_with_plugins!( cwd: "tests/fixtures/formats", plugin: ("nu_plugin_example"), - "[a b] | example collect-external" + "[a b] | example collect-bytes" ); assert_eq!(actual.out, "ab"); } #[test] -fn collect_external_accepts_list_of_binary() { +fn collect_bytes_accepts_list_of_binary() { let actual = nu_with_plugins!( cwd: "tests/fixtures/formats", plugin: ("nu_plugin_example"), - "[0x[41] 0x[42]] | example collect-external" + "[0x[41] 0x[42]] | example collect-bytes" ); assert_eq!(actual.out, "AB"); } #[test] -fn collect_external_produces_raw_input() { +fn collect_bytes_produces_byte_stream() { let actual = nu_with_plugins!( cwd: "tests/fixtures/formats", plugin: ("nu_plugin_example"), - "[a b c] | example collect-external | describe" + "[a b c] | example collect-bytes | describe" ); - assert_eq!(actual.out, "raw input"); + assert_eq!(actual.out, "byte stream"); } #[test] -fn collect_external_big_stream() { +fn collect_bytes_big_stream() { // This in particular helps to ensure that a big stream can be both read and written at the same // time without deadlocking let actual = nu_with_plugins!( @@ -158,9 +160,8 @@ fn collect_external_big_stream() { plugin: ("nu_plugin_example"), r#"( seq 1 10000 | - to text | - each { into string } | - example collect-external | + each {|i| ($i | into string) ++ (char newline) } | + example collect-bytes | lines | length )"# @@ -190,3 +191,18 @@ fn generate_sequence() { assert_eq!(actual.out, "[0,2,4,6,8,10]"); } + +#[rstest] +#[timeout(std::time::Duration::from_secs(6))] +fn echo_interactivity_on_slow_pipelines() { + // This test works by putting 0 on the upstream immediately, followed by 1 after 10 seconds. + // If values aren't streamed to the plugin as they become available, `example echo` won't emit + // anything until both 0 and 1 are available. The desired behavior is that `example echo` gets + // the 0 immediately, which is consumed by `first`, allowing the pipeline to terminate early. + let actual = nu_with_plugins!( + cwd: "tests/fixtures/formats", + plugin: ("nu_plugin_example"), + r#"[1] | each { |n| sleep 10sec; $n } | prepend 0 | example echo | first"# + ); + assert_eq!(actual.out, "0"); +} diff --git a/tests/repl/mod.rs b/tests/repl/mod.rs new file mode 100644 index 0000000000..5ed7f29ba5 --- /dev/null +++ b/tests/repl/mod.rs @@ -0,0 +1,27 @@ +mod test_bits; +mod test_cell_path; +mod test_commandline; +mod test_conditionals; +mod test_config; +mod test_config_path; +mod test_converters; +mod test_custom_commands; +mod test_engine; +mod test_env; +mod test_help; +mod test_hiding; +mod test_ide; +mod test_iteration; +mod test_known_external; +mod test_math; +mod test_modules; +mod test_parser; +mod test_ranges; +mod test_regex; +mod test_signatures; +mod test_spread; +mod test_stdlib; +mod test_strings; +mod test_table_operations; +mod test_type_check; +mod tests; diff --git a/src/tests/test_bits.rs b/tests/repl/test_bits.rs similarity index 97% rename from src/tests/test_bits.rs rename to tests/repl/test_bits.rs index fdb672d023..410f48d83b 100644 --- a/src/tests/test_bits.rs +++ b/tests/repl/test_bits.rs @@ -1,4 +1,4 @@ -use crate::tests::{run_test, TestResult}; +use crate::repl::tests::{run_test, TestResult}; #[test] fn bits_and() -> TestResult { diff --git a/src/tests/test_cell_path.rs b/tests/repl/test_cell_path.rs similarity index 98% rename from src/tests/test_cell_path.rs rename to tests/repl/test_cell_path.rs index 56b6751638..c89c50259b 100644 --- a/src/tests/test_cell_path.rs +++ b/tests/repl/test_cell_path.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; // Tests for null / null / Value::Nothing #[test] diff --git a/src/tests/test_commandline.rs b/tests/repl/test_commandline.rs similarity index 98% rename from src/tests/test_commandline.rs rename to tests/repl/test_commandline.rs index 130faf933a..0443ee4d6c 100644 --- a/src/tests/test_commandline.rs +++ b/tests/repl/test_commandline.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; #[test] fn commandline_test_get_empty() -> TestResult { diff --git a/src/tests/test_conditionals.rs b/tests/repl/test_conditionals.rs similarity index 96% rename from src/tests/test_conditionals.rs rename to tests/repl/test_conditionals.rs index 20307ffe65..dd1e16c5a8 100644 --- a/src/tests/test_conditionals.rs +++ b/tests/repl/test_conditionals.rs @@ -1,4 +1,4 @@ -use crate::tests::{run_test, TestResult}; +use crate::repl::tests::{run_test, TestResult}; #[test] fn if_test1() -> TestResult { diff --git a/src/tests/test_config.rs b/tests/repl/test_config.rs similarity index 98% rename from src/tests/test_config.rs rename to tests/repl/test_config.rs index 96bd7f714d..005ae63171 100644 --- a/src/tests/test_config.rs +++ b/tests/repl/test_config.rs @@ -1,5 +1,4 @@ -use super::{fail_test, run_test, run_test_std}; -use crate::tests::TestResult; +use crate::repl::tests::{fail_test, run_test, run_test_std, TestResult}; #[test] fn mutate_nu_config() -> TestResult { diff --git a/src/tests/test_config_path.rs b/tests/repl/test_config_path.rs similarity index 100% rename from src/tests/test_config_path.rs rename to tests/repl/test_config_path.rs diff --git a/src/tests/test_converters.rs b/tests/repl/test_converters.rs similarity index 96% rename from src/tests/test_converters.rs rename to tests/repl/test_converters.rs index c41868539e..cfa165e95d 100644 --- a/src/tests/test_converters.rs +++ b/tests/repl/test_converters.rs @@ -1,4 +1,4 @@ -use crate::tests::{run_test, TestResult}; +use crate::repl::tests::{run_test, TestResult}; #[test] fn from_json_1() -> TestResult { diff --git a/src/tests/test_custom_commands.rs b/tests/repl/test_custom_commands.rs similarity index 96% rename from src/tests/test_custom_commands.rs rename to tests/repl/test_custom_commands.rs index 4cc81878e4..1f36b0e90c 100644 --- a/src/tests/test_custom_commands.rs +++ b/tests/repl/test_custom_commands.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, run_test_contains, TestResult}; +use crate::repl::tests::{fail_test, run_test, run_test_contains, TestResult}; use nu_test_support::nu; use pretty_assertions::assert_eq; @@ -274,3 +274,9 @@ fn dont_allow_implicit_casting_between_glob_and_string() -> TestResult { "can't convert", ) } + +#[test] +fn allow_pass_negative_float() -> TestResult { + run_test("def spam [val: float] { $val }; spam -1.4", "-1.4")?; + run_test("def spam [val: float] { $val }; spam -2", "-2") +} diff --git a/src/tests/test_engine.rs b/tests/repl/test_engine.rs similarity index 99% rename from src/tests/test_engine.rs rename to tests/repl/test_engine.rs index 9f5be84763..c1eb919afa 100644 --- a/src/tests/test_engine.rs +++ b/tests/repl/test_engine.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; use rstest::rstest; #[test] diff --git a/src/tests/test_env.rs b/tests/repl/test_env.rs similarity index 91% rename from src/tests/test_env.rs rename to tests/repl/test_env.rs index 159b3d0a20..7963f7580b 100644 --- a/src/tests/test_env.rs +++ b/tests/repl/test_env.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; use nu_test_support::nu; #[test] diff --git a/src/tests/test_help.rs b/tests/repl/test_help.rs similarity index 94% rename from src/tests/test_help.rs rename to tests/repl/test_help.rs index b529f7cd19..16d168587a 100644 --- a/src/tests/test_help.rs +++ b/tests/repl/test_help.rs @@ -1,4 +1,4 @@ -use crate::tests::{run_test, TestResult}; +use crate::repl::tests::{run_test, TestResult}; use rstest::rstest; #[rstest] diff --git a/src/tests/test_hiding.rs b/tests/repl/test_hiding.rs similarity index 99% rename from src/tests/test_hiding.rs rename to tests/repl/test_hiding.rs index 81484cc51e..ee2016c54d 100644 --- a/src/tests/test_hiding.rs +++ b/tests/repl/test_hiding.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; // TODO: Test the use/hide tests also as separate lines in REPL (i.e., with merging the delta in between) #[test] diff --git a/src/tests/test_ide.rs b/tests/repl/test_ide.rs similarity index 76% rename from src/tests/test_ide.rs rename to tests/repl/test_ide.rs index b5e40cfe8d..45cc87aa4d 100644 --- a/src/tests/test_ide.rs +++ b/tests/repl/test_ide.rs @@ -1,4 +1,4 @@ -use crate::tests::{test_ide_contains, TestResult}; +use crate::repl::tests::{test_ide_contains, TestResult}; #[test] fn parser_recovers() -> TestResult { diff --git a/src/tests/test_iteration.rs b/tests/repl/test_iteration.rs similarity index 94% rename from src/tests/test_iteration.rs rename to tests/repl/test_iteration.rs index da3eef9ce2..bcc21b9ef9 100644 --- a/src/tests/test_iteration.rs +++ b/tests/repl/test_iteration.rs @@ -1,4 +1,4 @@ -use crate::tests::{run_test, TestResult}; +use crate::repl::tests::{run_test, TestResult}; #[test] fn better_block_types() -> TestResult { diff --git a/src/tests/test_known_external.rs b/tests/repl/test_known_external.rs similarity index 97% rename from src/tests/test_known_external.rs rename to tests/repl/test_known_external.rs index 586ee40cd3..c140f4ee90 100644 --- a/src/tests/test_known_external.rs +++ b/tests/repl/test_known_external.rs @@ -1,7 +1,6 @@ +use crate::repl::tests::{fail_test, run_test, run_test_contains, TestResult}; use std::process::Command; -use crate::tests::{fail_test, run_test, run_test_contains, TestResult}; - // cargo version prints a string of the form: // cargo 1.60.0 (d1fd9fe2c 2022-03-01) diff --git a/src/tests/test_math.rs b/tests/repl/test_math.rs similarity index 98% rename from src/tests/test_math.rs rename to tests/repl/test_math.rs index ed372b198a..af44f8a857 100644 --- a/src/tests/test_math.rs +++ b/tests/repl/test_math.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; #[test] fn add_simple() -> TestResult { diff --git a/src/tests/test_modules.rs b/tests/repl/test_modules.rs similarity index 98% rename from src/tests/test_modules.rs rename to tests/repl/test_modules.rs index 26ca62bf49..dd62c74b77 100644 --- a/src/tests/test_modules.rs +++ b/tests/repl/test_modules.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; #[test] fn module_def_imports_1() -> TestResult { diff --git a/src/tests/test_parser.rs b/tests/repl/test_parser.rs similarity index 99% rename from src/tests/test_parser.rs rename to tests/repl/test_parser.rs index 3d19f05a9c..c0e7279b7b 100644 --- a/src/tests/test_parser.rs +++ b/tests/repl/test_parser.rs @@ -1,9 +1,7 @@ -use crate::tests::{fail_test, run_test, run_test_with_env, TestResult}; +use crate::repl::tests::{fail_test, run_test, run_test_contains, run_test_with_env, TestResult}; use nu_test_support::{nu, nu_repl_code}; use std::collections::HashMap; -use super::run_test_contains; - #[test] fn env_shorthand() -> TestResult { run_test("FOO=BAR if false { 3 } else { 4 }", "4") diff --git a/src/tests/test_ranges.rs b/tests/repl/test_ranges.rs similarity index 92% rename from src/tests/test_ranges.rs rename to tests/repl/test_ranges.rs index 3a007196e6..96f59eea84 100644 --- a/src/tests/test_ranges.rs +++ b/tests/repl/test_ranges.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; #[test] fn int_in_inc_range() -> TestResult { diff --git a/src/tests/test_regex.rs b/tests/repl/test_regex.rs similarity index 96% rename from src/tests/test_regex.rs rename to tests/repl/test_regex.rs index adb59cc4a7..6f9063c81a 100644 --- a/src/tests/test_regex.rs +++ b/tests/repl/test_regex.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; #[test] fn contains() -> TestResult { diff --git a/src/tests/test_signatures.rs b/tests/repl/test_signatures.rs similarity index 99% rename from src/tests/test_signatures.rs rename to tests/repl/test_signatures.rs index 1ca395c90f..5e265974d5 100644 --- a/src/tests/test_signatures.rs +++ b/tests/repl/test_signatures.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; #[test] fn list_annotations() -> TestResult { diff --git a/src/tests/test_spread.rs b/tests/repl/test_spread.rs similarity index 98% rename from src/tests/test_spread.rs rename to tests/repl/test_spread.rs index 3dceea7089..419188be33 100644 --- a/src/tests/test_spread.rs +++ b/tests/repl/test_spread.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; use nu_test_support::nu; #[test] diff --git a/src/tests/test_stdlib.rs b/tests/repl/test_stdlib.rs similarity index 86% rename from src/tests/test_stdlib.rs rename to tests/repl/test_stdlib.rs index 3be252ef77..401fd3c0aa 100644 --- a/src/tests/test_stdlib.rs +++ b/tests/repl/test_stdlib.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test_std, TestResult}; +use crate::repl::tests::{fail_test, run_test_std, TestResult}; #[test] fn library_loaded() -> TestResult { diff --git a/src/tests/test_strings.rs b/tests/repl/test_strings.rs similarity index 91% rename from src/tests/test_strings.rs rename to tests/repl/test_strings.rs index 7d2ae1de84..fa6c419e23 100644 --- a/src/tests/test_strings.rs +++ b/tests/repl/test_strings.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; #[test] fn cjk_in_substrings() -> TestResult { @@ -154,6 +154,18 @@ fn raw_string_inside_closure() -> TestResult { } #[test] -fn incomplete_raw_string() -> TestResult { - fail_test("r#abc", "expected '") +fn incomplete_string() -> TestResult { + fail_test("r#abc", "expected '")?; + fail_test("r#'bc", "expected closing '#")?; + fail_test("'ab\"", "expected closing '")?; + fail_test("\"ab'", "expected closing \"")?; + fail_test( + r#"def func [] { + { + "A": ""B" # <- the quote is bad + } +} +"#, + "expected closing \"", + ) } diff --git a/src/tests/test_table_operations.rs b/tests/repl/test_table_operations.rs similarity index 99% rename from src/tests/test_table_operations.rs rename to tests/repl/test_table_operations.rs index 9971261190..5d61e0d902 100644 --- a/src/tests/test_table_operations.rs +++ b/tests/repl/test_table_operations.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; #[test] fn illegal_column_duplication() -> TestResult { diff --git a/src/tests/test_type_check.rs b/tests/repl/test_type_check.rs similarity index 98% rename from src/tests/test_type_check.rs rename to tests/repl/test_type_check.rs index ed912f8fff..87216e6672 100644 --- a/src/tests/test_type_check.rs +++ b/tests/repl/test_type_check.rs @@ -1,4 +1,4 @@ -use crate::tests::{fail_test, run_test, TestResult}; +use crate::repl::tests::{fail_test, run_test, TestResult}; #[test] fn chained_operator_typecheck() -> TestResult { diff --git a/src/tests.rs b/tests/repl/tests.rs similarity index 89% rename from src/tests.rs rename to tests/repl/tests.rs index 487173b9b1..e3a5e89471 100644 --- a/src/tests.rs +++ b/tests/repl/tests.rs @@ -1,30 +1,3 @@ -mod test_bits; -mod test_cell_path; -mod test_commandline; -mod test_conditionals; -mod test_config; -mod test_config_path; -mod test_converters; -mod test_custom_commands; -mod test_engine; -mod test_env; -mod test_help; -mod test_hiding; -mod test_ide; -mod test_iteration; -mod test_known_external; -mod test_math; -mod test_modules; -mod test_parser; -mod test_ranges; -mod test_regex; -mod test_signatures; -mod test_spread; -mod test_stdlib; -mod test_strings; -mod test_table_operations; -mod test_type_check; - use assert_cmd::prelude::*; use pretty_assertions::assert_eq; use std::collections::HashMap; diff --git a/tests/shell/pipeline/commands/external.rs b/tests/shell/pipeline/commands/external.rs index 88d443e4fd..3cbed5a124 100644 --- a/tests/shell/pipeline/commands/external.rs +++ b/tests/shell/pipeline/commands/external.rs @@ -314,6 +314,7 @@ mod external_words { use super::nu; use nu_test_support::fs::Stub::FileWithContent; use nu_test_support::{pipeline, playground::Playground}; + #[test] fn relaxed_external_words() { let actual = nu!(" @@ -323,6 +324,12 @@ mod external_words { assert_eq!(actual.out, "joturner@foo.bar.baz"); } + #[test] + fn raw_string_as_external_argument() { + let actual = nu!("nu --testbin cococo r#'asdf'#"); + assert_eq!(actual.out, "asdf"); + } + //FIXME: jt: limitation in testing - can't use single ticks currently #[ignore] #[test] diff --git a/tests/shell/pipeline/commands/internal.rs b/tests/shell/pipeline/commands/internal.rs index 7bc75b07ad..6c2226ff65 100644 --- a/tests/shell/pipeline/commands/internal.rs +++ b/tests/shell/pipeline/commands/internal.rs @@ -1131,13 +1131,13 @@ fn pipe_input_to_print() { #[test] fn err_pipe_input_to_print() { let actual = nu!(r#""foo" e>| print"#); - assert!(actual.err.contains("only works on external streams")); + assert!(actual.err.contains("only works on external commands")); } #[test] fn outerr_pipe_input_to_print() { let actual = nu!(r#""foo" o+e>| print"#); - assert!(actual.err.contains("only works on external streams")); + assert!(actual.err.contains("only works on external commands")); } #[test]