From 3522bead97f2b9995d12ad986d0ce28aee6585df Mon Sep 17 00:00:00 2001 From: JT <547158+jntrnr@users.noreply.github.com> Date: Fri, 24 Dec 2021 18:22:11 +1100 Subject: [PATCH] Add string stream and binary stream, add text decoding (#570) * WIP * Add binary/string streams and text decoding * Make string collection fallible * Oops, forgot pretty hex * Oops, forgot pretty hex * clippy --- Cargo.lock | 52 ++- Cargo.toml | 1 + crates/nu-command/Cargo.toml | 3 +- crates/nu-command/src/core_commands/echo.rs | 2 +- crates/nu-command/src/default_context.rs | 1 + crates/nu-command/src/filters/columns.rs | 4 +- crates/nu-command/src/filters/drop/column.rs | 3 +- crates/nu-command/src/filters/each.rs | 75 ++++- crates/nu-command/src/filters/lines.rs | 50 ++- crates/nu-command/src/filters/par_each.rs | 87 ++++- crates/nu-command/src/filters/reject.rs | 3 +- crates/nu-command/src/filters/select.rs | 3 +- crates/nu-command/src/filters/wrap.rs | 12 +- .../nu-command/src/formats/from/delimited.rs | 2 +- crates/nu-command/src/formats/from/eml.rs | 2 +- crates/nu-command/src/formats/from/ics.rs | 2 +- crates/nu-command/src/formats/from/ini.rs | 2 +- crates/nu-command/src/formats/from/json.rs | 2 +- crates/nu-command/src/formats/from/ssv.rs | 2 +- crates/nu-command/src/formats/from/toml.rs | 2 +- crates/nu-command/src/formats/from/url.rs | 2 +- crates/nu-command/src/formats/from/vcf.rs | 2 +- crates/nu-command/src/formats/from/xml.rs | 2 +- crates/nu-command/src/formats/from/yaml.rs | 2 +- crates/nu-command/src/formats/to/html.rs | 2 +- crates/nu-command/src/math/utils.rs | 6 +- crates/nu-command/src/path/join.rs | 6 +- crates/nu-command/src/random/dice.rs | 2 +- crates/nu-command/src/strings/decode.rs | 107 +++++++ .../nu-command/src/strings/format/command.rs | 2 +- crates/nu-command/src/strings/mod.rs | 2 + crates/nu-command/src/strings/parse.rs | 2 +- crates/nu-command/src/system/run_external.rs | 55 ++-- crates/nu-command/src/viewers/griddle.rs | 2 +- crates/nu-command/src/viewers/table.rs | 50 ++- crates/nu-engine/src/eval.rs | 2 +- crates/nu-plugin/src/plugin/declaration.rs | 18 +- crates/nu-pretty-hex/Cargo.lock | 201 ++++++++++++ crates/nu-pretty-hex/Cargo.toml | 27 ++ crates/nu-pretty-hex/LICENSE | 21 ++ crates/nu-pretty-hex/README.md | 81 +++++ crates/nu-pretty-hex/src/lib.rs | 66 ++++ crates/nu-pretty-hex/src/main.rs | 50 +++ crates/nu-pretty-hex/src/pretty_hex.rs | 299 ++++++++++++++++++ crates/nu-pretty-hex/tests/256.txt | 17 + crates/nu-pretty-hex/tests/data | 1 + crates/nu-pretty-hex/tests/tests.rs | 175 ++++++++++ crates/nu-protocol/src/pipeline_data.rs | 123 +++++-- crates/nu-protocol/src/value/stream.rs | 88 ++++++ src/main.rs | 29 +- 50 files changed, 1633 insertions(+), 119 deletions(-) create mode 100644 crates/nu-command/src/strings/decode.rs create mode 100644 crates/nu-pretty-hex/Cargo.lock create mode 100644 crates/nu-pretty-hex/Cargo.toml create mode 100644 crates/nu-pretty-hex/LICENSE create mode 100644 crates/nu-pretty-hex/README.md create mode 100644 crates/nu-pretty-hex/src/lib.rs create mode 100644 crates/nu-pretty-hex/src/main.rs create mode 100644 crates/nu-pretty-hex/src/pretty_hex.rs create mode 100644 crates/nu-pretty-hex/tests/256.txt create mode 100644 crates/nu-pretty-hex/tests/data create mode 100644 crates/nu-pretty-hex/tests/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 3e3b69b972..1158ad75dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,7 +78,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcb2392079bf27198570d6af79ecbd9ec7d8f16d3ec6b60933922fdb66287127" dependencies = [ - "heapless", + "heapless 0.5.6", "nom 4.2.3", ] @@ -851,6 +851,7 @@ dependencies = [ "nu-parser", "nu-path", "nu-plugin", + "nu-pretty-hex", "nu-protocol", "nu-table", "nu-term-grid", @@ -1102,6 +1103,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + [[package]] name = "hash_hasher" version = "2.0.3" @@ -1126,7 +1136,18 @@ checksum = "74911a68a1658cfcfb61bc0ccfbd536e3b6e906f8c2f7883ee50157e3e2184f1" dependencies = [ "as-slice", "generic-array 0.13.3", - "hash32", + "hash32 0.1.1", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e476c64197665c3725621f0ac3f9e5209aa5e889e02a08b1daf5f16dc5fd952" +dependencies = [ + "hash32 0.2.1", + "spin", "stable_deref_trait", ] @@ -1690,6 +1711,7 @@ dependencies = [ "digest 0.10.0", "dtparse", "eml-parser", + "encoding_rs", "glob", "htmlescape", "ical", @@ -1705,12 +1727,12 @@ dependencies = [ "nu-json", "nu-parser", "nu-path", + "nu-pretty-hex", "nu-protocol", "nu-table", "nu-term-grid", "num 0.4.0", "polars", - "pretty-hex", "quick-xml 0.22.0", "rand", "rayon", @@ -1792,6 +1814,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "nu-pretty-hex" +version = "0.41.0" +dependencies = [ + "heapless 0.7.9", + "nu-ansi-term", + "rand", +] + [[package]] name = "nu-protocol" version = "0.1.0" @@ -2339,12 +2370,6 @@ dependencies = [ "termtree", ] -[[package]] -name = "pretty-hex" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5c99d529f0d30937f6f4b8a86d988047327bb88d04d2c4afc356de74722131" - [[package]] name = "pretty_assertions" version = "1.0.0" @@ -2885,6 +2910,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" +[[package]] +name = "spin" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" +dependencies = [ + "lock_api", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" diff --git a/Cargo.toml b/Cargo.toml index d3a2e46515..ac0574b143 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ nu-engine = { path="./crates/nu-engine" } nu-json = { path="./crates/nu-json" } nu-parser = { path="./crates/nu-parser" } nu-path = { path="./crates/nu-path" } +nu-pretty-hex = { path = "./crates/nu-pretty-hex" } nu-protocol = { path = "./crates/nu-protocol" } nu-plugin = { path = "./crates/nu-plugin", optional = true } nu-table = { path = "./crates/nu-table" } diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 353ea04e51..4619db4339 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -11,6 +11,7 @@ build = "build.rs" nu-engine = { path = "../nu-engine" } nu-json = { path = "../nu-json" } nu-path = { path = "../nu-path" } +nu-pretty-hex = { path = "../nu-pretty-hex" } nu-protocol = { path = "../nu-protocol" } nu-table = { path = "../nu-table" } nu-term-grid = { path = "../nu-term-grid" } @@ -55,7 +56,6 @@ trash = { version = "2.0.2", optional = true } unicode-segmentation = "1.8.0" uuid = { version = "0.8.2", features = ["v4"] } htmlescape = "0.3.1" -pretty-hex = "0.2.1" zip = { version="0.5.9", optional=true } lazy_static = "1.4.0" strip-ansi-escapes = "0.1.1" @@ -66,6 +66,7 @@ digest = "0.10.0" md5 = { package = "md-5", version = "0.10.0" } sha2 = "0.10.0" base64 = "0.13.0" +encoding_rs = "0.8.30" num = { version = "0.4.0", optional = true } [target.'cfg(unix)'.dependencies] diff --git a/crates/nu-command/src/core_commands/echo.rs b/crates/nu-command/src/core_commands/echo.rs index 0f7c05e343..05396299b1 100644 --- a/crates/nu-command/src/core_commands/echo.rs +++ b/crates/nu-command/src/core_commands/echo.rs @@ -34,7 +34,7 @@ impl Command for Echo { let n = to_be_echoed.len(); match n.cmp(&1usize) { // More than one value is converted in a stream of values - std::cmp::Ordering::Greater => PipelineData::Stream( + std::cmp::Ordering::Greater => PipelineData::ListStream( ValueStream::from_stream(to_be_echoed.into_iter(), engine_state.ctrlc.clone()), None, ), diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index dcf090eedf..c97759eb91 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -111,6 +111,7 @@ pub fn create_default_context() -> EngineState { bind_command! { BuildString, Char, + Decode, Format, Parse, Size, diff --git a/crates/nu-command/src/filters/columns.rs b/crates/nu-command/src/filters/columns.rs index d4b4a30600..826bb9281e 100644 --- a/crates/nu-command/src/filters/columns.rs +++ b/crates/nu-command/src/filters/columns.rs @@ -72,7 +72,7 @@ fn getcol( .map(move |x| Value::String { val: x, span }) .into_pipeline_data(engine_state.ctrlc.clone())) } - PipelineData::Stream(stream, ..) => { + PipelineData::ListStream(stream, ..) => { let v: Vec<_> = stream.into_iter().collect(); let input_cols = get_input_cols(v); @@ -81,7 +81,7 @@ fn getcol( .map(move |x| Value::String { val: x, span }) .into_pipeline_data(engine_state.ctrlc.clone())) } - PipelineData::Value(_v, ..) => { + PipelineData::Value(..) | PipelineData::StringStream(..) | PipelineData::ByteStream(..) => { let cols = vec![]; let vals = vec![]; Ok(Value::Record { cols, vals, span }.into_pipeline_data()) diff --git a/crates/nu-command/src/filters/drop/column.rs b/crates/nu-command/src/filters/drop/column.rs index e5438314c9..3b4564a2a4 100644 --- a/crates/nu-command/src/filters/drop/column.rs +++ b/crates/nu-command/src/filters/drop/column.rs @@ -86,7 +86,7 @@ fn dropcol( .into_iter() .into_pipeline_data(engine_state.ctrlc.clone())) } - PipelineData::Stream(stream, ..) => { + PipelineData::ListStream(stream, ..) => { let mut output = vec![]; let v: Vec<_> = stream.into_iter().collect(); @@ -123,6 +123,7 @@ fn dropcol( Ok(Value::Record { cols, vals, span }.into_pipeline_data()) } + x => Ok(x), } } diff --git a/crates/nu-command/src/filters/each.rs b/crates/nu-command/src/filters/each.rs index 614d4d1512..436786b517 100644 --- a/crates/nu-command/src/filters/each.rs +++ b/crates/nu-command/src/filters/each.rs @@ -76,7 +76,7 @@ impl Command for Each { match input { PipelineData::Value(Value::Range { .. }, ..) | PipelineData::Value(Value::List { .. }, ..) - | PipelineData::Stream { .. } => Ok(input + | PipelineData::ListStream { .. } => Ok(input .into_iter() .enumerate() .map(move |(idx, x)| { @@ -109,6 +109,79 @@ impl Command for Each { } }) .into_pipeline_data(ctrlc)), + PipelineData::ByteStream(stream, ..) => Ok(stream + .into_iter() + .enumerate() + .map(move |(idx, x)| { + let x = Value::Binary { val: x, span }; + + if let Some(var) = block.signature.get_positional(0) { + if let Some(var_id) = &var.var_id { + if numbered { + stack.add_var( + *var_id, + Value::Record { + cols: vec!["index".into(), "item".into()], + vals: vec![ + Value::Int { + val: idx as i64, + span, + }, + x, + ], + span, + }, + ); + } else { + stack.add_var(*var_id, x); + } + } + } + + match eval_block(&engine_state, &mut stack, &block, PipelineData::new(span)) { + Ok(v) => v.into_value(span), + Err(error) => Value::Error { error }, + } + }) + .into_pipeline_data(ctrlc)), + PipelineData::StringStream(stream, ..) => Ok(stream + .into_iter() + .enumerate() + .map(move |(idx, x)| { + let x = match x { + Ok(x) => Value::String { val: x, span }, + Err(err) => return Value::Error { error: err }, + }; + + if let Some(var) = block.signature.get_positional(0) { + if let Some(var_id) = &var.var_id { + if numbered { + stack.add_var( + *var_id, + Value::Record { + cols: vec!["index".into(), "item".into()], + vals: vec![ + Value::Int { + val: idx as i64, + span, + }, + x, + ], + span, + }, + ); + } else { + stack.add_var(*var_id, x); + } + } + } + + match eval_block(&engine_state, &mut stack, &block, PipelineData::new(span)) { + Ok(v) => v.into_value(span), + Err(error) => Value::Error { error }, + } + }) + .into_pipeline_data(ctrlc)), PipelineData::Value(Value::Record { cols, vals, .. }, ..) => { let mut output_cols = vec![]; let mut output_vals = vec![]; diff --git a/crates/nu-command/src/filters/lines.rs b/crates/nu-command/src/filters/lines.rs index aaa985be8c..acf59be63e 100644 --- a/crates/nu-command/src/filters/lines.rs +++ b/crates/nu-command/src/filters/lines.rs @@ -27,10 +27,11 @@ impl Command for Lines { fn run( &self, engine_state: &EngineState, - _stack: &mut Stack, + stack: &mut Stack, call: &Call, input: PipelineData, ) -> Result { + let head = call.head; let skip_empty = call.has_flag("skip-emtpy"); match input { #[allow(clippy::needless_collect)] @@ -53,7 +54,7 @@ impl Command for Lines { Ok(iter.into_pipeline_data(engine_state.ctrlc.clone())) } - PipelineData::Stream(stream, ..) => { + PipelineData::ListStream(stream, ..) => { let iter = stream .into_iter() .filter_map(move |value| { @@ -81,10 +82,55 @@ impl Command for Lines { Ok(iter.into_pipeline_data(engine_state.ctrlc.clone())) } + PipelineData::StringStream(stream, span, ..) => { + let iter = stream + .into_iter() + .map(move |value| match value { + Ok(value) => value + .split(SPLIT_CHAR) + .filter_map(|s| { + if !s.is_empty() { + Some(Value::String { + val: s.into(), + span, + }) + } else { + None + } + }) + .collect::>(), + Err(err) => vec![Value::Error { error: err }], + }) + .flatten(); + + Ok(iter.into_pipeline_data(engine_state.ctrlc.clone())) + } PipelineData::Value(val, ..) => Err(ShellError::UnsupportedInput( format!("Not supported input: {}", val.as_string()?), call.head, )), + PipelineData::ByteStream(..) => { + let config = stack.get_config()?; + + //FIXME: Make sure this can fail in the future to let the user + //know to use a different encoding + let s = input.collect_string("", &config)?; + + let lines = s + .split(SPLIT_CHAR) + .map(|s| s.to_string()) + .collect::>(); + + let iter = lines.into_iter().filter_map(move |s| { + if skip_empty && s.is_empty() { + None + } else { + Some(Value::string(s, head)) + } + }); + + Ok(iter.into_pipeline_data(engine_state.ctrlc.clone())) + } } } } diff --git a/crates/nu-command/src/filters/par_each.rs b/crates/nu-command/src/filters/par_each.rs index 45c62ecea9..13016e4403 100644 --- a/crates/nu-command/src/filters/par_each.rs +++ b/crates/nu-command/src/filters/par_each.rs @@ -139,7 +139,7 @@ impl Command for ParEach { .into_iter() .flatten() .into_pipeline_data(ctrlc)), - PipelineData::Stream(stream, ..) => Ok(stream + PipelineData::ListStream(stream, ..) => Ok(stream .enumerate() .par_bridge() .map(move |(idx, x)| { @@ -179,6 +179,91 @@ impl Command for ParEach { .into_iter() .flatten() .into_pipeline_data(ctrlc)), + PipelineData::StringStream(stream, ..) => Ok(stream + .enumerate() + .par_bridge() + .map(move |(idx, x)| { + let x = match x { + Ok(x) => Value::String { val: x, span }, + Err(err) => return Value::Error { error: err }.into_pipeline_data(), + }; + let block = engine_state.get_block(block_id); + + let mut stack = stack.clone(); + + if let Some(var) = block.signature.get_positional(0) { + if let Some(var_id) = &var.var_id { + if numbered { + stack.add_var( + *var_id, + Value::Record { + cols: vec!["index".into(), "item".into()], + vals: vec![ + Value::Int { + val: idx as i64, + span, + }, + x, + ], + span, + }, + ); + } else { + stack.add_var(*var_id, x); + } + } + } + + match eval_block(&engine_state, &mut stack, block, PipelineData::new(span)) { + Ok(v) => v, + Err(error) => Value::Error { error }.into_pipeline_data(), + } + }) + .collect::>() + .into_iter() + .flatten() + .into_pipeline_data(ctrlc)), + PipelineData::ByteStream(stream, ..) => Ok(stream + .enumerate() + .par_bridge() + .map(move |(idx, x)| { + let x = Value::Binary { val: x, span }; + let block = engine_state.get_block(block_id); + + let mut stack = stack.clone(); + + if let Some(var) = block.signature.get_positional(0) { + if let Some(var_id) = &var.var_id { + if numbered { + stack.add_var( + *var_id, + Value::Record { + cols: vec!["index".into(), "item".into()], + vals: vec![ + Value::Int { + val: idx as i64, + span, + }, + x, + ], + span, + }, + ); + } else { + stack.add_var(*var_id, x); + } + } + } + + match eval_block(&engine_state, &mut stack, block, PipelineData::new(span)) { + Ok(v) => v, + Err(error) => Value::Error { error }.into_pipeline_data(), + } + }) + .collect::>() + .into_iter() + .flatten() + .into_pipeline_data(ctrlc)), PipelineData::Value(Value::Record { cols, vals, .. }, ..) => { let mut output_cols = vec![]; let mut output_vals = vec![]; diff --git a/crates/nu-command/src/filters/reject.rs b/crates/nu-command/src/filters/reject.rs index f950812912..81fcc3fabd 100644 --- a/crates/nu-command/src/filters/reject.rs +++ b/crates/nu-command/src/filters/reject.rs @@ -82,7 +82,7 @@ fn reject( .into_iter() .into_pipeline_data(engine_state.ctrlc.clone())) } - PipelineData::Stream(stream, ..) => { + PipelineData::ListStream(stream, ..) => { let mut output = vec![]; let v: Vec<_> = stream.into_iter().collect(); @@ -119,6 +119,7 @@ fn reject( Ok(Value::Record { cols, vals, span }.into_pipeline_data()) } + x => Ok(x), } } diff --git a/crates/nu-command/src/filters/select.rs b/crates/nu-command/src/filters/select.rs index 53367e7b4c..f1fa4e7780 100644 --- a/crates/nu-command/src/filters/select.rs +++ b/crates/nu-command/src/filters/select.rs @@ -95,7 +95,7 @@ fn select( .into_iter() .into_pipeline_data(engine_state.ctrlc.clone())) } - PipelineData::Stream(stream, ..) => Ok(stream + PipelineData::ListStream(stream, ..) => Ok(stream .map(move |x| { let mut cols = vec![]; let mut vals = vec![]; @@ -130,6 +130,7 @@ fn select( Ok(Value::Record { cols, vals, span }.into_pipeline_data()) } + _ => Ok(PipelineData::new(span)), } } diff --git a/crates/nu-command/src/filters/wrap.rs b/crates/nu-command/src/filters/wrap.rs index 46f6fe5b3e..4309882a30 100644 --- a/crates/nu-command/src/filters/wrap.rs +++ b/crates/nu-command/src/filters/wrap.rs @@ -43,13 +43,23 @@ impl Command for Wrap { span, }) .into_pipeline_data(engine_state.ctrlc.clone())), - PipelineData::Stream(stream, ..) => Ok(stream + PipelineData::ListStream(stream, ..) => Ok(stream .map(move |x| Value::Record { cols: vec![name.clone()], vals: vec![x], span, }) .into_pipeline_data(engine_state.ctrlc.clone())), + PipelineData::StringStream(stream, ..) => Ok(Value::String { + val: stream.into_string("")?, + span, + } + .into_pipeline_data()), + PipelineData::ByteStream(stream, ..) => Ok(Value::Binary { + val: stream.into_vec(), + span, + } + .into_pipeline_data()), PipelineData::Value(input, ..) => Ok(Value::Record { cols: vec![name], vals: vec![input], diff --git a/crates/nu-command/src/formats/from/delimited.rs b/crates/nu-command/src/formats/from/delimited.rs index eafdd4e53d..b8c0191548 100644 --- a/crates/nu-command/src/formats/from/delimited.rs +++ b/crates/nu-command/src/formats/from/delimited.rs @@ -52,7 +52,7 @@ pub fn from_delimited_data( name: Span, config: &Config, ) -> Result { - let concat_string = input.collect_string("", config); + let concat_string = input.collect_string("", config)?; Ok( from_delimited_string_to_value(concat_string, noheaders, sep, name) diff --git a/crates/nu-command/src/formats/from/eml.rs b/crates/nu-command/src/formats/from/eml.rs index fb6af3c63c..36a1a94a03 100644 --- a/crates/nu-command/src/formats/from/eml.rs +++ b/crates/nu-command/src/formats/from/eml.rs @@ -183,7 +183,7 @@ fn from_eml( head: Span, config: &Config, ) -> Result { - let value = input.collect_string("", config); + let value = input.collect_string("", config)?; let body_preview = preview_body .map(|b| b.item as usize) diff --git a/crates/nu-command/src/formats/from/ics.rs b/crates/nu-command/src/formats/from/ics.rs index b49a6e0f8c..aacb77e9bd 100644 --- a/crates/nu-command/src/formats/from/ics.rs +++ b/crates/nu-command/src/formats/from/ics.rs @@ -93,7 +93,7 @@ END:VCALENDAR' | from ics", } fn from_ics(input: PipelineData, head: Span, config: &Config) -> Result { - let input_string = input.collect_string("", config); + let input_string = input.collect_string("", config)?; let input_bytes = input_string.as_bytes(); let buf_reader = BufReader::new(input_bytes); let parser = ical::IcalParser::new(buf_reader); diff --git a/crates/nu-command/src/formats/from/ini.rs b/crates/nu-command/src/formats/from/ini.rs index 6273c2a000..f7b83ec5cd 100644 --- a/crates/nu-command/src/formats/from/ini.rs +++ b/crates/nu-command/src/formats/from/ini.rs @@ -88,7 +88,7 @@ pub fn from_ini_string_to_value(s: String, span: Span) -> Result Result { - let concat_string = input.collect_string("", config); + let concat_string = input.collect_string("", config)?; match from_ini_string_to_value(concat_string, head) { Ok(x) => Ok(x.into_pipeline_data()), diff --git a/crates/nu-command/src/formats/from/json.rs b/crates/nu-command/src/formats/from/json.rs index 3b3a9d8dea..4afb0f5b09 100644 --- a/crates/nu-command/src/formats/from/json.rs +++ b/crates/nu-command/src/formats/from/json.rs @@ -76,7 +76,7 @@ impl Command for FromJson { ) -> Result { let span = call.head; let config = stack.get_config().unwrap_or_default(); - let mut string_input = input.collect_string("", &config); + let mut string_input = input.collect_string("", &config)?; string_input.push('\n'); // TODO: turn this into a structured underline of the nu_json error diff --git a/crates/nu-command/src/formats/from/ssv.rs b/crates/nu-command/src/formats/from/ssv.rs index c378b6ba1d..d2cadf1b66 100644 --- a/crates/nu-command/src/formats/from/ssv.rs +++ b/crates/nu-command/src/formats/from/ssv.rs @@ -275,7 +275,7 @@ fn from_ssv( let minimum_spaces: Option> = call.get_flag(engine_state, stack, "minimum-spaces")?; - let concat_string = input.collect_string("", &config); + let concat_string = input.collect_string("", &config)?; let split_at = match minimum_spaces { Some(number) => number.item, None => DEFAULT_MINIMUM_SPACES, diff --git a/crates/nu-command/src/formats/from/toml.rs b/crates/nu-command/src/formats/from/toml.rs index 378e92216d..477b2c3ec1 100644 --- a/crates/nu-command/src/formats/from/toml.rs +++ b/crates/nu-command/src/formats/from/toml.rs @@ -74,7 +74,7 @@ b = [1, 2]' | from toml", ) -> Result { let span = call.head; let config = stack.get_config().unwrap_or_default(); - let mut string_input = input.collect_string("", &config); + let mut string_input = input.collect_string("", &config)?; string_input.push('\n'); Ok(convert_string_to_value(string_input, span)?.into_pipeline_data()) } diff --git a/crates/nu-command/src/formats/from/url.rs b/crates/nu-command/src/formats/from/url.rs index dab7da89c2..1ee91d4296 100644 --- a/crates/nu-command/src/formats/from/url.rs +++ b/crates/nu-command/src/formats/from/url.rs @@ -54,7 +54,7 @@ impl Command for FromUrl { } fn from_url(input: PipelineData, head: Span, config: &Config) -> Result { - let concat_string = input.collect_string("", config); + let concat_string = input.collect_string("", config)?; let result = serde_urlencoded::from_str::>(&concat_string); diff --git a/crates/nu-command/src/formats/from/vcf.rs b/crates/nu-command/src/formats/from/vcf.rs index 5c7566609b..fbd1d648d6 100644 --- a/crates/nu-command/src/formats/from/vcf.rs +++ b/crates/nu-command/src/formats/from/vcf.rs @@ -124,7 +124,7 @@ END:VCARD' | from vcf", } fn from_vcf(input: PipelineData, head: Span, config: &Config) -> Result { - let input_string = input.collect_string("", config); + let input_string = input.collect_string("", config)?; let input_bytes = input_string.as_bytes(); let cursor = std::io::Cursor::new(input_bytes); let parser = ical::VcardParser::new(cursor); diff --git a/crates/nu-command/src/formats/from/xml.rs b/crates/nu-command/src/formats/from/xml.rs index 8d87dc8a31..bf6756cc82 100644 --- a/crates/nu-command/src/formats/from/xml.rs +++ b/crates/nu-command/src/formats/from/xml.rs @@ -179,7 +179,7 @@ pub fn from_xml_string_to_value(s: String, span: Span) -> Result Result { - let concat_string = input.collect_string("", config); + let concat_string = input.collect_string("", config)?; match from_xml_string_to_value(concat_string, head) { Ok(x) => Ok(x.into_pipeline_data()), diff --git a/crates/nu-command/src/formats/from/yaml.rs b/crates/nu-command/src/formats/from/yaml.rs index d8e8fcfdab..64a6d8deda 100644 --- a/crates/nu-command/src/formats/from/yaml.rs +++ b/crates/nu-command/src/formats/from/yaml.rs @@ -206,7 +206,7 @@ pub fn from_yaml_string_to_value(s: String, span: Span) -> Result Result { - let concat_string = input.collect_string("", config); + let concat_string = input.collect_string("", config)?; match from_yaml_string_to_value(concat_string, head) { Ok(x) => Ok(x.into_pipeline_data()), diff --git a/crates/nu-command/src/formats/to/html.rs b/crates/nu-command/src/formats/to/html.rs index 1a69bb37fc..60fc130e2d 100644 --- a/crates/nu-command/src/formats/to/html.rs +++ b/crates/nu-command/src/formats/to/html.rs @@ -444,7 +444,7 @@ fn html_value(value: Value, config: &Config) -> String { let mut output_string = String::new(); match value { Value::Binary { val, .. } => { - let output = pretty_hex::pretty_hex(&val); + let output = nu_pretty_hex::pretty_hex(&val); output_string.push_str("
");
             output_string.push_str(&output);
             output_string.push_str("
"); diff --git a/crates/nu-command/src/math/utils.rs b/crates/nu-command/src/math/utils.rs index 68418c0828..27dcbcf82d 100644 --- a/crates/nu-command/src/math/utils.rs +++ b/crates/nu-command/src/math/utils.rs @@ -62,7 +62,7 @@ pub fn calculate( mf: impl Fn(&[Value], &Span) -> Result, ) -> Result { match values { - PipelineData::Stream(s, ..) => helper_for_tables(&s.collect::>(), name, mf), + PipelineData::ListStream(s, ..) => helper_for_tables(&s.collect::>(), name, mf), PipelineData::Value(Value::List { ref vals, .. }, ..) => match &vals[..] { [Value::Record { .. }, _end @ ..] => helper_for_tables(vals, name, mf), _ => mf(vals, &name), @@ -88,5 +88,9 @@ pub fn calculate( mf(&new_vals?, &name) } PipelineData::Value(val, ..) => mf(&[val], &name), + _ => Err(ShellError::UnsupportedInput( + "Input data is not supported by this command.".to_string(), + name, + )), } } diff --git a/crates/nu-command/src/path/join.rs b/crates/nu-command/src/path/join.rs index 4f4dbca62f..69e67be55a 100644 --- a/crates/nu-command/src/path/join.rs +++ b/crates/nu-command/src/path/join.rs @@ -71,13 +71,17 @@ the output of 'path parse' and 'path split' subcommands."# PipelineData::Value(val, md) => { Ok(PipelineData::Value(handle_value(val, &args, head), md)) } - PipelineData::Stream(stream, md) => Ok(PipelineData::Stream( + PipelineData::ListStream(stream, md) => Ok(PipelineData::ListStream( ValueStream::from_stream( stream.map(move |val| handle_value(val, &args, head)), engine_state.ctrlc.clone(), ), md, )), + _ => Err(ShellError::UnsupportedInput( + "Input data is not supported by this command.".to_string(), + head, + )), } } diff --git a/crates/nu-command/src/random/dice.rs b/crates/nu-command/src/random/dice.rs index de44f2f35c..f18f6edc93 100644 --- a/crates/nu-command/src/random/dice.rs +++ b/crates/nu-command/src/random/dice.rs @@ -79,7 +79,7 @@ fn dice( } }); - Ok(PipelineData::Stream( + Ok(PipelineData::ListStream( ValueStream::from_stream(iter, engine_state.ctrlc.clone()), None, )) diff --git a/crates/nu-command/src/strings/decode.rs b/crates/nu-command/src/strings/decode.rs new file mode 100644 index 0000000000..0616511fc2 --- /dev/null +++ b/crates/nu-command/src/strings/decode.rs @@ -0,0 +1,107 @@ +use encoding_rs::Encoding; +use nu_engine::CallExt; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{ + Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Spanned, SyntaxShape, + Value, +}; + +#[derive(Clone)] +pub struct Decode; + +impl Command for Decode { + fn name(&self) -> &str { + "decode" + } + + fn usage(&self) -> &str { + "Decode bytes as a string." + } + + fn signature(&self) -> nu_protocol::Signature { + Signature::build("decode") + .required("encoding", SyntaxShape::String, "the text encoding to use") + .category(Category::Strings) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Decode the output of an external command", + example: "cat myfile.q | decode utf-8", + result: None, + }] + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let head = call.head; + let encoding: Spanned = call.req(engine_state, stack, 0)?; + + match input { + PipelineData::ByteStream(stream, ..) => { + let bytes: Vec = stream.flatten().collect(); + + let encoding = match Encoding::for_label(encoding.item.as_bytes()) { + None => Err(ShellError::SpannedLabeledError( + format!( + r#"{} is not a valid encoding, refer to https://docs.rs/encoding_rs/0.8.23/encoding_rs/#statics for a valid list of encodings"#, + encoding.item + ), + "invalid encoding".into(), + encoding.span, + )), + Some(encoding) => Ok(encoding), + }?; + + let result = encoding.decode(&bytes); + + Ok(Value::String { + val: result.0.to_string(), + span: head, + } + .into_pipeline_data()) + } + PipelineData::Value(Value::Binary { val: bytes, .. }, ..) => { + let encoding = match Encoding::for_label(encoding.item.as_bytes()) { + None => Err(ShellError::SpannedLabeledError( + format!( + r#"{} is not a valid encoding, refer to https://docs.rs/encoding_rs/0.8.23/encoding_rs/#statics for a valid list of encodings"#, + encoding.item + ), + "invalid encoding".into(), + encoding.span, + )), + Some(encoding) => Ok(encoding), + }?; + + let result = encoding.decode(&bytes); + + Ok(Value::String { + val: result.0.to_string(), + span: head, + } + .into_pipeline_data()) + } + _ => Err(ShellError::UnsupportedInput( + "non-binary input".into(), + head, + )), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_examples() { + crate::test_examples(Decode) + } +} diff --git a/crates/nu-command/src/strings/format/command.rs b/crates/nu-command/src/strings/format/command.rs index db36518086..4f6f351124 100644 --- a/crates/nu-command/src/strings/format/command.rs +++ b/crates/nu-command/src/strings/format/command.rs @@ -151,7 +151,7 @@ fn format( } } - Ok(PipelineData::Stream( + Ok(PipelineData::ListStream( ValueStream::from_stream(list.into_iter(), None), None, )) diff --git a/crates/nu-command/src/strings/mod.rs b/crates/nu-command/src/strings/mod.rs index eaf0cf901f..e30e207a73 100644 --- a/crates/nu-command/src/strings/mod.rs +++ b/crates/nu-command/src/strings/mod.rs @@ -1,5 +1,6 @@ mod build_string; mod char_; +mod decode; mod format; mod parse; mod size; @@ -8,6 +9,7 @@ mod str_; pub use build_string::BuildString; pub use char_::Char; +pub use decode::*; pub use format::*; pub use parse::*; pub use size::Size; diff --git a/crates/nu-command/src/strings/parse.rs b/crates/nu-command/src/strings/parse.rs index 8e4bb2d6ae..2b5051c5f5 100644 --- a/crates/nu-command/src/strings/parse.rs +++ b/crates/nu-command/src/strings/parse.rs @@ -126,7 +126,7 @@ fn operate( } } - Ok(PipelineData::Stream( + Ok(PipelineData::ListStream( ValueStream::from_stream(parsed.into_iter(), ctrlc), None, )) diff --git a/crates/nu-command/src/system/run_external.rs b/crates/nu-command/src/system/run_external.rs index 66efc61e52..7113c767b7 100644 --- a/crates/nu-command/src/system/run_external.rs +++ b/crates/nu-command/src/system/run_external.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::collections::HashMap; use std::env; use std::io::{BufRead, BufReader, Write}; @@ -10,14 +9,14 @@ use std::sync::mpsc; use nu_engine::env_to_strings; use nu_protocol::engine::{EngineState, Stack}; use nu_protocol::{ast::Call, engine::Command, ShellError, Signature, SyntaxShape, Value}; -use nu_protocol::{Category, Config, IntoInterruptiblePipelineData, PipelineData, Span, Spanned}; +use nu_protocol::{ByteStream, Category, Config, PipelineData, Spanned}; use itertools::Itertools; use nu_engine::CallExt; use regex::Regex; -const OUTPUT_BUFFER_SIZE: usize = 8192; +const OUTPUT_BUFFER_SIZE: usize = 1024; #[derive(Clone)] pub struct External; @@ -137,6 +136,7 @@ impl<'call> ExternalCommand<'call> { config: Config, ) -> Result { let mut process = self.create_command(); + let head = self.name.span; let ctrlc = engine_state.ctrlc.clone(); @@ -223,11 +223,7 @@ impl<'call> ExternalCommand<'call> { // from bytes to String. If no replacements are required, then the // borrowed value is a proper UTF-8 string. The Owned option represents // a string where the values had to be replaced, thus marking it as bytes - let data = match String::from_utf8_lossy(bytes) { - Cow::Borrowed(s) => Data::String(s.into()), - Cow::Owned(_) => Data::Bytes(bytes.to_vec()), - }; - + let bytes = bytes.to_vec(); let length = bytes.len(); buf_read.consume(length); @@ -237,7 +233,7 @@ impl<'call> ExternalCommand<'call> { } } - match tx.send(data) { + match tx.send(bytes) { Ok(_) => continue, Err(_) => break, } @@ -249,11 +245,16 @@ impl<'call> ExternalCommand<'call> { Ok(_) => Ok(()), } }); - // The ValueStream is consumed by the next expression in the pipeline - let value = - ChannelReceiver::new(rx, self.name.span).into_pipeline_data(output_ctrlc); + let receiver = ChannelReceiver::new(rx); - Ok(value) + Ok(PipelineData::ByteStream( + ByteStream { + stream: Box::new(receiver), + ctrlc: output_ctrlc, + }, + head, + None, + )) } } } @@ -345,42 +346,24 @@ fn trim_enclosing_quotes(input: &str) -> String { } } -// The piped data from stdout from the external command can be either String -// or binary. We use this enum to pass the data from the spawned process -#[derive(Debug)] -enum Data { - String(String), - Bytes(Vec), -} - // Receiver used for the ValueStream // It implements iterator so it can be used as a ValueStream struct ChannelReceiver { - rx: mpsc::Receiver, - span: Span, + rx: mpsc::Receiver>, } impl ChannelReceiver { - pub fn new(rx: mpsc::Receiver, span: Span) -> Self { - Self { rx, span } + pub fn new(rx: mpsc::Receiver>) -> Self { + Self { rx } } } impl Iterator for ChannelReceiver { - type Item = Value; + type Item = Vec; fn next(&mut self) -> Option { match self.rx.recv() { - Ok(v) => match v { - Data::String(s) => Some(Value::String { - val: s, - span: self.span, - }), - Data::Bytes(b) => Some(Value::Binary { - val: b, - span: self.span, - }), - }, + Ok(v) => Some(v), Err(_) => None, } } diff --git a/crates/nu-command/src/viewers/griddle.rs b/crates/nu-command/src/viewers/griddle.rs index d0b36990f2..85f7aac307 100644 --- a/crates/nu-command/src/viewers/griddle.rs +++ b/crates/nu-command/src/viewers/griddle.rs @@ -86,7 +86,7 @@ prints out the list properly."# Ok(PipelineData::new(call.head)) } } - PipelineData::Stream(stream, ..) => { + PipelineData::ListStream(stream, ..) => { // dbg!("value::stream"); let data = convert_to_list(stream, &config, call.head); if let Some(items) = data { diff --git a/crates/nu-command/src/viewers/table.rs b/crates/nu-command/src/viewers/table.rs index b76512c991..d5ac7f144b 100644 --- a/crates/nu-command/src/viewers/table.rs +++ b/crates/nu-command/src/viewers/table.rs @@ -4,8 +4,8 @@ use nu_engine::{env_to_string, CallExt}; use nu_protocol::ast::{Call, PathMember}; use nu_protocol::engine::{Command, EngineState, Stack}; use nu_protocol::{ - Category, Config, DataSource, IntoInterruptiblePipelineData, IntoPipelineData, PipelineData, - PipelineMetadata, ShellError, Signature, Span, SyntaxShape, Value, ValueStream, + Category, Config, DataSource, IntoPipelineData, PipelineData, PipelineMetadata, ShellError, + Signature, Span, StringStream, SyntaxShape, Value, ValueStream, }; use nu_table::{StyledString, TextStyle, Theme}; use std::sync::atomic::{AtomicBool, Ordering}; @@ -47,6 +47,7 @@ impl Command for Table { call: &Call, input: PipelineData, ) -> Result { + let head = call.head; let ctrlc = engine_state.ctrlc.clone(); let config = stack.get_config().unwrap_or_default(); let color_hm = get_color_config(&config); @@ -60,6 +61,20 @@ impl Command for Table { }; match input { + PipelineData::ByteStream(stream, ..) => Ok(PipelineData::StringStream( + StringStream::from_stream( + stream.map(move |x| { + Ok(if x.iter().all(|x| x.is_ascii()) { + format!("{}", String::from_utf8_lossy(&x)) + } else { + format!("{}\n", nu_pretty_hex::pretty_hex(&x)) + }) + }), + ctrlc, + ), + head, + None, + )), PipelineData::Value(Value::List { vals, .. }, ..) => { let table = convert_to_table(row_offset, &vals, ctrlc, &config, call.head)?; @@ -75,7 +90,7 @@ impl Command for Table { Ok(PipelineData::new(call.head)) } } - PipelineData::Stream(stream, metadata) => { + PipelineData::ListStream(stream, metadata) => { let stream = match metadata { Some(PipelineMetadata { data_source: DataSource::Ls, @@ -161,14 +176,20 @@ impl Command for Table { let head = call.head; - Ok(PagingTableCreator { - row_offset, - config, - ctrlc: ctrlc.clone(), + Ok(PipelineData::StringStream( + StringStream::from_stream( + PagingTableCreator { + row_offset, + config, + ctrlc: ctrlc.clone(), + head, + stream, + }, + ctrlc, + ), head, - stream, - } - .into_pipeline_data(ctrlc)) + None, + )) } PipelineData::Value(Value::Record { cols, vals, .. }, ..) => { let mut output = vec![]; @@ -363,7 +384,7 @@ struct PagingTableCreator { } impl Iterator for PagingTableCreator { - type Item = Value; + type Item = Result; fn next(&mut self) -> Option { let mut batch = vec![]; @@ -418,12 +439,9 @@ impl Iterator for PagingTableCreator { Ok(Some(table)) => { let result = nu_table::draw_table(&table, term_width, &color_hm, &self.config); - Some(Value::String { - val: result, - span: self.head, - }) + Some(Ok(result)) } - Err(err) => Some(Value::Error { error: err }), + Err(err) => Some(Err(err)), _ => None, } } diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index e85e921c7a..f0fd12b83f 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -435,7 +435,7 @@ pub fn eval_subexpression( let config = stack.get_config().unwrap_or_default(); - let mut s = input.collect_string("", &config); + let mut s = input.collect_string("", &config)?; if s.ends_with('\n') { s.pop(); } diff --git a/crates/nu-plugin/src/plugin/declaration.rs b/crates/nu-plugin/src/plugin/declaration.rs index a3c870f4c5..62c41c3445 100644 --- a/crates/nu-plugin/src/plugin/declaration.rs +++ b/crates/nu-plugin/src/plugin/declaration.rs @@ -72,7 +72,7 @@ impl Command for PluginDeclaration { let input = match input { PipelineData::Value(value, ..) => value, - PipelineData::Stream(stream, ..) => { + PipelineData::ListStream(stream, ..) => { let values = stream.collect::>(); Value::List { @@ -80,6 +80,22 @@ impl Command for PluginDeclaration { span: call.head, } } + PipelineData::StringStream(stream, ..) => { + let val = stream.into_string("")?; + + Value::String { + val, + span: call.head, + } + } + PipelineData::ByteStream(stream, ..) => { + let val = stream.into_vec(); + + Value::Binary { + val, + span: call.head, + } + } }; // Create message to plugin to indicate that signature is required and diff --git a/crates/nu-pretty-hex/Cargo.lock b/crates/nu-pretty-hex/Cargo.lock new file mode 100644 index 0000000000..1bfc1820e4 --- /dev/null +++ b/crates/nu-pretty-hex/Cargo.lock @@ -0,0 +1,201 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "as-slice" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45403b49e3954a4b8428a0ac21a4b7afadccf92bfd96273f1a58cd4812496ae0" +dependencies = [ + "generic-array 0.12.4", + "generic-array 0.13.3", + "generic-array 0.14.4", + "stable_deref_trait", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f797e67af32588215eaaab8327027ee8e71b9dd0b2b26996aedf20c030fce309" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hash32" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4041af86e63ac4298ce40e5cca669066e75b6f1aa3390fe2561ffa5e1d9f4cc" +dependencies = [ + "byteorder", +] + +[[package]] +name = "heapless" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634bd4d29cbf24424d0a4bfcbf80c6960129dc24424752a7d1d1390607023422" +dependencies = [ + "as-slice", + "generic-array 0.14.4", + "hash32", + "stable_deref_trait", +] + +[[package]] +name = "libc" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" + +[[package]] +name = "nu-ansi-term" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd69a141e8fdfa5ac882d8b816db2b9ad138ef7e3baa7cb753a9b3789aa8c7e" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-pretty-hex" +version = "0.2.1" +dependencies = [ + "heapless", + "nu-ansi-term", + "rand", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "typenum" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/crates/nu-pretty-hex/Cargo.toml b/crates/nu-pretty-hex/Cargo.toml new file mode 100644 index 0000000000..6024763efa --- /dev/null +++ b/crates/nu-pretty-hex/Cargo.toml @@ -0,0 +1,27 @@ +[package] +authors = ["Andrei Volnin ", "The Nu Project Contributors"] +description = "Pretty hex dump of bytes slice in the common style." +edition = "2018" +license = "MIT" +name = "nu-pretty-hex" +version = "0.41.0" + +[lib] +doctest = false +name = "nu_pretty_hex" +path = "src/lib.rs" + +[[bin]] +name = "nu_pretty_hex" +path = "src/main.rs" + +[dependencies] +nu-ansi-term = "0.39.0" +rand = "0.8.3" + +[dev-dependencies] +heapless = { version = "0.7.8", default-features = false } + +# [features] +# default = ["alloc"] +# alloc = [] diff --git a/crates/nu-pretty-hex/LICENSE b/crates/nu-pretty-hex/LICENSE new file mode 100644 index 0000000000..f70169fa2f --- /dev/null +++ b/crates/nu-pretty-hex/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Andrei Volnin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/nu-pretty-hex/README.md b/crates/nu-pretty-hex/README.md new file mode 100644 index 0000000000..7c640cc28f --- /dev/null +++ b/crates/nu-pretty-hex/README.md @@ -0,0 +1,81 @@ +# nu-pretty-hex + +An update of prett-hex to make it prettier + +[![crates.io](https://img.shields.io/crates/v/pretty-hex.svg)](https://crates.io/crates/pretty-hex) +[![docs.rs](https://docs.rs/pretty-hex/badge.svg)](https://docs.rs/pretty-hex) + +A Rust library providing pretty hex dump. + +A `simple_hex()` way renders one-line hex dump, a `pretty_hex()` way renders +columned multi-line hex dump with addressing and ASCII representation. +A `config_hex()` way renders hex dump in specified format. + +## Inspiration + +[Hexed](https://github.com/adolfohw/hexed) \ +[Hexyl](https://github.com/sharkdp/hexyl) \ +[Pretty-hex](https://github.com/wolandr/pretty-hex) + +## Example of `simple_hex()` + +```rust +use pretty_hex::*; + +let v = vec![222, 173, 190, 239, 202, 254, 32, 24]; +assert_eq!(simple_hex(&v), format!("{}", v.hex_dump())); + +println!("{}", v.hex_dump()); + +``` + +Output: + +```text +de ad be ef ca fe 20 18 +``` + +## Example of `pretty_hex()` + +```rust +use pretty_hex::*; + +let v: &[u8] = &random::<[u8;30]>(); +assert_eq!(pretty_hex(&v), format!("{:?}", v.hex_dump())); + +println!("{:?}", v.hex_dump()); +``` + +Output: + +```text +Length: 30 (0x1e) bytes +0000: 6b 4e 1a c3 af 03 d2 1e 7e 73 ba c8 bd 84 0f 83 kN......~s...... +0010: 89 d5 cf 90 23 67 4b 48 db b1 bc 35 bf ee ....#gKH...5.. +``` + +## Example of `config_hex()` + +```rust +use pretty_hex::*; + +let cfg = HexConfig {title: false, width: 8, group: 0, ..HexConfig::default() }; + +let v = &include_bytes!("data"); +assert_eq!(config_hex(&v, cfg), format!("{:?}", v.hex_conf(cfg))); + +println!("{:?}", v.hex_conf(cfg)); +``` + +Output: + +```text +0000: 6b 4e 1a c3 af 03 d2 1e kN...... +0008: 7e 73 ba c8 bd 84 0f 83 ~s...... +0010: 89 d5 cf 90 23 67 4b 48 ....#gKH +0018: db b1 bc 35 bf ee ...5.. +``` + +--- + +Inspired by [haskell's pretty-hex](https://hackage.haskell.org/package/pretty-hex-1.0). diff --git a/crates/nu-pretty-hex/src/lib.rs b/crates/nu-pretty-hex/src/lib.rs new file mode 100644 index 0000000000..d34542a7ee --- /dev/null +++ b/crates/nu-pretty-hex/src/lib.rs @@ -0,0 +1,66 @@ +// #![no_std] + +//! A Rust library providing pretty hex dump. +//! +//! A `simple_hex()` way renders one-line hex dump, and a `pretty_hex()` way renders +//! columned multi-line hex dump with addressing and ASCII representation. +//! A `config_hex()` way renders hex dump in specified format. +//! +//! ## Example of `simple_hex()` +//! ``` +//! use pretty_hex::*; +//! +//! let v = vec![222, 173, 190, 239, 202, 254, 32, 24]; +//! # #[cfg(feature = "alloc")] +//! assert_eq!(simple_hex(&v), format!("{}", v.hex_dump())); +//! +//! println!("{}", v.hex_dump()); +//! ``` +//! Output: +//! +//! ```text +//! de ad be ef ca fe 20 18 +//! ``` +//! ## Example of `pretty_hex()` +//! ``` +//! use pretty_hex::*; +//! +//! let v = &include_bytes!("../tests/data"); +//! # #[cfg(feature = "alloc")] +//! assert_eq!(pretty_hex(&v), format!("{:?}", v.hex_dump())); +//! +//! println!("{:?}", v.hex_dump()); +//! ``` +//! Output: +//! +//! ```text +//! Length: 30 (0x1e) bytes +//! 0000: 6b 4e 1a c3 af 03 d2 1e 7e 73 ba c8 bd 84 0f 83 kN......~s...... +//! 0010: 89 d5 cf 90 23 67 4b 48 db b1 bc 35 bf ee ....#gKH...5.. +//! ``` +//! ## Example of `config_hex()` +//! ``` +//! use pretty_hex::*; +//! +//! let cfg = HexConfig {title: false, width: 8, group: 0, ..HexConfig::default() }; +//! +//! let v = &include_bytes!("../tests/data"); +//! # #[cfg(feature = "alloc")] +//! assert_eq!(config_hex(&v, cfg), format!("{:?}", v.hex_conf(cfg))); +//! +//! println!("{:?}", v.hex_conf(cfg)); +//! ``` +//! Output: +//! +//! ```text +//! 0000: 6b 4e 1a c3 af 03 d2 1e kN...... +//! 0008: 7e 73 ba c8 bd 84 0f 83 ~s...... +//! 0010: 89 d5 cf 90 23 67 4b 48 ....#gKH +//! 0018: db b1 bc 35 bf ee ...5.. +//! ``` + +#[cfg(feature = "alloc")] +extern crate alloc; + +mod pretty_hex; +pub use pretty_hex::*; diff --git a/crates/nu-pretty-hex/src/main.rs b/crates/nu-pretty-hex/src/main.rs new file mode 100644 index 0000000000..4156cb5c2a --- /dev/null +++ b/crates/nu-pretty-hex/src/main.rs @@ -0,0 +1,50 @@ +use nu_pretty_hex::*; + +fn main() { + let config = HexConfig { + title: true, + ascii: true, + width: 16, + group: 4, + chunk: 1, + skip: Some(10), + // length: Some(5), + // length: None, + length: Some(50), + }; + + let my_string = "Darren Schroeder 😉"; + println!("ConfigHex\n{}\n", config_hex(&my_string, config)); + println!("SimpleHex\n{}\n", simple_hex(&my_string)); + println!("PrettyHex\n{}\n", pretty_hex(&my_string)); + println!("ConfigHex\n{}\n", config_hex(&my_string, config)); + + // let mut my_str = String::new(); + // for x in 0..256 { + // my_str.push(x as u8); + // } + let mut v: Vec = vec![]; + for x in 0..=127 { + v.push(x); + } + let my_str = String::from_utf8_lossy(&v[..]); + + println!("First128\n{}\n", pretty_hex(&my_str.as_bytes())); + println!( + "First128-Param\n{}\n", + config_hex(&my_str.as_bytes(), config) + ); + + let mut r_str = String::new(); + for _ in 0..=127 { + r_str.push(rand::random::() as char); + } + + println!("Random127\n{}\n", pretty_hex(&r_str)); +} + +//chunk 0 44617272656e20536368726f65646572 Darren Schroeder +//chunk 1 44 61 72 72 65 6e 20 53 63 68 72 6f 65 64 65 72 Darren Schroeder +//chunk 2 461 7272 656e 2053 6368 726f 6564 6572 Darren Schroeder +//chunk 3 46172 72656e 205363 68726f 656465 72 Darren Schroeder +//chunk 4 44617272 656e2053 6368726f 65646572 Darren Schroeder diff --git a/crates/nu-pretty-hex/src/pretty_hex.rs b/crates/nu-pretty-hex/src/pretty_hex.rs new file mode 100644 index 0000000000..99ebaf022c --- /dev/null +++ b/crates/nu-pretty-hex/src/pretty_hex.rs @@ -0,0 +1,299 @@ +use core::primitive::str; +use core::{default::Default, fmt}; +use nu_ansi_term::{Color, Style}; + +/// Returns a one-line hexdump of `source` grouped in default format without header +/// and ASCII column. +pub fn simple_hex>(source: &T) -> String { + let mut writer = String::new(); + hex_write(&mut writer, source, HexConfig::simple(), None).unwrap_or(()); + writer +} + +/// Dump `source` as hex octets in default format without header and ASCII column to the `writer`. +pub fn simple_hex_write(writer: &mut W, source: &T) -> fmt::Result +where + T: AsRef<[u8]>, + W: fmt::Write, +{ + hex_write(writer, source, HexConfig::simple(), None) +} + +/// Return a multi-line hexdump in default format complete with addressing, hex digits, +/// and ASCII representation. +pub fn pretty_hex>(source: &T) -> String { + let mut writer = String::new(); + hex_write(&mut writer, source, HexConfig::default(), Some(true)).unwrap_or(()); + writer +} + +/// Write multi-line hexdump in default format complete with addressing, hex digits, +/// and ASCII representation to the writer. +pub fn pretty_hex_write(writer: &mut W, source: &T) -> fmt::Result +where + T: AsRef<[u8]>, + W: fmt::Write, +{ + hex_write(writer, source, HexConfig::default(), Some(true)) +} + +/// Return a hexdump of `source` in specified format. +pub fn config_hex>(source: &T, cfg: HexConfig) -> String { + let mut writer = String::new(); + hex_write(&mut writer, source, cfg, Some(true)).unwrap_or(()); + writer +} + +/// Configuration parameters for hexdump. +#[derive(Clone, Copy, Debug)] +pub struct HexConfig { + /// Write first line header with data length. + pub title: bool, + /// Append ASCII representation column. + pub ascii: bool, + /// Source bytes per row. 0 for single row without address prefix. + pub width: usize, + /// Chunks count per group. 0 for single group (column). + pub group: usize, + /// Source bytes per chunk (word). 0 for single word. + pub chunk: usize, + /// Bytes from 0 to skip + pub skip: Option, + /// Length to return + pub length: Option, +} + +/// Default configuration with `title`, `ascii`, 16 source bytes `width` grouped to 4 separate +/// hex bytes. Using in `pretty_hex`, `pretty_hex_write` and `fmt::Debug` implementation. +impl Default for HexConfig { + fn default() -> HexConfig { + HexConfig { + title: true, + ascii: true, + width: 16, + group: 4, + chunk: 1, + skip: None, + length: None, + } + } +} + +impl HexConfig { + /// Returns configuration for `simple_hex`, `simple_hex_write` and `fmt::Display` implementation. + pub fn simple() -> Self { + HexConfig::default().to_simple() + } + + fn delimiter(&self, i: usize) -> &'static str { + if i > 0 && self.chunk > 0 && i % self.chunk == 0 { + if self.group > 0 && i % (self.group * self.chunk) == 0 { + " " + } else { + " " + } + } else { + "" + } + } + + fn to_simple(self) -> Self { + HexConfig { + title: false, + ascii: false, + width: 0, + ..self + } + } +} + +fn categorize_byte(byte: &u8) -> (Style, Option) { + // This section is here so later we can configure these items + let null_char_style = Style::default().fg(Color::Fixed(242)); + let null_char = Some('0'); + let ascii_printable_style = Style::default().fg(Color::Cyan).bold(); + let ascii_printable = None; + let ascii_space_style = Style::default().fg(Color::Green).bold(); + let ascii_space = Some(' '); + let ascii_white_space_style = Style::default().fg(Color::Green).bold(); + let ascii_white_space = Some('_'); + let ascii_other_style = Style::default().fg(Color::Purple).bold(); + let ascii_other = Some('•'); + let non_ascii_style = Style::default().fg(Color::Yellow).bold(); + let non_ascii = Some('×'); // or Some('.') + + if byte == &0 { + (null_char_style, null_char) + } else if byte.is_ascii_graphic() { + (ascii_printable_style, ascii_printable) + } else if byte.is_ascii_whitespace() { + // 0x20 == 32 decimal - replace with a real space + if byte == &32 { + (ascii_space_style, ascii_space) + } else { + (ascii_white_space_style, ascii_white_space) + } + } else if byte.is_ascii() { + (ascii_other_style, ascii_other) + } else { + (non_ascii_style, non_ascii) + } +} + +/// Write hex dump in specified format. +pub fn hex_write( + writer: &mut W, + source: &T, + cfg: HexConfig, + with_color: Option, +) -> fmt::Result +where + T: AsRef<[u8]>, + W: fmt::Write, +{ + let use_color = with_color.unwrap_or(false); + + if source.as_ref().is_empty() { + return Ok(()); + } + + let amount = match cfg.length { + Some(len) => len, + None => source.as_ref().len(), + }; + + let skip = cfg.skip.unwrap_or(0); + + let source_part_vec: Vec = source + .as_ref() + .iter() + .skip(skip) + .take(amount) + .map(|&x| x as u8) + .collect(); + + if cfg.title { + if use_color { + writeln!( + writer, + "Length: {0} (0x{0:x}) bytes | {1}printable {2}whitespace {3}ascii_other {4}non_ascii{5}", + source_part_vec.len(), + Style::default().fg(Color::Cyan).bold().prefix(), + Style::default().fg(Color::Green).bold().prefix(), + Style::default().fg(Color::Purple).bold().prefix(), + Style::default().fg(Color::Yellow).bold().prefix(), + Style::default().fg(Color::Yellow).suffix() + )?; + } else { + writeln!(writer, "Length: {0} (0x{0:x}) bytes", source_part_vec.len(),)?; + } + } + + let lines = source_part_vec.chunks(if cfg.width > 0 { + cfg.width + } else { + source_part_vec.len() + }); + + let lines_len = lines.len(); + + for (i, row) in lines.enumerate() { + if cfg.width > 0 { + let style = Style::default().fg(Color::Cyan); + if use_color { + write!( + writer, + "{}{:08x}{}: ", + style.prefix(), + i * cfg.width + skip, + style.suffix() + )?; + } else { + write!(writer, "{:08x}: ", i * cfg.width + skip,)?; + } + } + for (i, x) in row.as_ref().iter().enumerate() { + if use_color { + let (style, _char) = categorize_byte(x); + write!( + writer, + "{}{}{:02x}{}", + cfg.delimiter(i), + style.prefix(), + x, + style.suffix() + )?; + } else { + write!(writer, "{}{:02x}", cfg.delimiter(i), x,)?; + } + } + if cfg.ascii { + for j in row.len()..cfg.width { + write!(writer, "{} ", cfg.delimiter(j))?; + } + write!(writer, " ")?; + for x in row { + let (style, a_char) = categorize_byte(x); + let replacement_char = match a_char { + Some(c) => c, + None => *x as char, + }; + if use_color { + write!( + writer, + "{}{}{}", + style.prefix(), + replacement_char, + style.suffix() + )?; + } else { + write!(writer, "{}", replacement_char,)?; + } + } + } + if i + 1 < lines_len { + writeln!(writer)?; + } + } + Ok(()) +} + +/// Reference wrapper for use in arguments formatting. +pub struct Hex<'a, T: 'a>(&'a T, HexConfig); + +impl<'a, T: 'a + AsRef<[u8]>> fmt::Display for Hex<'a, T> { + /// Formats the value by `simple_hex_write` using the given formatter. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + hex_write(f, self.0, self.1.to_simple(), None) + } +} + +impl<'a, T: 'a + AsRef<[u8]>> fmt::Debug for Hex<'a, T> { + /// Formats the value by `pretty_hex_write` using the given formatter. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + hex_write(f, self.0, self.1, None) + } +} + +/// Allows generates hex dumps to a formatter. +pub trait PrettyHex: Sized { + /// Wrap self reference for use in `std::fmt::Display` and `std::fmt::Debug` + /// formatting as hex dumps. + fn hex_dump(&self) -> Hex; + + /// Wrap self reference for use in `std::fmt::Display` and `std::fmt::Debug` + /// formatting as hex dumps in specified format. + fn hex_conf(&self, cfg: HexConfig) -> Hex; +} + +impl PrettyHex for T +where + T: AsRef<[u8]>, +{ + fn hex_dump(&self) -> Hex { + Hex(self, HexConfig::default()) + } + fn hex_conf(&self, cfg: HexConfig) -> Hex { + Hex(self, cfg) + } +} diff --git a/crates/nu-pretty-hex/tests/256.txt b/crates/nu-pretty-hex/tests/256.txt new file mode 100644 index 0000000000..0e08e874f7 --- /dev/null +++ b/crates/nu-pretty-hex/tests/256.txt @@ -0,0 +1,17 @@ +Length: 256 (0x100) bytes +0000: 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f ................ +0010: 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f ................ +0020: 20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f !"#$%&'()*+,-./ +0030: 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d 3e 3f 0123456789:;<=>? +0040: 40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f @ABCDEFGHIJKLMNO +0050: 50 51 52 53 54 55 56 57 58 59 5a 5b 5c 5d 5e 5f PQRSTUVWXYZ[\]^_ +0060: 60 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 6f `abcdefghijklmno +0070: 70 71 72 73 74 75 76 77 78 79 7a 7b 7c 7d 7e 7f pqrstuvwxyz{|}~. +0080: 80 81 82 83 84 85 86 87 88 89 8a 8b 8c 8d 8e 8f ................ +0090: 90 91 92 93 94 95 96 97 98 99 9a 9b 9c 9d 9e 9f ................ +00a0: a0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab ac ad ae af ................ +00b0: b0 b1 b2 b3 b4 b5 b6 b7 b8 b9 ba bb bc bd be bf ................ +00c0: c0 c1 c2 c3 c4 c5 c6 c7 c8 c9 ca cb cc cd ce cf ................ +00d0: d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 da db dc dd de df ................ +00e0: e0 e1 e2 e3 e4 e5 e6 e7 e8 e9 ea eb ec ed ee ef ................ +00f0: f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff ................ \ No newline at end of file diff --git a/crates/nu-pretty-hex/tests/data b/crates/nu-pretty-hex/tests/data new file mode 100644 index 0000000000..998d80f715 --- /dev/null +++ b/crates/nu-pretty-hex/tests/data @@ -0,0 +1 @@ +kNï~sȽϐ#gKH۱5 \ No newline at end of file diff --git a/crates/nu-pretty-hex/tests/tests.rs b/crates/nu-pretty-hex/tests/tests.rs new file mode 100644 index 0000000000..a147c496ce --- /dev/null +++ b/crates/nu-pretty-hex/tests/tests.rs @@ -0,0 +1,175 @@ +// #![no_std] + +#[cfg(feature = "alloc")] +extern crate alloc; +extern crate nu_pretty_hex; + +#[cfg(feature = "alloc")] +use alloc::{format, string::String, vec, vec::Vec}; +use nu_pretty_hex::*; + +#[cfg(feature = "alloc")] +#[test] +fn test_simple() { + let bytes: Vec = (0..16).collect(); + let expected = "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f"; + assert_eq!(expected, simple_hex(&bytes)); + assert_eq!(expected, bytes.hex_dump().to_string()); + assert_eq!(simple_hex(&bytes), config_hex(&bytes, HexConfig::simple())); + + let mut have = String::new(); + simple_hex_write(&mut have, &bytes).unwrap(); + assert_eq!(expected, have); + + let str = "string"; + let string: String = String::from("string"); + let slice: &[u8] = &[0x73, 0x74, 0x72, 0x69, 0x6e, 0x67]; + assert_eq!(simple_hex(&str), "73 74 72 69 6e 67"); + assert_eq!(simple_hex(&str), simple_hex(&string)); + assert_eq!(simple_hex(&str), simple_hex(&slice)); + + assert!(simple_hex(&vec![]).is_empty()); +} + +#[cfg(feature = "alloc")] +#[test] +fn test_pretty() { + let bytes: Vec = (0..256).map(|x| x as u8).collect(); + let want = include_str!("256.txt"); + + let mut hex = String::new(); + pretty_hex_write(&mut hex, &bytes).unwrap(); + assert_eq!(want, hex); + assert_eq!(want, format!("{:?}", bytes.hex_dump())); + assert_eq!(want, pretty_hex(&bytes)); + assert_eq!(want, config_hex(&bytes, HexConfig::default())); + + assert_eq!("Length: 0 (0x0) bytes\n", pretty_hex(&vec![])); +} + +#[cfg(feature = "alloc")] +#[test] +fn test_config() { + let cfg = HexConfig { + title: false, + ascii: false, + width: 0, + group: 0, + chunk: 0, + }; + assert!(config_hex(&vec![], cfg).is_empty()); + assert_eq!("2425262728", config_hex(&"$%&'(", cfg)); + + let v = include_bytes!("data"); + let cfg = HexConfig { + title: false, + group: 8, + ..HexConfig::default() + }; + let hex = "0000: 6b 4e 1a c3 af 03 d2 1e 7e 73 ba c8 bd 84 0f 83 kN......~s......\n\ + 0010: 89 d5 cf 90 23 67 4b 48 db b1 bc 35 bf ee ....#gKH...5.."; + assert_eq!(hex, config_hex(&v, cfg)); + assert_eq!(hex, format!("{:?}", v.hex_conf(cfg))); + let mut str = String::new(); + hex_write(&mut str, v, cfg).unwrap(); + assert_eq!(hex, str); + + assert_eq!( + config_hex( + &v, + HexConfig { + ascii: false, + ..cfg + } + ), + "0000: 6b 4e 1a c3 af 03 d2 1e 7e 73 ba c8 bd 84 0f 83\n\ + 0010: 89 d5 cf 90 23 67 4b 48 db b1 bc 35 bf ee" + ); + + assert_eq!( + config_hex( + &v, + HexConfig { + ascii: false, + group: 4, + chunk: 2, + ..cfg + } + ), + "0000: 6b4e 1ac3 af03 d21e 7e73 bac8 bd84 0f83\n\ + 0010: 89d5 cf90 2367 4b48 dbb1 bc35 bfee" + ); + + let v: Vec = (0..21).collect(); + let want = r##"Length: 21 (0x15) bytes +0000: 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f ................ +0010: 10 11 12 13 14 ....."##; + assert_eq!(want, pretty_hex(&v)); + + let v: Vec = (0..13).collect(); + assert_eq!( + config_hex( + &v, + HexConfig { + title: false, + ascii: true, + width: 11, + group: 2, + chunk: 3 + } + ), + "0000: 000102 030405 060708 090a ...........\n\ + 000b: 0b0c .." + ); + + let v: Vec = (0..19).collect(); + assert_eq!( + config_hex( + &v, + HexConfig { + title: false, + ascii: true, + width: 16, + group: 3, + chunk: 3 + } + ), + "0000: 000102 030405 060708 090a0b 0c0d0e 0f ................\n\ + 0010: 101112 ..." + ); + + let cfg = HexConfig { + title: false, + group: 0, + ..HexConfig::default() + }; + assert_eq!( + format!("{:?}", v.hex_conf(cfg)), + "0000: 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f ................\n\ + 0010: 10 11 12 ..." + ); + assert_eq!( + v.hex_conf(cfg).to_string(), + "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12" + ); +} + +// This test case checks that hex_write works even without the alloc crate. +// Decorators to this function like simple_hex_write or PrettyHex::hex_dump() +// will be tested when the alloc feature is selected because it feels quite +// cumbersome to set up these tests without the comodity from `alloc`. +#[test] +fn test_hex_write_with_simple_config() { + let config = HexConfig::simple(); + let bytes = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + let expected = + core::str::from_utf8(b"00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f").unwrap(); + // let expected = + // "\u{1b}[38;5;242m00\u{1b}[0m \u{1b}[1;35m01\u{1b}[0m \u{1b}[1;35m02\u{1b}[0m \u{1b}[1;"; + let mut buffer = heapless::Vec::::new(); + + hex_write(&mut buffer, &bytes, config, None).unwrap(); + + let have = core::str::from_utf8(&buffer).unwrap(); + assert_eq!(expected, have); +} diff --git a/crates/nu-protocol/src/pipeline_data.rs b/crates/nu-protocol/src/pipeline_data.rs index d26a5c1d19..b6c85991a8 100644 --- a/crates/nu-protocol/src/pipeline_data.rs +++ b/crates/nu-protocol/src/pipeline_data.rs @@ -1,6 +1,8 @@ use std::sync::{atomic::AtomicBool, Arc}; -use crate::{ast::PathMember, Config, ShellError, Span, Value, ValueStream}; +use crate::{ + ast::PathMember, ByteStream, Config, ShellError, Span, StringStream, Value, ValueStream, +}; /// The foundational abstraction for input and output to commands /// @@ -34,7 +36,9 @@ use crate::{ast::PathMember, Config, ShellError, Span, Value, ValueStream}; #[derive(Debug)] pub enum PipelineData { Value(Value, Option), - Stream(ValueStream, Option), + ListStream(ValueStream, Option), + StringStream(StringStream, Span, Option), + ByteStream(ByteStream, Span, Option), } #[derive(Debug, Clone)] @@ -54,7 +58,9 @@ impl PipelineData { pub fn metadata(&self) -> Option { match self { - PipelineData::Stream(_, x) => x.clone(), + PipelineData::ListStream(_, x) => x.clone(), + PipelineData::ByteStream(_, _, x) => x.clone(), + PipelineData::StringStream(_, _, x) => x.clone(), PipelineData::Value(_, x) => x.clone(), } } @@ -63,27 +69,49 @@ impl PipelineData { match self { PipelineData::Value(Value::Nothing { .. }, ..) => Value::nothing(span), PipelineData::Value(v, ..) => v, - PipelineData::Stream(s, ..) => Value::List { + PipelineData::ListStream(s, ..) => Value::List { vals: s.collect(), span, // FIXME? }, + PipelineData::StringStream(s, ..) => { + let mut output = String::new(); + + for item in s { + match item { + Ok(s) => output.push_str(&s), + Err(err) => return Value::Error { error: err }, + } + } + Value::String { + val: output, + span, // FIXME? + } + } + PipelineData::ByteStream(s, ..) => Value::Binary { + val: s.flatten().collect(), + span, // FIXME? + }, } } pub fn into_interruptible_iter(self, ctrlc: Option>) -> PipelineIterator { let mut iter = self.into_iter(); - if let PipelineIterator(PipelineData::Stream(s, ..)) = &mut iter { + if let PipelineIterator(PipelineData::ListStream(s, ..)) = &mut iter { s.ctrlc = ctrlc; } iter } - pub fn collect_string(self, separator: &str, config: &Config) -> String { + pub fn collect_string(self, separator: &str, config: &Config) -> Result { match self { - PipelineData::Value(v, ..) => v.into_string(separator, config), - PipelineData::Stream(s, ..) => s.into_string(separator, config), + PipelineData::Value(v, ..) => Ok(v.into_string(separator, config)), + PipelineData::ListStream(s, ..) => Ok(s.into_string(separator, config)), + PipelineData::StringStream(s, ..) => s.into_string(separator), + PipelineData::ByteStream(s, ..) => { + Ok(String::from_utf8_lossy(&s.flatten().collect::>()).to_string()) + } } } @@ -94,12 +122,13 @@ impl PipelineData { ) -> Result { match self { // FIXME: there are probably better ways of doing this - PipelineData::Stream(stream, ..) => Value::List { + PipelineData::ListStream(stream, ..) => Value::List { vals: stream.collect(), span: head, } .follow_cell_path(cell_path), PipelineData::Value(v, ..) => v.follow_cell_path(cell_path), + _ => Err(ShellError::IOError("can't follow stream paths".into())), } } @@ -111,12 +140,13 @@ impl PipelineData { ) -> Result<(), ShellError> { match self { // FIXME: there are probably better ways of doing this - PipelineData::Stream(stream, ..) => Value::List { + PipelineData::ListStream(stream, ..) => Value::List { vals: stream.collect(), span: head, } .update_cell_path(cell_path, callback), PipelineData::Value(v, ..) => v.update_cell_path(cell_path, callback), + _ => Ok(()), } } @@ -134,7 +164,14 @@ impl PipelineData { PipelineData::Value(Value::List { vals, .. }, ..) => { Ok(vals.into_iter().map(f).into_pipeline_data(ctrlc)) } - PipelineData::Stream(stream, ..) => Ok(stream.map(f).into_pipeline_data(ctrlc)), + PipelineData::ListStream(stream, ..) => Ok(stream.map(f).into_pipeline_data(ctrlc)), + PipelineData::StringStream(stream, span, ..) => Ok(stream + .map(move |x| match x { + Ok(s) => f(Value::String { val: s, span }), + Err(err) => Value::Error { error: err }, + }) + .into_pipeline_data(ctrlc)), + PipelineData::Value(Value::Range { val, .. }, ..) => { Ok(val.into_range_iter()?.map(f).into_pipeline_data(ctrlc)) } @@ -142,6 +179,11 @@ impl PipelineData { Value::Error { error } => Err(error), v => Ok(v.into_pipeline_data()), }, + PipelineData::ByteStream(_, span, ..) => Err(ShellError::UnsupportedInput( + "Binary output from this command may need to be decoded using the 'decode' command" + .into(), + span, + )), } } @@ -161,14 +203,27 @@ impl PipelineData { PipelineData::Value(Value::List { vals, .. }, ..) => { Ok(vals.into_iter().map(f).flatten().into_pipeline_data(ctrlc)) } - PipelineData::Stream(stream, ..) => { + PipelineData::ListStream(stream, ..) => { Ok(stream.map(f).flatten().into_pipeline_data(ctrlc)) } + PipelineData::StringStream(stream, span, ..) => Ok(stream + .map(move |x| match x { + Ok(s) => Value::String { val: s, span }, + Err(err) => Value::Error { error: err }, + }) + .map(f) + .flatten() + .into_pipeline_data(ctrlc)), PipelineData::Value(Value::Range { val, .. }, ..) => match val.into_range_iter() { Ok(iter) => Ok(iter.map(f).flatten().into_pipeline_data(ctrlc)), Err(error) => Err(error), }, PipelineData::Value(v, ..) => Ok(f(v).into_iter().into_pipeline_data(ctrlc)), + PipelineData::ByteStream(_, span, ..) => Err(ShellError::UnsupportedInput( + "Binary output from this command may need to be decoded using the 'decode' command" + .into(), + span, + )), } } @@ -185,7 +240,15 @@ impl PipelineData { PipelineData::Value(Value::List { vals, .. }, ..) => { Ok(vals.into_iter().filter(f).into_pipeline_data(ctrlc)) } - PipelineData::Stream(stream, ..) => Ok(stream.filter(f).into_pipeline_data(ctrlc)), + PipelineData::ListStream(stream, ..) => Ok(stream.filter(f).into_pipeline_data(ctrlc)), + PipelineData::StringStream(stream, span, ..) => Ok(stream + .map(move |x| match x { + Ok(s) => Value::String { val: s, span }, + Err(err) => Value::Error { error: err }, + }) + .filter(f) + .into_pipeline_data(ctrlc)), + PipelineData::Value(Value::Range { val, .. }, ..) => { Ok(val.into_range_iter()?.filter(f).into_pipeline_data(ctrlc)) } @@ -196,16 +259,15 @@ impl PipelineData { Ok(Value::Nothing { span: v.span()? }.into_pipeline_data()) } } + PipelineData::ByteStream(_, span, ..) => Err(ShellError::UnsupportedInput( + "Binary output from this command may need to be decoded using the 'decode' command" + .into(), + span, + )), } } } -// impl Default for PipelineData { -// fn default() -> Self { -// PipelineData::new() -// } -// } - pub struct PipelineIterator(PipelineData); impl IntoIterator for PipelineData { @@ -216,7 +278,7 @@ impl IntoIterator for PipelineData { fn into_iter(self) -> Self::IntoIter { match self { PipelineData::Value(Value::List { vals, .. }, metadata) => { - PipelineIterator(PipelineData::Stream( + PipelineIterator(PipelineData::ListStream( ValueStream { stream: Box::new(vals.into_iter()), ctrlc: None, @@ -226,14 +288,14 @@ impl IntoIterator for PipelineData { } PipelineData::Value(Value::Range { val, .. }, metadata) => { match val.into_range_iter() { - Ok(iter) => PipelineIterator(PipelineData::Stream( + Ok(iter) => PipelineIterator(PipelineData::ListStream( ValueStream { stream: Box::new(iter), ctrlc: None, }, metadata, )), - Err(error) => PipelineIterator(PipelineData::Stream( + Err(error) => PipelineIterator(PipelineData::ListStream( ValueStream { stream: Box::new(std::iter::once(Value::Error { error })), ctrlc: None, @@ -254,7 +316,18 @@ impl Iterator for PipelineIterator { match &mut self.0 { PipelineData::Value(Value::Nothing { .. }, ..) => None, PipelineData::Value(v, ..) => Some(std::mem::take(v)), - PipelineData::Stream(stream, ..) => stream.next(), + PipelineData::ListStream(stream, ..) => stream.next(), + PipelineData::StringStream(stream, span, ..) => stream.next().map(|x| match x { + Ok(x) => Value::String { + val: x, + span: *span, + }, + Err(err) => Value::Error { error: err }, + }), + PipelineData::ByteStream(stream, span, ..) => stream.next().map(|x| Value::Binary { + val: x, + span: *span, + }), } } } @@ -288,7 +361,7 @@ where ::Item: Into, { fn into_pipeline_data(self, ctrlc: Option>) -> PipelineData { - PipelineData::Stream( + PipelineData::ListStream( ValueStream { stream: Box::new(self.into_iter().map(Into::into)), ctrlc, @@ -302,7 +375,7 @@ where metadata: PipelineMetadata, ctrlc: Option>, ) -> PipelineData { - PipelineData::Stream( + PipelineData::ListStream( ValueStream { stream: Box::new(self.into_iter().map(Into::into)), ctrlc, diff --git a/crates/nu-protocol/src/value/stream.rs b/crates/nu-protocol/src/value/stream.rs index 728bd0f567..9497886541 100644 --- a/crates/nu-protocol/src/value/stream.rs +++ b/crates/nu-protocol/src/value/stream.rs @@ -7,6 +7,94 @@ use std::{ }, }; +/// A single buffer of binary data streamed over multiple parts. Optionally contains ctrl-c that can be used +/// to break the stream. +pub struct ByteStream { + pub stream: Box> + Send + 'static>, + pub ctrlc: Option>, +} +impl ByteStream { + pub fn into_vec(self) -> Vec { + self.flatten().collect::>() + } +} +impl Debug for ByteStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ByteStream").finish() + } +} + +impl Iterator for ByteStream { + type Item = Vec; + + fn next(&mut self) -> Option { + if let Some(ctrlc) = &self.ctrlc { + if ctrlc.load(Ordering::SeqCst) { + None + } else { + self.stream.next() + } + } else { + self.stream.next() + } + } +} + +/// A single string streamed over multiple parts. Optionally contains ctrl-c that can be used +/// to break the stream. +pub struct StringStream { + pub stream: Box> + Send + 'static>, + pub ctrlc: Option>, +} +impl StringStream { + pub fn into_string(self, separator: &str) -> Result { + let mut output = String::new(); + + let mut first = true; + for s in self.stream { + output.push_str(&s?); + + if !first { + output.push_str(separator); + } else { + first = false; + } + } + Ok(output) + } + + pub fn from_stream( + input: impl Iterator> + Send + 'static, + ctrlc: Option>, + ) -> StringStream { + StringStream { + stream: Box::new(input), + ctrlc, + } + } +} +impl Debug for StringStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StringStream").finish() + } +} + +impl Iterator for StringStream { + type Item = Result; + + fn next(&mut self) -> Option { + if let Some(ctrlc) = &self.ctrlc { + if ctrlc.load(Ordering::SeqCst) { + None + } else { + self.stream.next() + } + } else { + self.stream.next() + } + } +} + /// A potentially infinite stream of values, optinally with a mean to send a Ctrl-C signal to stop /// the stream from continuing. /// diff --git a/src/main.rs b/src/main.rs index 409ac66339..9653f7e03a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -583,6 +583,28 @@ fn print_pipeline_data( let config = stack.get_config().unwrap_or_default(); + match input { + PipelineData::StringStream(stream, _, _) => { + for s in stream { + print!("{}", s?); + let _ = std::io::stdout().flush(); + } + return Ok(()); + } + PipelineData::ByteStream(stream, _, _) => { + for v in stream { + let s = if v.iter().all(|x| x.is_ascii()) { + format!("{}", String::from_utf8_lossy(&v)) + } else { + format!("{}\n", nu_pretty_hex::pretty_hex(&v)) + }; + println!("{}", s); + } + return Ok(()); + } + _ => {} + } + match engine_state.find_decl("table".as_bytes()) { Some(decl_id) => { let table = @@ -663,7 +685,12 @@ fn update_prompt<'prompt>( } }; - nu_prompt.update_prompt(evaluated_prompt); + match evaluated_prompt { + Ok(evaluated_prompt) => { + nu_prompt.update_prompt(evaluated_prompt); + } + _ => nu_prompt.update_prompt(String::new()), + } nu_prompt as &dyn Prompt }