From 479f0a566e132ab47ba931c663fec2177209f48f Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Wed, 4 Sep 2019 18:48:40 +1200 Subject: [PATCH] Covert to_* commands to work on whole table --- src/cli.rs | 3 - src/commands.rs | 4 -- src/commands/from_array.rs | 43 -------------- src/commands/to_array.rs | 38 ------------- src/commands/to_bson.rs | 50 +++++++++++----- src/commands/to_csv.rs | 113 +++++++++++++++++++++++++++---------- src/commands/to_json.rs | 50 ++++++++++------ src/commands/to_sqlite.rs | 20 +++---- src/commands/to_toml.rs | 52 +++++++++++------ src/commands/to_tsv.rs | 112 ++++++++++++++++++++++++++---------- src/commands/to_yaml.rs | 51 +++++++++++------ tests/filters_test.rs | 2 +- 12 files changed, 317 insertions(+), 221 deletions(-) delete mode 100644 src/commands/from_array.rs delete mode 100644 src/commands/to_array.rs diff --git a/src/cli.rs b/src/cli.rs index f70fa42d54..9b41c9dd21 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -179,7 +179,6 @@ pub async fn cli() -> Result<(), Box> { whole_stream_command(Reject), whole_stream_command(Reverse), whole_stream_command(Trim), - whole_stream_command(ToArray), whole_stream_command(ToBSON), whole_stream_command(ToCSV), whole_stream_command(ToJSON), @@ -192,8 +191,6 @@ pub async fn cli() -> Result<(), Box> { whole_stream_command(Tags), whole_stream_command(First), whole_stream_command(Last), - whole_stream_command(FromArray), - whole_stream_command(FromArray), whole_stream_command(FromCSV), whole_stream_command(FromTSV), whole_stream_command(FromINI), diff --git a/src/commands.rs b/src/commands.rs index bc6452090f..cd44fbb09b 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -15,7 +15,6 @@ pub(crate) mod enter; pub(crate) mod exit; pub(crate) mod fetch; pub(crate) mod first; -pub(crate) mod from_array; pub(crate) mod from_bson; pub(crate) mod from_csv; pub(crate) mod from_ini; @@ -52,7 +51,6 @@ pub(crate) mod split_column; pub(crate) mod split_row; pub(crate) mod table; pub(crate) mod tags; -pub(crate) mod to_array; pub(crate) mod to_bson; pub(crate) mod to_csv; pub(crate) mod to_json; @@ -81,7 +79,6 @@ pub(crate) use enter::Enter; pub(crate) use exit::Exit; pub(crate) use fetch::Fetch; pub(crate) use first::First; -pub(crate) use from_array::FromArray; pub(crate) use from_bson::FromBSON; pub(crate) use from_csv::FromCSV; pub(crate) use from_ini::FromINI; @@ -119,7 +116,6 @@ pub(crate) use split_column::SplitColumn; pub(crate) use split_row::SplitRow; pub(crate) use table::Table; pub(crate) use tags::Tags; -pub(crate) use to_array::ToArray; pub(crate) use to_bson::ToBSON; pub(crate) use to_csv::ToCSV; pub(crate) use to_json::ToJSON; diff --git a/src/commands/from_array.rs b/src/commands/from_array.rs deleted file mode 100644 index 93ba87ecea..0000000000 --- a/src/commands/from_array.rs +++ /dev/null @@ -1,43 +0,0 @@ -use crate::commands::WholeStreamCommand; -use crate::object::Value; -use crate::prelude::*; - -pub struct FromArray; - -impl WholeStreamCommand for FromArray { - fn name(&self) -> &str { - "from-array" - } - - fn signature(&self) -> Signature { - Signature::build("from-array") - } - - fn usage(&self) -> &str { - "Expand an array/list into rows" - } - - fn run( - &self, - args: CommandArgs, - registry: &CommandRegistry, - ) -> Result { - from_array(args, registry) - } -} - -fn from_array(args: CommandArgs, _registry: &CommandRegistry) -> Result { - let stream = args - .input - .values - .map(|item| match item { - Tagged { - item: Value::List(vec), - .. - } => VecDeque::from(vec), - x => VecDeque::from(vec![x]), - }) - .flatten(); - - Ok(stream.to_output_stream()) -} diff --git a/src/commands/to_array.rs b/src/commands/to_array.rs deleted file mode 100644 index 04c429e1b4..0000000000 --- a/src/commands/to_array.rs +++ /dev/null @@ -1,38 +0,0 @@ -use crate::commands::WholeStreamCommand; -use crate::object::Value; -use crate::prelude::*; - -pub struct ToArray; - -impl WholeStreamCommand for ToArray { - fn name(&self) -> &str { - "to-array" - } - - fn signature(&self) -> Signature { - Signature::build("to-array") - } - - fn usage(&self) -> &str { - "Collapse rows into a single list." - } - - fn run( - &self, - args: CommandArgs, - registry: &CommandRegistry, - ) -> Result { - to_array(args, registry) - } -} - -fn to_array(args: CommandArgs, registry: &CommandRegistry) -> Result { - let args = args.evaluate_once(registry)?; - let span = args.call_info.name_span; - let out = args.input.values.collect(); - - Ok(out - .map(move |vec: Vec<_>| stream![Value::List(vec).simple_spanned(span)]) - .flatten_stream() - .from_input_stream()) -} diff --git a/src/commands/to_bson.rs b/src/commands/to_bson.rs index 73232e75b0..8da3cf2a6d 100644 --- a/src/commands/to_bson.rs +++ b/src/commands/to_bson.rs @@ -236,21 +236,41 @@ fn bson_value_to_bytes(bson: Bson, span: Span) -> Result, ShellError> { fn to_bson(args: CommandArgs, registry: &CommandRegistry) -> Result { let args = args.evaluate_once(registry)?; let name_span = args.name_span(); - let out = args.input; + let stream = async_stream_block! { + let input: Vec> = args.input.values.collect().await; - Ok(out - .values - .map( - move |a| match bson_value_to_bytes(value_to_bson_value(&a)?, name_span) { - Ok(x) => ReturnSuccess::value(Value::Binary(x).simple_spanned(name_span)), - _ => Err(ShellError::labeled_error_with_secondary( + let to_process_input = if input.len() > 1 { + let tag = input[0].tag; + vec![Tagged { item: Value::List(input), tag } ] + } else if input.len() == 1 { + input + } else { + vec![] + }; + + for value in to_process_input { + match value_to_bson_value(&value) { + Ok(bson_value) => { + match bson_value_to_bytes(bson_value, name_span) { + Ok(x) => yield ReturnSuccess::value( + Value::Binary(x).simple_spanned(name_span), + ), + _ => yield Err(ShellError::labeled_error_with_secondary( + "Expected an object with BSON-compatible structure.span() from pipeline", + "requires BSON-compatible input", + name_span, + "originates from here".to_string(), + value.span(), + )), + } + } + _ => yield Err(ShellError::labeled_error( "Expected an object with BSON-compatible structure from pipeline", - "requires BSON-compatible input: Must be Array or Object", - name_span, - format!("{} originates from here", a.item.type_name()), - a.span(), - )), - }, - ) - .to_output_stream()) + "requires BSON-compatible input", + name_span)) + } + } + }; + + Ok(stream.to_output_stream()) } diff --git a/src/commands/to_csv.rs b/src/commands/to_csv.rs index 58ad208192..058ad585de 100644 --- a/src/commands/to_csv.rs +++ b/src/commands/to_csv.rs @@ -16,8 +16,7 @@ impl WholeStreamCommand for ToCSV { } fn signature(&self) -> Signature { - Signature::build("to-csv") - .switch("headerless") + Signature::build("to-csv").switch("headerless") } fn usage(&self) -> &str { @@ -47,7 +46,7 @@ pub fn value_to_csv_value(v: &Value) -> Value { } } -fn to_string_helper(v: &Value) -> Result> { +fn to_string_helper(v: &Value) -> Result { match v { Value::Primitive(Primitive::Date(d)) => Ok(d.to_string()), Value::Primitive(Primitive::Bytes(b)) => Ok(format!("{}", b)), @@ -55,11 +54,23 @@ fn to_string_helper(v: &Value) -> Result> { Value::List(_) => return Ok(String::from("[list list]")), Value::Object(_) => return Ok(String::from("[object]")), Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()), - _ => return Err("Bad input".into()), + _ => return Err(ShellError::string("Unexpected value")), } } -pub fn to_string(v: &Value) -> Result> { +fn merge_descriptors(values: &[Tagged]) -> Vec { + let mut ret = vec![]; + for value in values { + for desc in value.data_descriptors() { + if !ret.contains(&desc) { + ret.push(desc); + } + } + } + ret +} + +pub fn to_string(v: &Value) -> Result { match v { Value::Object(o) => { let mut wtr = WriterBuilder::new().from_writer(vec![]); @@ -68,13 +79,46 @@ pub fn to_string(v: &Value) -> Result> { for (k, v) in o.entries.iter() { fields.push_back(k.clone()); + values.push_back(to_string_helper(&v)?); } wtr.write_record(fields).expect("can not write."); wtr.write_record(values).expect("can not write."); - return Ok(String::from_utf8(wtr.into_inner()?)?); + return Ok(String::from_utf8( + wtr.into_inner() + .map_err(|_| ShellError::string("Could not convert record"))?, + ) + .map_err(|_| ShellError::string("Could not convert record"))?); + } + Value::List(list) => { + let mut wtr = WriterBuilder::new().from_writer(vec![]); + + let merged_descriptors = merge_descriptors(&list); + wtr.write_record(&merged_descriptors) + .expect("can not write."); + + for l in list { + let mut row = vec![]; + for desc in &merged_descriptors { + match l.item.get_data_by_key(&desc) { + Some(s) => { + row.push(to_string_helper(s)?); + } + None => { + row.push(String::new()); + } + } + } + wtr.write_record(&row).expect("can not write"); + } + + return Ok(String::from_utf8( + wtr.into_inner() + .map_err(|_| ShellError::string("Could not convert record"))?, + ) + .map_err(|_| ShellError::string("Could not convert record"))?); } _ => return to_string_helper(&v), } @@ -85,29 +129,40 @@ fn to_csv( RunnableContext { input, name, .. }: RunnableContext, ) -> Result { let name_span = name; - let out = input; + let stream = async_stream_block! { + let input: Vec> = input.values.collect().await; - Ok(out - .values - .map(move |a| match to_string(&value_to_csv_value(&a.item)) { - Ok(x) => { - let converted = if headerless { - x.lines().skip(1).collect() - } else { - x - }; + let to_process_input = if input.len() > 1 { + let tag = input[0].tag; + vec![Tagged { item: Value::List(input), tag } ] + } else if input.len() == 1 { + input + } else { + vec![] + }; - ReturnSuccess::value( - Value::Primitive(Primitive::String(converted)).simple_spanned(name_span), - ) - } - _ => Err(ShellError::labeled_error_with_secondary( - "Expected an object with CSV-compatible structure from pipeline", - "requires CSV-compatible input", - name_span, - format!("{} originates from here", a.item.type_name()), - a.span(), - )), - }) - .to_output_stream()) + for value in to_process_input { + match to_string(&value_to_csv_value(&value.item)) { + Ok(x) => { + let converted = if headerless { + x.lines().skip(1).collect() + } else { + x + }; + yield ReturnSuccess::value(Value::Primitive(Primitive::String(converted)).simple_spanned(name_span)) + } + _ => { + yield Err(ShellError::labeled_error_with_secondary( + "Expected an object with CSV-compatible structure.span() from pipeline", + "requires CSV-compatible input", + name_span, + "originates from here".to_string(), + value.span(), + )) + } + } + } + }; + + Ok(stream.to_output_stream()) } diff --git a/src/commands/to_json.rs b/src/commands/to_json.rs index 9849e691e3..9af5985cb1 100644 --- a/src/commands/to_json.rs +++ b/src/commands/to_json.rs @@ -80,23 +80,41 @@ fn json_list(input: &Vec>) -> Result, Shell fn to_json(args: CommandArgs, registry: &CommandRegistry) -> Result { let args = args.evaluate_once(registry)?; let name_span = args.name_span(); - let out = args.input; + let stream = async_stream_block! { + let input: Vec> = args.input.values.collect().await; - Ok(out - .values - .map( - move |a| match serde_json::to_string(&value_to_json_value(&a)?) { - Ok(x) => ReturnSuccess::value( - Value::Primitive(Primitive::String(x)).simple_spanned(name_span), - ), - _ => Err(ShellError::labeled_error_with_secondary( + let to_process_input = if input.len() > 1 { + let tag = input[0].tag; + vec![Tagged { item: Value::List(input), tag } ] + } else if input.len() == 1 { + input + } else { + vec![] + }; + + for value in to_process_input { + match value_to_json_value(&value) { + Ok(json_value) => { + match serde_json::to_string(&json_value) { + Ok(x) => yield ReturnSuccess::value( + Value::Primitive(Primitive::String(x)).simple_spanned(name_span), + ), + _ => yield Err(ShellError::labeled_error_with_secondary( + "Expected an object with JSON-compatible structure.span() from pipeline", + "requires JSON-compatible input", + name_span, + "originates from here".to_string(), + value.span(), + )), + } + } + _ => yield Err(ShellError::labeled_error( "Expected an object with JSON-compatible structure from pipeline", "requires JSON-compatible input", - name_span, - format!("{} originates from here", a.item.type_name()), - a.span(), - )), - }, - ) - .to_output_stream()) + name_span)) + } + } + }; + + Ok(stream.to_output_stream()) } diff --git a/src/commands/to_sqlite.rs b/src/commands/to_sqlite.rs index e229057637..25c8acdc84 100644 --- a/src/commands/to_sqlite.rs +++ b/src/commands/to_sqlite.rs @@ -201,19 +201,19 @@ fn to_sqlite(args: CommandArgs, registry: &CommandRegistry) -> Result = args.input.into_vec().await; - match sqlite_input_stream_to_bytes(values) { - Ok(out) => { - yield ReturnSuccess::value(out) - } - Err(_) => { + let input: Vec> = args.input.values.collect().await; + + match sqlite_input_stream_to_bytes(input) { + Ok(out) => yield ReturnSuccess::value(out), + _ => { yield Err(ShellError::labeled_error( - "Expected an object with SQLite-compatible structure from pipeline", + "Expected an object with SQLite-compatible structure.span() from pipeline", "requires SQLite-compatible input", name_span, - )) - } - }; + )) + }, + } }; + Ok(stream.to_output_stream()) } diff --git a/src/commands/to_toml.rs b/src/commands/to_toml.rs index 8c44e21b2c..3960368662 100644 --- a/src/commands/to_toml.rs +++ b/src/commands/to_toml.rs @@ -75,23 +75,41 @@ fn collect_values(input: &Vec>) -> Result, ShellE fn to_toml(args: CommandArgs, registry: &CommandRegistry) -> Result { let args = args.evaluate_once(registry)?; let name_span = args.name_span(); - let out = args.input; + let stream = async_stream_block! { + let input: Vec> = args.input.values.collect().await; - Ok(out - .values - .map(move |a| match toml::to_string(&value_to_toml_value(&a)?) { - Ok(val) => { - return ReturnSuccess::value( - Value::Primitive(Primitive::String(val)).simple_spanned(name_span), - ) + let to_process_input = if input.len() > 1 { + let tag = input[0].tag; + vec![Tagged { item: Value::List(input), tag } ] + } else if input.len() == 1 { + input + } else { + vec![] + }; + + for value in to_process_input { + match value_to_toml_value(&value) { + Ok(toml_value) => { + match toml::to_string(&toml_value) { + Ok(x) => yield ReturnSuccess::value( + Value::Primitive(Primitive::String(x)).simple_spanned(name_span), + ), + _ => yield Err(ShellError::labeled_error_with_secondary( + "Expected an object with TOML-compatible structure.span() from pipeline", + "requires TOML-compatible input", + name_span, + "originates from here".to_string(), + value.span(), + )), + } + } + _ => yield Err(ShellError::labeled_error( + "Expected an object with TOML-compatible structure from pipeline", + "requires TOML-compatible input", + name_span)) } - _ => Err(ShellError::labeled_error_with_secondary( - "Expected an object with TOML-compatible structure from pipeline", - "requires TOML-compatible input", - name_span, - format!("{} originates from here", a.item.type_name()), - a.span(), - )), - }) - .to_output_stream()) + } + }; + + Ok(stream.to_output_stream()) } diff --git a/src/commands/to_tsv.rs b/src/commands/to_tsv.rs index 1a229d768e..d4958b773b 100644 --- a/src/commands/to_tsv.rs +++ b/src/commands/to_tsv.rs @@ -16,8 +16,7 @@ impl WholeStreamCommand for ToTSV { } fn signature(&self) -> Signature { - Signature::build("to-tsv") - .switch("headerless") + Signature::build("to-tsv").switch("headerless") } fn usage(&self) -> &str { @@ -47,7 +46,7 @@ pub fn value_to_tsv_value(v: &Value) -> Value { } } -fn to_string_helper(v: &Value) -> Result> { +fn to_string_helper(v: &Value) -> Result { match v { Value::Primitive(Primitive::Date(d)) => Ok(d.to_string()), Value::Primitive(Primitive::Bytes(b)) => Ok(format!("{}", b)), @@ -55,11 +54,23 @@ fn to_string_helper(v: &Value) -> Result> { Value::List(_) => return Ok(String::from("[list list]")), Value::Object(_) => return Ok(String::from("[object]")), Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()), - _ => return Err("Bad input".into()), + _ => Err(ShellError::string("Unexpected value")), } } -pub fn to_string(v: &Value) -> Result> { +fn merge_descriptors(values: &[Tagged]) -> Vec { + let mut ret = vec![]; + for value in values { + for desc in value.data_descriptors() { + if !ret.contains(&desc) { + ret.push(desc); + } + } + } + ret +} + +pub fn to_string(v: &Value) -> Result { match v { Value::Object(o) => { let mut wtr = WriterBuilder::new().delimiter(b'\t').from_writer(vec![]); @@ -74,7 +85,39 @@ pub fn to_string(v: &Value) -> Result> { wtr.write_record(fields).expect("can not write."); wtr.write_record(values).expect("can not write."); - return Ok(String::from_utf8(wtr.into_inner()?)?); + return Ok(String::from_utf8( + wtr.into_inner() + .map_err(|_| ShellError::string("Could not convert record"))?, + ) + .map_err(|_| ShellError::string("Could not convert record"))?); + } + Value::List(list) => { + let mut wtr = WriterBuilder::new().delimiter(b'\t').from_writer(vec![]); + + let merged_descriptors = merge_descriptors(&list); + wtr.write_record(&merged_descriptors) + .expect("can not write."); + + for l in list { + let mut row = vec![]; + for desc in &merged_descriptors { + match l.item.get_data_by_key(&desc) { + Some(s) => { + row.push(to_string_helper(s)?); + } + None => { + row.push(String::new()); + } + } + } + wtr.write_record(&row).expect("can not write"); + } + + return Ok(String::from_utf8( + wtr.into_inner() + .map_err(|_| ShellError::string("Could not convert record"))?, + ) + .map_err(|_| ShellError::string("Could not convert record"))?); } _ => return to_string_helper(&v), } @@ -85,29 +128,40 @@ fn to_tsv( RunnableContext { input, name, .. }: RunnableContext, ) -> Result { let name_span = name; - let out = input; + let stream = async_stream_block! { + let input: Vec> = input.values.collect().await; - Ok(out - .values - .map(move |a| match to_string(&value_to_tsv_value(&a.item)) { - Ok(x) => { - let converted = if headerless { - x.lines().skip(1).collect() - } else { - x - }; + let to_process_input = if input.len() > 1 { + let tag = input[0].tag; + vec![Tagged { item: Value::List(input), tag } ] + } else if input.len() == 1 { + input + } else { + vec![] + }; - ReturnSuccess::value( - Value::Primitive(Primitive::String(converted)).simple_spanned(name_span), - ) - } - _ => Err(ShellError::labeled_error_with_secondary( - "Expected an object with TSV-compatible structure from pipeline", - "requires TSV-compatible input", - name_span, - format!("{} originates from here", a.item.type_name()), - a.span(), - )), - }) - .to_output_stream()) + for value in to_process_input { + match to_string(&value_to_tsv_value(&value.item)) { + Ok(x) => { + let converted = if headerless { + x.lines().skip(1).collect() + } else { + x + }; + yield ReturnSuccess::value(Value::Primitive(Primitive::String(converted)).simple_spanned(name_span)) + } + _ => { + yield Err(ShellError::labeled_error_with_secondary( + "Expected an object with TSV-compatible structure.span() from pipeline", + "requires TSV-compatible input", + name_span, + "originates from here".to_string(), + value.span(), + )) + } + } + } + }; + + Ok(stream.to_output_stream()) } diff --git a/src/commands/to_yaml.rs b/src/commands/to_yaml.rs index 8ef6e90da2..86995a74a8 100644 --- a/src/commands/to_yaml.rs +++ b/src/commands/to_yaml.rs @@ -76,22 +76,41 @@ pub fn value_to_yaml_value(v: &Tagged) -> Result Result { let args = args.evaluate_once(registry)?; let name_span = args.name_span(); - let out = args.input; - Ok(out - .values - .map( - move |a| match serde_yaml::to_string(&value_to_yaml_value(&a)?) { - Ok(x) => ReturnSuccess::value( - Value::Primitive(Primitive::String(x)).simple_spanned(name_span), - ), - _ => Err(ShellError::labeled_error_with_secondary( + let stream = async_stream_block! { + let input: Vec> = args.input.values.collect().await; + + let to_process_input = if input.len() > 1 { + let tag = input[0].tag; + vec![Tagged { item: Value::List(input), tag } ] + } else if input.len() == 1 { + input + } else { + vec![] + }; + + for value in to_process_input { + match value_to_yaml_value(&value) { + Ok(yaml_value) => { + match serde_yaml::to_string(&yaml_value) { + Ok(x) => yield ReturnSuccess::value( + Value::Primitive(Primitive::String(x)).simple_spanned(name_span), + ), + _ => yield Err(ShellError::labeled_error_with_secondary( + "Expected an object with YAML-compatible structure.span() from pipeline", + "requires YAML-compatible input", + name_span, + "originates from here".to_string(), + value.span(), + )), + } + } + _ => yield Err(ShellError::labeled_error( "Expected an object with YAML-compatible structure from pipeline", "requires YAML-compatible input", - name_span, - format!("{} originates from here", a.item.type_name()), - a.span(), - )), - }, - ) - .to_output_stream()) + name_span)) + } + } + }; + + Ok(stream.to_output_stream()) } diff --git a/tests/filters_test.rs b/tests/filters_test.rs index e3ebcd1a6d..b44ac5c47d 100644 --- a/tests/filters_test.rs +++ b/tests/filters_test.rs @@ -218,8 +218,8 @@ fn converts_structured_table_to_json_text() { | split-column "," name luck | pick name | to-json - | nth 0 | from-json + | nth 0 | get name | echo $it "#