From b5029545588d5277809354ce81feee78f2367b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20N=2E=20Robalino?= Date: Tue, 27 Aug 2019 06:05:51 -0500 Subject: [PATCH 1/2] [from|to]json additions/refactoring. --- src/commands/from_json.rs | 91 ++++++++++++++++++++++++++------------- tests/filters_test.rs | 79 ++++++++++++++++++++++++++++++--- 2 files changed, 133 insertions(+), 37 deletions(-) diff --git a/src/commands/from_json.rs b/src/commands/from_json.rs index dba9e0209f..ab4241f219 100644 --- a/src/commands/from_json.rs +++ b/src/commands/from_json.rs @@ -5,21 +5,26 @@ use crate::prelude::*; pub struct FromJSON; -impl WholeStreamCommand for FromJSON { - fn run( - &self, - args: CommandArgs, - registry: &CommandRegistry, - ) -> Result { - from_json(args, registry) - } +#[derive(Deserialize)] +pub struct FromJSONArgs { + objects: bool, +} +impl WholeStreamCommand for FromJSON { fn name(&self) -> &str { "from-json" } fn signature(&self) -> Signature { - Signature::build("from-json") + Signature::build("from-json").switch("objects") + } + + fn run( + &self, + args: CommandArgs, + registry: &CommandRegistry, + ) -> Result { + args.process(registry, from_json)?.run() } } @@ -64,10 +69,11 @@ pub fn from_json_string_to_value( Ok(convert_json_value_to_nu_value(&v, tag)) } -fn from_json(args: CommandArgs, registry: &CommandRegistry) -> Result { - let args = args.evaluate_once(registry)?; - let span = args.name_span(); - let input = args.input; +fn from_json( + FromJSONArgs { objects }: FromJSONArgs, + RunnableContext { input, name, .. }: RunnableContext, +) -> Result { + let name_span = name; let stream = async_stream_block! { let values: Vec> = input.values.collect().await; @@ -86,7 +92,7 @@ fn from_json(args: CommandArgs, registry: &CommandRegistry) -> Result yield Err(ShellError::labeled_error_with_secondary( "Expected a string from pipeline", "requires string input", - span, + name_span, "value originates from here", value_tag.span, )), @@ -94,25 +100,50 @@ fn from_json(args: CommandArgs, registry: &CommandRegistry) -> Result - match x { - Tagged { item: Value::List(list), .. } => { - for l in list { - yield ReturnSuccess::value(l); + + if objects { + for json_str in concat_string.lines() { + if json_str.is_empty() { + continue; + } + + match from_json_string_to_value(json_str.to_string(), name_span) { + Ok(x) => + yield ReturnSuccess::value(x), + Err(_) => { + if let Some(last_tag) = latest_tag { + yield Err(ShellError::labeled_error_with_secondary( + "Could nnot parse as JSON", + "input cannot be parsed as JSON", + name_span, + "value originates from here", + last_tag.span)) } } - x => yield ReturnSuccess::value(x), } - Err(_) => if let Some(last_tag) = latest_tag { - yield Err(ShellError::labeled_error_with_secondary( - "Could not parse as JSON", - "input cannot be parsed as JSON", - span, - "value originates from here", - last_tag.span, - )) - } , + } + } else { + match from_json_string_to_value(concat_string, name_span) { + Ok(x) => + match x { + Tagged { item: Value::List(list), .. } => { + for l in list { + yield ReturnSuccess::value(l); + } + } + x => yield ReturnSuccess::value(x), + } + Err(_) => { + if let Some(last_tag) = latest_tag { + yield Err(ShellError::labeled_error_with_secondary( + "Could not parse as JSON", + "input cannot be parsed as JSON", + name_span, + "value originates from here", + last_tag.span)) + } + } + } } }; diff --git a/tests/filters_test.rs b/tests/filters_test.rs index 710f0b89a8..e8ce3db4ad 100644 --- a/tests/filters_test.rs +++ b/tests/filters_test.rs @@ -16,7 +16,7 @@ fn can_convert_table_to_csv_text_and_from_csv_text_back_into_table() { #[test] fn converts_structured_table_to_csv_text() { Playground::setup_for("filter_to_csv_test_1").with_files(vec![FileWithContentToBeTrimmed( - "sample.csv", + "sample.txt", r#" importer,shipper,tariff_item,name,origin Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain @@ -27,7 +27,7 @@ fn converts_structured_table_to_csv_text() { nu!( output, cwd("tests/fixtures/nuplayground/filter_to_csv_test_1"), - "open sample.csv --raw | lines | split-column \",\" a b c d origin | last 1 | to-csv | lines | nth 1 | echo \"$it\"" + r#"open sample.txt | lines | split-column ",s" a b c d origin | last 1 | to-csv | lines | nth 1 | echo "$it""# ); assert!(output.contains("Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia")); @@ -36,7 +36,7 @@ fn converts_structured_table_to_csv_text() { #[test] fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() { Playground::setup_for("filter_to_csv_test_2").with_files(vec![FileWithContentToBeTrimmed( - "sample.csv", + "sample.txt", r#" importer,shipper,tariff_item,name,origin Plasticos Rival,Reverte,2509000000,Calcium carbonate,Spain @@ -47,7 +47,7 @@ fn converts_structured_table_to_csv_text_skipping_headers_after_conversion() { nu!( output, cwd("tests/fixtures/nuplayground/filter_to_csv_test_2"), - "open sample.csv --raw | lines | split-column \",\" a b c d origin | last 1 | to-csv --headerless | echo \"$it\"" + r#"open sample.txt | lines | split-column "," a b c d origin | last 1 | to-csv --headerless | echo "$it""# ); assert!(output.contains("Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia")); @@ -106,6 +106,71 @@ fn can_convert_table_to_json_text_and_from_json_text_back_into_table() { assert_eq!(output, "markup"); } +#[test] +fn converts_from_json_text_to_structured_table() { + Playground::setup_for("filter_from_json_test_1").with_files(vec![FileWithContentToBeTrimmed( + "katz.txt", + r#" + { + "katz": [ + {"name": "Yehuda", "rusty_luck": 1}, + {"name": "Jonathan", "rusty_luck": 1}, + {"name": "Andres", "rusty_luck": 1}, + {"name":"GorbyPuff", "rusty_luck": 1} + ] + } + "#, + )]); + + nu!( + output, + cwd("tests/fixtures/nuplayground/filter_from_json_test_1"), + "open katz.txt | from-json | get katz | get rusty_luck | sum | echo $it" + ); + + assert_eq!(output, "4"); +} + +#[test] +fn converts_from_json_text_recognizing_objects_independendtly_to_structured_table() { + Playground::setup_for("filter_from_json_test_2").with_files(vec![FileWithContentToBeTrimmed( + "katz.txt", + r#" + {"name": "Yehuda", "rusty_luck": 1} + {"name": "Jonathan", "rusty_luck": 1} + {"name": "Andres", "rusty_luck": 1} + {"name":"GorbyPuff", "rusty_luck": 3} + "#, + )]); + + nu!( + output, + cwd("tests/fixtures/nuplayground/filter_from_json_test_2"), + r#"open katz.txt | from-json --objects | where name == "GorbyPuff" | get rusty_luck | echo $it"# + ); + + assert_eq!(output, "3"); +} + +#[test] +fn converts_structured_table_to_json_text() { + Playground::setup_for("filter_to_json_test_1").with_files(vec![FileWithContentToBeTrimmed( + "sample.txt", + r#" + JonAndrehudaTZ,3 + GorbyPuff,100 + "#, + )]); + + nu!( + output, + cwd("tests/fixtures/nuplayground/filter_to_json_test_1"), + r#"open sample.txt | lines | split-column "," name luck | pick name | to-json | nth 0 | from-json | get name | echo $it"# + ); + + assert_eq!(output, "JonAndrehudaTZ"); +} + #[test] fn can_convert_json_text_to_bson_and_back_into_table() { nu!( @@ -144,7 +209,7 @@ fn can_sort_by_column() { nu!( output, cwd("tests/fixtures/formats"), - "open cargo_sample.toml --raw | lines | skip 1 | first 4 | split-column \"=\" | sort-by Column1 | skip 1 | first 1 | get Column1 | trim | echo $it" + r#"open cargo_sample.toml --raw | lines | skip 1 | first 4 | split-column "=" | sort-by Column1 | skip 1 | first 1 | get Column1 | trim | echo $it"# ); assert_eq!(output, "description"); @@ -155,7 +220,7 @@ fn can_sort_by_column_reverse() { nu!( output, cwd("tests/fixtures/formats"), - "open cargo_sample.toml --raw | lines | skip 1 | first 4 | split-column \"=\" | sort-by Column1 --reverse | skip 1 | first 1 | get Column1 | trim | echo $it" + r#"open cargo_sample.toml --raw | lines | skip 1 | first 4 | split-column "=" | sort-by Column1 --reverse | skip 1 | first 1 | get Column1 | trim | echo $it"# ); assert_eq!(output, "name"); @@ -166,7 +231,7 @@ fn can_split_by_column() { nu!( output, cwd("tests/fixtures/formats"), - "open cargo_sample.toml --raw | lines | skip 1 | first 1 | split-column \"=\" | get Column1 | trim | echo $it" + r#"open cargo_sample.toml --raw | lines | skip 1 | first 1 | split-column "=" | get Column1 | trim | echo $it"# ); assert_eq!(output, "name"); From 162c8b42745d41812e0ad597c373621232771e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20N=2E=20Robalino?= Date: Tue, 27 Aug 2019 06:14:19 -0500 Subject: [PATCH 2/2] swept clean quotes --- tests/command_ls_tests.rs | 6 +++--- tests/command_open_tests.rs | 2 +- tests/command_rm_tests.rs | 2 +- tests/commands_test.rs | 2 +- tests/external_tests.rs | 2 -- tests/filter_str_tests.rs | 4 ++-- tests/filters_test.rs | 2 +- tests/tests.rs | 4 ++-- 8 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/command_ls_tests.rs b/tests/command_ls_tests.rs index 6223709805..20f299764a 100644 --- a/tests/command_ls_tests.rs +++ b/tests/command_ls_tests.rs @@ -18,7 +18,7 @@ fn ls_lists_regular_files() { nu!( output, cwd(&full_path), - r#"ls | get name | lines | split-column "." | get Column2 | str Column2 --to-int | sum | echo $it"# + r#"ls | get name | lines | split-column "." | get Column2 | str --to-int | sum | echo $it"# ); assert_eq!(output, "30"); @@ -40,7 +40,7 @@ fn ls_lists_regular_files_using_asterisk_wildcard() { nu!( output, cwd(&full_path), - "ls *.txt | get name | lines| split-column \".\" | get Column2 | str Column2 --to-int | sum | echo $it" + r#"ls *.txt | get name | lines| split-column "." | get Column2 | str --to-int | sum | echo $it"# ); assert_eq!(output, "3"); @@ -62,7 +62,7 @@ fn ls_lists_regular_files_using_question_mark_wildcard() { nu!( output, cwd(&full_path), - "ls *.??.txt | get name | lines| split-column \".\" | get Column2 | str Column2 --to-int | sum | echo $it" + r#"ls *.??.txt | get name | lines| split-column "." | get Column2 | str --to-int | sum | echo $it"# ); assert_eq!(output, "30"); diff --git a/tests/command_open_tests.rs b/tests/command_open_tests.rs index 3f6da5adf5..d962d3e88d 100644 --- a/tests/command_open_tests.rs +++ b/tests/command_open_tests.rs @@ -17,7 +17,7 @@ fn recognizes_csv() { nu!( output, cwd("tests/fixtures/nuplayground/open_recognizes_csv_test"), - "open nu.zion.csv | where author == \"Andres N. Robalino\" | get source | echo $it" + r#"open nu.zion.csv | where author == "Andres N. Robalino" | get source | echo $it"# ); assert_eq!(output, "Ecuador"); diff --git a/tests/command_rm_tests.rs b/tests/command_rm_tests.rs index 76e7128208..126465a2da 100644 --- a/tests/command_rm_tests.rs +++ b/tests/command_rm_tests.rs @@ -51,7 +51,7 @@ fn rm_removes_files_with_wildcard() { nu!( _output, cwd("tests/fixtures/nuplayground/rm_wildcard_test_1"), - "rm \"src/*/*/*.rs\"" + r#"rm "src/*/*/*.rs""# ); assert!(!h::files_exist_at( diff --git a/tests/commands_test.rs b/tests/commands_test.rs index 6cdc4d7940..1c8ddd25d8 100644 --- a/tests/commands_test.rs +++ b/tests/commands_test.rs @@ -7,7 +7,7 @@ use helpers as h; fn lines() { nu!(output, cwd("tests/fixtures/formats"), - "open cargo_sample.toml --raw | lines | skip-while $it != \"[dependencies]\" | skip 1 | first 1 | split-column \"=\" | get Column1 | trim | echo $it" + r#"open cargo_sample.toml --raw | lines | skip-while $it != "[dependencies]" | skip 1 | first 1 | split-column "=" | get Column1 | trim | echo $it"# ); assert_eq!(output, "rustyline"); diff --git a/tests/external_tests.rs b/tests/external_tests.rs index 5db04dc27d..ba62c5e127 100644 --- a/tests/external_tests.rs +++ b/tests/external_tests.rs @@ -4,8 +4,6 @@ use helpers::in_directory as cwd; #[test] fn external_command() { - // Echo should exist on all currently supported platforms. A better approach might - // be to generate a dummy executable as part of the tests with known semantics. nu!(output, cwd("tests/fixtures"), "echo 1"); assert!(output.contains("1")); diff --git a/tests/filter_str_tests.rs b/tests/filter_str_tests.rs index dd51080c6e..630c144c87 100644 --- a/tests/filter_str_tests.rs +++ b/tests/filter_str_tests.rs @@ -119,7 +119,7 @@ fn find_and_replaces() { nu!( output, cwd("tests/fixtures/nuplayground/plugin_str_find_and_replaces_test"), - "open sample.toml | str fortune.teller.phone --find-replace KATZ \"5289\" | get fortune.teller.phone | echo $it" + r#"open sample.toml | str fortune.teller.phone --find-replace KATZ "5289" | get fortune.teller.phone | echo $it"# ); assert_eq!(output, "1-800-5289"); @@ -140,7 +140,7 @@ fn find_and_replaces_without_passing_field() { nu!( output, cwd("tests/fixtures/nuplayground/plugin_str_find_and_replaces_without_passing_field_test"), - "open sample.toml | get fortune.teller.phone | str --find-replace KATZ \"5289\" | echo $it" + r#"open sample.toml | get fortune.teller.phone | str --find-replace KATZ "5289" | echo $it"# ); assert_eq!(output, "1-800-5289"); diff --git a/tests/filters_test.rs b/tests/filters_test.rs index e8ce3db4ad..d8976c1cc1 100644 --- a/tests/filters_test.rs +++ b/tests/filters_test.rs @@ -27,7 +27,7 @@ fn converts_structured_table_to_csv_text() { nu!( output, cwd("tests/fixtures/nuplayground/filter_to_csv_test_1"), - r#"open sample.txt | lines | split-column ",s" a b c d origin | last 1 | to-csv | lines | nth 1 | echo "$it""# + r#"open sample.txt | lines | split-column "," a b c d origin | last 1 | to-csv | lines | nth 1 | echo "$it""# ); assert!(output.contains("Tigre Ecuador,OMYA Andina,3824909999,Calcium carbonate,Colombia")); diff --git a/tests/tests.rs b/tests/tests.rs index cc20c972ff..2090e8e044 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -27,7 +27,7 @@ fn external_has_correct_quotes() { fn add_plugin() { nu!(output, cwd("tests/fixtures/formats"), - "open cargo_sample.toml | add dev-dependencies.newdep \"1\" | get dev-dependencies.newdep | echo $it" + r#"open cargo_sample.toml | add dev-dependencies.newdep "1" | get dev-dependencies.newdep | echo $it"# ); assert_eq!(output, "1"); @@ -37,7 +37,7 @@ fn add_plugin() { fn edit_plugin() { nu!(output, cwd("tests/fixtures/formats"), - "open cargo_sample.toml | edit dev-dependencies.pretty_assertions \"7\" | get dev-dependencies.pretty_assertions | echo $it" + r#"open cargo_sample.toml | edit dev-dependencies.pretty_assertions "7" | get dev-dependencies.pretty_assertions | echo $it"# ); assert_eq!(output, "7");