From 8237d1568306f190831034046e3cb8ff8897f164 Mon Sep 17 00:00:00 2001 From: dannou812 <93219882+dannou812@users.noreply.github.com> Date: Wed, 20 Mar 2024 22:14:31 +0100 Subject: [PATCH] `to json -r` not removing whitespaces fix (#11948) fixes #11900 # Description Use `serde_json` instead. # User-Facing Changes The problem described in the issue now no longer persists. No whitespace in the output of `to json --raw` Output of unicode escape changed to consistent `\uffff` # Tests + Formatting I corrected all Tests that were affected by this change. --- Cargo.lock | 5 +++-- crates/nu-command/tests/commands/cal.rs | 6 +++--- crates/nu-command/tests/commands/each.rs | 2 +- crates/nu-command/tests/commands/find.rs | 6 +++--- crates/nu-command/tests/commands/math/mod.rs | 4 ++-- crates/nu-command/tests/commands/sort_by.rs | 8 ++++---- crates/nu-json/Cargo.toml | 1 + crates/nu-json/src/ser.rs | 9 +++++---- crates/nu-std/tests/test_formats.nu | 16 ++++++++-------- src/tests/test_converters.rs | 16 ++++++++++++---- src/tests/test_strings.rs | 2 +- src/tests/test_table_operations.rs | 2 +- tests/shell/pipeline/commands/internal.rs | 2 +- 13 files changed, 45 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c2be22f2c4..cc252302df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2994,6 +2994,7 @@ dependencies = [ "linked-hash-map", "num-traits", "serde", + "serde_json", ] [[package]] @@ -4946,9 +4947,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.112" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d1bd37ce2324cf3bf85e5a25f96eb4baf0d5aa6eba43e7ae8958870c4ec48ed" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ "indexmap", "itoa", diff --git a/crates/nu-command/tests/commands/cal.rs b/crates/nu-command/tests/commands/cal.rs index 81c4119a42..651ab8d3cd 100644 --- a/crates/nu-command/tests/commands/cal.rs +++ b/crates/nu-command/tests/commands/cal.rs @@ -5,7 +5,7 @@ fn cal_full_year() { let actual = nu!("cal -y --full-year 2010 | first | to json -r"); let first_week_2010_json = - r#"{"year": 2010,"su": null,"mo": null,"tu": null,"we": null,"th": null,"fr": 1,"sa": 2}"#; + r#"{"year":2010,"su":null,"mo":null,"tu":null,"we":null,"th":null,"fr":1,"sa":2}"#; assert_eq!(actual.out, first_week_2010_json); } @@ -18,7 +18,7 @@ fn cal_february_2020_leap_year() { "# )); - let cal_february_json = r#"[{"year": 2020,"month": "february","su": null,"mo": null,"tu": null,"we": null,"th": null,"fr": null,"sa": 1},{"year": 2020,"month": "february","su": 2,"mo": 3,"tu": 4,"we": 5,"th": 6,"fr": 7,"sa": 8},{"year": 2020,"month": "february","su": 9,"mo": 10,"tu": 11,"we": 12,"th": 13,"fr": 14,"sa": 15},{"year": 2020,"month": "february","su": 16,"mo": 17,"tu": 18,"we": 19,"th": 20,"fr": 21,"sa": 22},{"year": 2020,"month": "february","su": 23,"mo": 24,"tu": 25,"we": 26,"th": 27,"fr": 28,"sa": 29}]"#; + let cal_february_json = r#"[{"year":2020,"month":"february","su":null,"mo":null,"tu":null,"we":null,"th":null,"fr":null,"sa":1},{"year":2020,"month":"february","su":2,"mo":3,"tu":4,"we":5,"th":6,"fr":7,"sa":8},{"year":2020,"month":"february","su":9,"mo":10,"tu":11,"we":12,"th":13,"fr":14,"sa":15},{"year":2020,"month":"february","su":16,"mo":17,"tu":18,"we":19,"th":20,"fr":21,"sa":22},{"year":2020,"month":"february","su":23,"mo":24,"tu":25,"we":26,"th":27,"fr":28,"sa":29}]"#; assert_eq!(actual.out, cal_february_json); } @@ -53,7 +53,7 @@ fn cal_week_day_start_mo() { "# )); - let cal_january_json = r#"[{"month": "january","mo": null,"tu": null,"we": 1,"th": 2,"fr": 3,"sa": 4,"su": 5},{"month": "january","mo": 6,"tu": 7,"we": 8,"th": 9,"fr": 10,"sa": 11,"su": 12},{"month": "january","mo": 13,"tu": 14,"we": 15,"th": 16,"fr": 17,"sa": 18,"su": 19},{"month": "january","mo": 20,"tu": 21,"we": 22,"th": 23,"fr": 24,"sa": 25,"su": 26},{"month": "january","mo": 27,"tu": 28,"we": 29,"th": 30,"fr": 31,"sa": null,"su": null}]"#; + let cal_january_json = r#"[{"month":"january","mo":null,"tu":null,"we":1,"th":2,"fr":3,"sa":4,"su":5},{"month":"january","mo":6,"tu":7,"we":8,"th":9,"fr":10,"sa":11,"su":12},{"month":"january","mo":13,"tu":14,"we":15,"th":16,"fr":17,"sa":18,"su":19},{"month":"january","mo":20,"tu":21,"we":22,"th":23,"fr":24,"sa":25,"su":26},{"month":"january","mo":27,"tu":28,"we":29,"th":30,"fr":31,"sa":null,"su":null}]"#; assert_eq!(actual.out, cal_january_json); } diff --git a/crates/nu-command/tests/commands/each.rs b/crates/nu-command/tests/commands/each.rs index 599597e66c..663e07a9f4 100644 --- a/crates/nu-command/tests/commands/each.rs +++ b/crates/nu-command/tests/commands/each.rs @@ -32,7 +32,7 @@ fn each_window_stride() { fn each_no_args_in_block() { let actual = nu!("echo [[foo bar]; [a b] [c d] [e f]] | each {|i| $i | to json -r } | get 1"); - assert_eq!(actual.out, r#"{"foo": "c","bar": "d"}"#); + assert_eq!(actual.out, r#"{"foo":"c","bar":"d"}"#); } #[test] diff --git a/crates/nu-command/tests/commands/find.rs b/crates/nu-command/tests/commands/find.rs index a1c4a208a8..ed811f2a57 100644 --- a/crates/nu-command/tests/commands/find.rs +++ b/crates/nu-command/tests/commands/find.rs @@ -14,7 +14,7 @@ fn find_with_list_search_with_string() { fn find_with_list_search_with_char() { let actual = nu!("[moe larry curly] | find l | to json -r"); - assert_eq!(actual.out, "[\"\u{1b}[37m\u{1b}[0m\u{1b}[41;37ml\u{1b}[0m\u{1b}[37marry\u{1b}[0m\",\"\u{1b}[37mcur\u{1b}[0m\u{1b}[41;37ml\u{1b}[0m\u{1b}[37my\u{1b}[0m\"]"); + assert_eq!(actual.out, "[\"\\u001b[37m\\u001b[0m\\u001b[41;37ml\\u001b[0m\\u001b[37marry\\u001b[0m\",\"\\u001b[37mcur\\u001b[0m\\u001b[41;37ml\\u001b[0m\\u001b[37my\\u001b[0m\"]"); } #[test] @@ -48,7 +48,7 @@ fn find_with_filepath_search_with_string() { assert_eq!( actual.out, - "[\"\u{1b}[37m\u{1b}[0m\u{1b}[41;37marep\u{1b}[0m\u{1b}[37mas.clu\u{1b}[0m\"]" + "[\"\\u001b[37m\\u001b[0m\\u001b[41;37marep\\u001b[0m\\u001b[37mas.clu\\u001b[0m\"]" ); } @@ -57,7 +57,7 @@ fn find_with_filepath_search_with_multiple_patterns() { let actual = nu!(r#"["amigos.txt","arepas.clu","los.txt","tres.txt"] | find arep ami | to json -r"#); - assert_eq!(actual.out, "[\"\u{1b}[37m\u{1b}[0m\u{1b}[41;37mami\u{1b}[0m\u{1b}[37mgos.txt\u{1b}[0m\",\"\u{1b}[37m\u{1b}[0m\u{1b}[41;37marep\u{1b}[0m\u{1b}[37mas.clu\u{1b}[0m\"]"); + assert_eq!(actual.out, "[\"\\u001b[37m\\u001b[0m\\u001b[41;37mami\\u001b[0m\\u001b[37mgos.txt\\u001b[0m\",\"\\u001b[37m\\u001b[0m\\u001b[41;37marep\\u001b[0m\\u001b[37mas.clu\\u001b[0m\"]"); } #[test] diff --git a/crates/nu-command/tests/commands/math/mod.rs b/crates/nu-command/tests/commands/math/mod.rs index e9620388a4..3b120f730a 100644 --- a/crates/nu-command/tests/commands/math/mod.rs +++ b/crates/nu-command/tests/commands/math/mod.rs @@ -420,7 +420,7 @@ fn compound_where() { "# )); - assert_eq!(actual.out, r#"[{"a": 2,"b": 1}]"#); + assert_eq!(actual.out, r#"[{"a":2,"b":1}]"#); } #[test] @@ -431,7 +431,7 @@ fn compound_where_paren() { "# )); - assert_eq!(actual.out, r#"[{"a": 2,"b": 1},{"a": 2,"b": 2}]"#); + assert_eq!(actual.out, r#"[{"a":2,"b":1},{"a":2,"b":2}]"#); } // TODO: these ++ tests are not really testing *math* functionality, maybe find another place for them diff --git a/crates/nu-command/tests/commands/sort_by.rs b/crates/nu-command/tests/commands/sort_by.rs index 1adeed1f46..a79d57e622 100644 --- a/crates/nu-command/tests/commands/sort_by.rs +++ b/crates/nu-command/tests/commands/sort_by.rs @@ -62,7 +62,7 @@ fn ls_sort_by_name_sensitive() { " )); - let json_output = r#"[{"name": "B.txt"},{"name": "C"},{"name": "a.txt"}]"#; + let json_output = r#"[{"name":"B.txt"},{"name":"C"},{"name":"a.txt"}]"#; assert_eq!(actual.out, json_output); } @@ -79,7 +79,7 @@ fn ls_sort_by_name_insensitive() { " )); - let json_output = r#"[{"name": "a.txt"},{"name": "B.txt"},{"name": "C"}]"#; + let json_output = r#"[{"name":"a.txt"},{"name":"B.txt"},{"name":"C"}]"#; assert_eq!(actual.out, json_output); } @@ -95,7 +95,7 @@ fn ls_sort_by_type_name_sensitive() { " )); - let json_output = r#"[{"name": "C","type": "Dir"},{"name": "B.txt","type": "File"},{"name": "a.txt","type": "File"}]"#; + let json_output = r#"[{"name":"C","type":"Dir"},{"name":"B.txt","type":"File"},{"name":"a.txt","type":"File"}]"#; assert_eq!(actual.out, json_output); } @@ -111,7 +111,7 @@ fn ls_sort_by_type_name_insensitive() { " )); - let json_output = r#"[{"name": "C","type": "Dir"},{"name": "a.txt","type": "File"},{"name": "B.txt","type": "File"}]"#; + let json_output = r#"[{"name":"C","type":"Dir"},{"name":"a.txt","type":"File"},{"name":"B.txt","type":"File"}]"#; assert_eq!(actual.out, json_output); } diff --git a/crates/nu-json/Cargo.toml b/crates/nu-json/Cargo.toml index d0da473c17..bdd3bfd307 100644 --- a/crates/nu-json/Cargo.toml +++ b/crates/nu-json/Cargo.toml @@ -20,6 +20,7 @@ default = ["preserve_order"] linked-hash-map = { version = "0.5", optional = true } num-traits = "0.2" serde = "1.0" +serde_json = "1.0.114" [dev-dependencies] # nu-path = { path="../nu-path", version = "0.91.1" } diff --git a/crates/nu-json/src/ser.rs b/crates/nu-json/src/ser.rs index 128a294e2f..cf345a1f64 100644 --- a/crates/nu-json/src/ser.rs +++ b/crates/nu-json/src/ser.rs @@ -1032,8 +1032,9 @@ pub fn to_string_raw(value: &T) -> Result where T: ser::Serialize, { - let vec = to_vec(value)?; - let string = String::from_utf8(vec)?; - let output = string.lines().map(str::trim).collect(); - Ok(output) + let result = serde_json::to_string(value); + match result { + Ok(result_string) => Ok(result_string), + Err(error) => Err(Error::Io(std::io::Error::from(error))), + } } diff --git a/crates/nu-std/tests/test_formats.nu b/crates/nu-std/tests/test_formats.nu index d4dcce3ddf..59e7ddd94c 100644 --- a/crates/nu-std/tests/test_formats.nu +++ b/crates/nu-std/tests/test_formats.nu @@ -2,12 +2,12 @@ use std assert def test_data_multiline [] { let lines = [ - "{\"a\": 1}", - "{\"a\": 2}", - "{\"a\": 3}", - "{\"a\": 4}", - "{\"a\": 5}", - "{\"a\": 6}", + "{\"a\":1}", + "{\"a\":2}", + "{\"a\":3}", + "{\"a\":4}", + "{\"a\":5}", + "{\"a\":6}", ] if $nu.os-info.name == "windows" { @@ -73,7 +73,7 @@ def to_ndjson_multiple_objects [] { def to_ndjson_single_object [] { use std formats * let result = [{a:1}] | to ndjson | str trim - let expect = "{\"a\": 1}" + let expect = "{\"a\":1}" assert equal $result $expect "could not convert to NDJSON" } @@ -89,6 +89,6 @@ def to_jsonl_multiple_objects [] { def to_jsonl_single_object [] { use std formats * let result = [{a:1}] | to jsonl | str trim - let expect = "{\"a\": 1}" + let expect = "{\"a\":1}" assert equal $result $expect "could not convert to JSONL" } diff --git a/src/tests/test_converters.rs b/src/tests/test_converters.rs index 741e3ba7f3..c41868539e 100644 --- a/src/tests/test_converters.rs +++ b/src/tests/test_converters.rs @@ -18,7 +18,7 @@ fn from_json_2() -> TestResult { fn to_json_raw_flag_1() -> TestResult { run_test( "[[a b]; [jim susie] [3 4]] | to json -r", - r#"[{"a": "jim","b": "susie"},{"a": 3,"b": 4}]"#, + r#"[{"a":"jim","b":"susie"},{"a":3,"b":4}]"#, ) } @@ -26,7 +26,7 @@ fn to_json_raw_flag_1() -> TestResult { fn to_json_raw_flag_2() -> TestResult { run_test( "[[\"a b\" c]; [jim susie] [3 4]] | to json -r", - r#"[{"a b": "jim","c": "susie"},{"a b": 3,"c": 4}]"#, + r#"[{"a b":"jim","c":"susie"},{"a b":3,"c":4}]"#, ) } @@ -34,7 +34,7 @@ fn to_json_raw_flag_2() -> TestResult { fn to_json_raw_flag_3() -> TestResult { run_test( "[[\"a b\" \"c d\"]; [\"jim smith\" \"susie roberts\"] [3 4]] | to json -r", - r#"[{"a b": "jim smith","c d": "susie roberts"},{"a b": 3,"c d": 4}]"#, + r#"[{"a b":"jim smith","c d":"susie roberts"},{"a b":3,"c d":4}]"#, ) } @@ -42,6 +42,14 @@ fn to_json_raw_flag_3() -> TestResult { fn to_json_escaped() -> TestResult { run_test( r#"{foo: {bar: '[{"a":"b","c": 2}]'}} | to json --raw"#, - r#"{"foo":{"bar": "[{\"a\":\"b\",\"c\": 2}]"}}"#, + r#"{"foo":{"bar":"[{\"a\":\"b\",\"c\": 2}]"}}"#, + ) +} + +#[test] +fn to_json_raw_backslash_in_quotes() -> TestResult { + run_test( + r#"{a: '\', b: 'some text'} | to json -r"#, + r#"{"a":"\\","b":"some text"}"#, ) } diff --git a/src/tests/test_strings.rs b/src/tests/test_strings.rs index 4ba9dddc27..762ac7579f 100644 --- a/src/tests/test_strings.rs +++ b/src/tests/test_strings.rs @@ -68,6 +68,6 @@ fn case_insensitive_sort() -> TestResult { fn case_insensitive_sort_columns() -> TestResult { run_test( r#"[[version, package]; ["two", "Abc"], ["three", "abc"], ["four", "abc"]] | sort-by -i package version | to json --raw"#, - r#"[{"version": "four","package": "abc"},{"version": "three","package": "abc"},{"version": "two","package": "Abc"}]"#, + r#"[{"version":"four","package":"abc"},{"version":"three","package":"abc"},{"version":"two","package":"Abc"}]"#, ) } diff --git a/src/tests/test_table_operations.rs b/src/tests/test_table_operations.rs index 43c68f423f..9971261190 100644 --- a/src/tests/test_table_operations.rs +++ b/src/tests/test_table_operations.rs @@ -132,7 +132,7 @@ fn command_filter_reject_3() -> TestResult { fn command_filter_reject_4() -> TestResult { run_test( "[[lang, gems, grade]; [nu, 100, a]] | reject gems | to json -r", - r#"[{"lang": "nu","grade": "a"}]"#, + r#"[{"lang":"nu","grade":"a"}]"#, ) } diff --git a/tests/shell/pipeline/commands/internal.rs b/tests/shell/pipeline/commands/internal.rs index b0adca4ef3..dbc4b0a318 100644 --- a/tests/shell/pipeline/commands/internal.rs +++ b/tests/shell/pipeline/commands/internal.rs @@ -607,7 +607,7 @@ fn index_row() { let foo = [[name]; [joe] [bob]]; echo $foo.1 | to json --raw "); - assert_eq!(actual.out, r#"{"name": "bob"}"#); + assert_eq!(actual.out, r#"{"name":"bob"}"#); } #[test]