From ba483155d7dfb47eb8471cba160b3cc564b13034 Mon Sep 17 00:00:00 2001 From: Lily Mara <6109875+lily-mara@users.noreply.github.com> Date: Fri, 6 Aug 2021 09:46:19 -0700 Subject: [PATCH] Reimplement parsers with nu-serde (#3880) The nu-serde crate allows us to become much more generic with respect to how we convert output to `nu-protocol::Value`s. This allows us to remove a lot of the special-case code that we wrote for deserializing JSON values. Co-authored-by: Darren Schroeder <343840+fdncred@users.noreply.github.com> --- Cargo.lock | 16 ++-- crates/nu-command/Cargo.toml | 2 + .../src/commands/formats/from/ini.rs | 54 ++++++------- .../src/commands/formats/from/json.rs | 79 +++++++++++-------- .../src/commands/formats/from/toml.rs | 51 ++++-------- 5 files changed, 93 insertions(+), 109 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7354d5ce7..7fdbf4a82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2762,7 +2762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" dependencies = [ "serde 1.0.126", - "serde_test 1.0.126", + "serde_test 1.0.127", ] [[package]] @@ -3279,6 +3279,7 @@ dependencies = [ "nu-plugin", "nu-pretty-hex", "nu-protocol", + "nu-serde", "nu-source", "nu-stream", "nu-table", @@ -3316,6 +3317,7 @@ dependencies = [ "term 0.7.0", "term_size", "termcolor", + "thiserror", "titlecase", "toml", "trash", @@ -5488,9 +5490,9 @@ dependencies = [ [[package]] name = "serde_test" -version = "1.0.126" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd1055d1c20532080b9da5040ec8e27425f4d4573d8e29eb19ba4ff1e4b9da2d" +checksum = "de9e52f2f83e2608a121618b6d3885b514613aac702306232c4f035ff60fdb56" dependencies = [ "serde 1.0.126", ] @@ -6105,18 +6107,18 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" [[package]] name = "thiserror" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" +checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" +checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745" dependencies = [ "proc-macro2", "quote 1.0.9", diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 932be5dc0..0c005179a 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -19,6 +19,7 @@ nu-path = { version = "0.35.0", path="../nu-path" } nu-parser = { version = "0.35.0", path="../nu-parser" } nu-plugin = { version = "0.35.0", path="../nu-plugin" } nu-protocol = { version = "0.35.0", path="../nu-protocol" } +nu-serde = { version = "0.35.0", path="../nu-serde" } nu-source = { version = "0.35.0", path="../nu-source" } nu-stream = { version = "0.35.0", path="../nu-stream" } nu-table = { version = "0.35.0", path="../nu-table" } @@ -84,6 +85,7 @@ sha2 = "0.9.3" strip-ansi-escapes = "0.1.0" sxd-document = "0.3.2" sxd-xpath = "0.4.2" +thiserror = "1.0.26" tempfile = "3.2.0" term = { version="0.7.0", optional=true } term_size = "0.3.2" diff --git a/crates/nu-command/src/commands/formats/from/ini.rs b/crates/nu-command/src/commands/formats/from/ini.rs index 48ced9b15..950f8f064 100644 --- a/crates/nu-command/src/commands/formats/from/ini.rs +++ b/crates/nu-command/src/commands/formats/from/ini.rs @@ -1,9 +1,18 @@ use crate::prelude::*; use nu_engine::WholeStreamCommand; use nu_errors::ShellError; -use nu_protocol::{Primitive, Signature, TaggedDictBuilder, UntaggedValue, Value}; +use nu_protocol::{Signature, UntaggedValue, Value}; use std::collections::HashMap; +#[derive(Debug, thiserror::Error)] +pub enum DeserializationError { + #[error("Failed to parse input as INI")] + Ini(#[from] serde_ini::de::Error), + + #[error("Failed to convert to a nushell value")] + Nu(#[from] nu_serde::Error), +} + pub struct FromIni; impl WholeStreamCommand for FromIni { @@ -24,39 +33,13 @@ impl WholeStreamCommand for FromIni { } } -fn convert_ini_second_to_nu_value(v: &HashMap, tag: impl Into) -> Value { - let mut second = TaggedDictBuilder::new(tag); - - for (key, value) in v.iter() { - second.insert_untagged(key.clone(), Primitive::String(value.clone())); - } - - second.into_value() -} - -fn convert_ini_top_to_nu_value( - v: &HashMap>, - tag: impl Into, -) -> Value { - let tag = tag.into(); - let mut top_level = TaggedDictBuilder::new(tag.clone()); - - for (key, value) in v.iter() { - top_level.insert_value( - key.clone(), - convert_ini_second_to_nu_value(value, tag.clone()), - ); - } - - top_level.into_value() -} - pub fn from_ini_string_to_value( s: String, tag: impl Into, -) -> Result { +) -> Result { let v: HashMap> = serde_ini::from_str(&s)?; - Ok(convert_ini_top_to_nu_value(&v, tag)) + + Ok(nu_serde::to_value(v, tag)?) } fn from_ini(args: CommandArgs) -> Result { @@ -72,13 +55,20 @@ fn from_ini(args: CommandArgs) -> Result { } => Ok(list.into_iter().into_output_stream()), x => Ok(OutputStream::one(x)), }, - Err(_) => Err(ShellError::labeled_error_with_secondary( - "Could not parse as INI", + Err(DeserializationError::Ini(e)) => Err(ShellError::labeled_error_with_secondary( + format!("Could not parse as INI: {}", e), "input cannot be parsed as INI", &tag, "value originates from here", concat_string.tag, )), + Err(DeserializationError::Nu(e)) => Err(ShellError::labeled_error_with_secondary( + format!("Could not convert to nushell value: {}", e), + "input cannot be converted to nushell", + &tag, + "value originates from here", + concat_string.tag, + )), } } diff --git a/crates/nu-command/src/commands/formats/from/json.rs b/crates/nu-command/src/commands/formats/from/json.rs index 2341965db..7d0c3f65d 100644 --- a/crates/nu-command/src/commands/formats/from/json.rs +++ b/crates/nu-command/src/commands/formats/from/json.rs @@ -1,7 +1,16 @@ use crate::prelude::*; use nu_engine::WholeStreamCommand; use nu_errors::ShellError; -use nu_protocol::{Primitive, Signature, TaggedDictBuilder, UntaggedValue, Value}; +use nu_protocol::{Signature, UntaggedValue, Value}; + +#[derive(Debug, thiserror::Error)] +pub enum DeserializationError { + #[error("Failed to parse input as JSON")] + Json(#[from] nu_json::Error), + + #[error("Failed to convert JSON to a nushell value")] + Nu(#[from] Box), +} pub struct FromJson; @@ -27,39 +36,13 @@ impl WholeStreamCommand for FromJson { } } -fn convert_json_value_to_nu_value(v: &nu_json::Value, tag: impl Into) -> Value { - let tag = tag.into(); - let span = tag.span; - - match v { - nu_json::Value::Null => UntaggedValue::Primitive(Primitive::Nothing).into_value(&tag), - nu_json::Value::Bool(b) => UntaggedValue::boolean(*b).into_value(&tag), - nu_json::Value::F64(n) => UntaggedValue::decimal_from_float(*n, span).into_value(&tag), - nu_json::Value::U64(n) => UntaggedValue::big_int(*n).into_value(&tag), - nu_json::Value::I64(n) => UntaggedValue::int(*n).into_value(&tag), - nu_json::Value::String(s) => { - UntaggedValue::Primitive(Primitive::String(String::from(s))).into_value(&tag) - } - nu_json::Value::Array(a) => UntaggedValue::Table( - a.iter() - .map(|x| convert_json_value_to_nu_value(x, &tag)) - .collect(), - ) - .into_value(tag), - nu_json::Value::Object(o) => { - let mut collected = TaggedDictBuilder::new(&tag); - for (k, v) in o.iter() { - collected.insert_value(k.clone(), convert_json_value_to_nu_value(v, &tag)); - } - - collected.into_value() - } - } -} - -pub fn from_json_string_to_value(s: String, tag: impl Into) -> nu_json::Result { +pub fn from_json_string_to_value( + s: String, + tag: impl Into, +) -> Result { let v: nu_json::Value = nu_json::from_str(&s)?; - Ok(convert_json_value_to_nu_value(&v, tag)) + + Ok(nu_serde::to_value(v, tag).map_err(Box::new)?) } fn from_json(args: CommandArgs) -> Result { @@ -81,7 +64,19 @@ fn from_json(args: CommandArgs) -> Result { match from_json_string_to_value(json_str, &name_tag) { Ok(x) => Some(x), - Err(e) => { + Err(DeserializationError::Nu(e)) => { + let mut message = "Could not convert JSON to nushell value (".to_string(); + message.push_str(&e.to_string()); + message.push(')'); + Some(Value::error(ShellError::labeled_error_with_secondary( + message, + "input cannot be converted to nushell values", + name_tag.clone(), + "value originates from here", + concat_string.tag.clone(), + ))) + } + Err(DeserializationError::Json(e)) => { let mut message = "Could not parse as JSON (".to_string(); message.push_str(&e.to_string()); message.push(')'); @@ -107,7 +102,7 @@ fn from_json(args: CommandArgs) -> Result { x => Ok(OutputStream::one(x)), }, - Err(e) => { + Err(DeserializationError::Json(e)) => { let mut message = "Could not parse as JSON (".to_string(); message.push_str(&e.to_string()); message.push(')'); @@ -122,6 +117,20 @@ fn from_json(args: CommandArgs) -> Result { ), ))) } + Err(DeserializationError::Nu(e)) => { + let mut message = "Could not convert JSON to nushell value (".to_string(); + message.push_str(&e.to_string()); + message.push(')'); + Ok(OutputStream::one(Value::error( + ShellError::labeled_error_with_secondary( + message, + "input cannot be converted to nushell values", + name_tag, + "value originates from here", + concat_string.tag, + ), + ))) + } } } } diff --git a/crates/nu-command/src/commands/formats/from/toml.rs b/crates/nu-command/src/commands/formats/from/toml.rs index 25c09a85e..f65cd3d98 100644 --- a/crates/nu-command/src/commands/formats/from/toml.rs +++ b/crates/nu-command/src/commands/formats/from/toml.rs @@ -1,7 +1,16 @@ use crate::prelude::*; use nu_engine::WholeStreamCommand; use nu_errors::ShellError; -use nu_protocol::{Primitive, Signature, TaggedDictBuilder, UntaggedValue, Value}; +use nu_protocol::{Signature, UntaggedValue, Value}; + +#[derive(Debug, thiserror::Error)] +pub enum DeserializationError { + #[error("Failed to parse input as TOML")] + Toml(#[from] toml::de::Error), + + #[error("Failed to convert to a nushell value")] + Nu(#[from] Box), +} pub struct FromToml; @@ -23,41 +32,13 @@ impl WholeStreamCommand for FromToml { } } -pub fn convert_toml_value_to_nu_value(v: &toml::Value, tag: impl Into) -> Value { - let tag = tag.into(); - let span = tag.span; - - match v { - toml::Value::Boolean(b) => UntaggedValue::boolean(*b).into_value(tag), - toml::Value::Integer(n) => UntaggedValue::int(*n).into_value(tag), - toml::Value::Float(n) => UntaggedValue::decimal_from_float(*n, span).into_value(tag), - toml::Value::String(s) => { - UntaggedValue::Primitive(Primitive::String(String::from(s))).into_value(tag) - } - toml::Value::Array(a) => UntaggedValue::Table( - a.iter() - .map(|x| convert_toml_value_to_nu_value(x, &tag)) - .collect(), - ) - .into_value(tag), - toml::Value::Datetime(dt) => { - UntaggedValue::Primitive(Primitive::String(dt.to_string())).into_value(tag) - } - toml::Value::Table(t) => { - let mut collected = TaggedDictBuilder::new(&tag); - - for (k, v) in t.iter() { - collected.insert_value(k.clone(), convert_toml_value_to_nu_value(v, &tag)); - } - - collected.into_value() - } - } -} - -pub fn from_toml_string_to_value(s: String, tag: impl Into) -> Result { +pub fn from_toml_string_to_value( + s: String, + tag: impl Into, +) -> Result { let v: toml::Value = s.parse::()?; - Ok(convert_toml_value_to_nu_value(&v, tag)) + + Ok(nu_serde::to_value(v, tag).map_err(Box::new)?) } pub fn from_toml(args: CommandArgs) -> Result {