From 75cfee28b28392aee6b210afe8aa9cb7453443ee Mon Sep 17 00:00:00 2001 From: Luccas Mateus Date: Tue, 9 Nov 2021 22:02:33 -0300 Subject: [PATCH] `from yaml` and `from yml` (#322) * MathEval Variance and Stddev * Fix tests and linting * Typo * Deal with streams when they are not tables * `from yaml` and `from yml` `from yaml` and `from yml` from yaml and from yml * Fix collect_string * Fix tests and linting --- Cargo.lock | 46 ++++ crates/nu-command/Cargo.toml | 3 + crates/nu-command/src/default_context.rs | 2 + crates/nu-command/src/formats/from/mod.rs | 3 + crates/nu-command/src/formats/from/yaml.rs | 275 +++++++++++++++++++++ 5 files changed, 329 insertions(+) create mode 100644 crates/nu-command/src/formats/from/yaml.rs diff --git a/Cargo.lock b/Cargo.lock index 9ff264368c..49b8c7eefe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -420,6 +420,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + [[package]] name = "dunce" version = "1.0.2" @@ -491,6 +497,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -514,6 +526,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "indexmap" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "instant" version = "0.1.12" @@ -745,6 +767,7 @@ dependencies = [ "csv", "dialoguer", "glob", + "itertools", "lscolors", "meval", "nu-engine", @@ -756,6 +779,8 @@ dependencies = [ "nu-term-grid", "rand", "rayon", + "serde", + "serde_yaml", "sysinfo", "terminal_size", "thiserror", @@ -1285,6 +1310,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8c608a35705a5d3cdc9fbe403147647ff34b921f8e833e49306df898f9b20af" +dependencies = [ + "dtoa", + "indexmap", + "serde", + "yaml-rust", +] + [[package]] name = "signal-hook" version = "0.3.10" @@ -1631,6 +1668,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "zeroize" version = "1.4.3" diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 78102f0f29..127adee9c5 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -33,6 +33,9 @@ dialoguer = "0.9.0" rayon = "1.5.1" titlecase = "1.1.0" meval = "0.2.0" +serde = { version="1.0.123", features=["derive"] } +serde_yaml = "0.8.16" +itertools = "0.10.0" rand = "0.8" [features] diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index 3d42e28bc6..bae1c62b3d 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -47,6 +47,8 @@ pub fn create_default_context() -> EngineState { From, FromCsv, FromJson, + FromYaml, + FromYml, FromTsv, Get, Griddle, diff --git a/crates/nu-command/src/formats/from/mod.rs b/crates/nu-command/src/formats/from/mod.rs index 713f71d9c4..c7577d74a6 100644 --- a/crates/nu-command/src/formats/from/mod.rs +++ b/crates/nu-command/src/formats/from/mod.rs @@ -3,8 +3,11 @@ mod csv; mod delimited; mod json; mod tsv; +mod yaml; pub use self::csv::FromCsv; pub use command::From; pub use json::FromJson; pub use tsv::FromTsv; +pub use yaml::FromYaml; +pub use yaml::FromYml; diff --git a/crates/nu-command/src/formats/from/yaml.rs b/crates/nu-command/src/formats/from/yaml.rs new file mode 100644 index 0000000000..44ddca0096 --- /dev/null +++ b/crates/nu-command/src/formats/from/yaml.rs @@ -0,0 +1,275 @@ +use itertools::Itertools; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{ + Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Spanned, Value, +}; +use serde::de::Deserialize; +use std::collections::HashMap; + +#[derive(Clone)] +pub struct FromYaml; + +impl Command for FromYaml { + fn name(&self) -> &str { + "from yaml" + } + + fn signature(&self) -> Signature { + Signature::build("from yaml") + } + + fn usage(&self) -> &str { + "Parse text as .yaml/.yml and create table." + } + + fn examples(&self) -> Vec { + vec![ + Example { + example: "'a: 1' | from yaml", + description: "Converts yaml formatted string to table", + result: Some(Value::Record { + cols: vec!["a".to_string()], + vals: vec![Value::Int { + val: 1, + span: Span::unknown(), + }], + span: Span::unknown(), + }), + }, + Example { + example: "'[ a: 1, b: [1, 2] ]' | from yaml", + description: "Converts yaml formatted string to table", + result: Some(Value::List { + vals: vec![ + Value::Record { + cols: vec!["a".to_string()], + vals: vec![Value::test_int(1)], + span: Span::unknown(), + }, + Value::Record { + cols: vec!["b".to_string()], + vals: vec![Value::List { + vals: vec![Value::test_int(1), Value::test_int(2)], + span: Span::unknown(), + }], + span: Span::unknown(), + }, + ], + span: Span::unknown(), + }), + }, + ] + } + + fn run( + &self, + _engine_state: &EngineState, + _stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let head = call.head; + from_yaml(input, head) + } +} + +#[derive(Clone)] +pub struct FromYml; + +impl Command for FromYml { + fn name(&self) -> &str { + "from yml" + } + + fn signature(&self) -> Signature { + Signature::build("from yml") + } + + fn usage(&self) -> &str { + "Parse text as .yaml/.yml and create table." + } + + fn run( + &self, + _engine_state: &EngineState, + _stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let head = call.head; + from_yaml(input, head) + } +} + +fn convert_yaml_value_to_nu_value(v: &serde_yaml::Value, span: Span) -> Result { + let err_not_compatible_number = + ShellError::UnsupportedInput("Expected a compatible number".to_string(), span); + Ok(match v { + serde_yaml::Value::Bool(b) => Value::Bool { val: *b, span }, + serde_yaml::Value::Number(n) if n.is_i64() => Value::Int { + val: n.as_i64().ok_or(err_not_compatible_number)?, + span, + }, + serde_yaml::Value::Number(n) if n.is_f64() => Value::Float { + val: n.as_f64().ok_or(err_not_compatible_number)?, + span, + }, + serde_yaml::Value::String(s) => Value::String { + val: s.to_string(), + span, + }, + serde_yaml::Value::Sequence(a) => { + let result: Result, ShellError> = a + .iter() + .map(|x| convert_yaml_value_to_nu_value(x, span)) + .collect(); + Value::List { + vals: result?, + span, + } + } + serde_yaml::Value::Mapping(t) => { + let mut collected = Spanned { + item: HashMap::new(), + span, + }; + + for (k, v) in t { + // A ShellError that we re-use multiple times in the Mapping scenario + let err_unexpected_map = ShellError::UnsupportedInput( + format!("Unexpected YAML:\nKey: {:?}\nValue: {:?}", k, v), + span, + ); + match (k, v) { + (serde_yaml::Value::String(k), _) => { + collected + .item + .insert(k.clone(), convert_yaml_value_to_nu_value(v, span)?); + } + // Hard-code fix for cases where "v" is a string without quotations with double curly braces + // e.g. k = value + // value: {{ something }} + // Strangely, serde_yaml returns + // "value" -> Mapping(Mapping { map: {Mapping(Mapping { map: {String("something"): Null} }): Null} }) + (serde_yaml::Value::Mapping(m), serde_yaml::Value::Null) => { + return m + .iter() + .take(1) + .collect_vec() + .first() + .and_then(|e| match e { + (serde_yaml::Value::String(s), serde_yaml::Value::Null) => { + Some(Value::String { + val: "{{ ".to_owned() + s + " }}", + span, + }) + } + _ => None, + }) + .ok_or(err_unexpected_map); + } + (_, _) => { + return Err(err_unexpected_map); + } + } + } + + Value::from(collected) + } + serde_yaml::Value::Null => Value::nothing(span), + x => unimplemented!("Unsupported yaml case: {:?}", x), + }) +} + +pub fn from_yaml_string_to_value(s: String, span: Span) -> Result { + let mut documents = vec![]; + + for document in serde_yaml::Deserializer::from_str(&s) { + let v: serde_yaml::Value = serde_yaml::Value::deserialize(document).map_err(|x| { + ShellError::UnsupportedInput(format!("Could not load yaml: {}", x), span) + })?; + documents.push(convert_yaml_value_to_nu_value(&v, span)?); + } + + match documents.len() { + 0 => Ok(Value::nothing(span)), + 1 => Ok(documents.remove(0)), + _ => Ok(Value::List { + vals: documents, + span, + }), + } +} + +fn from_yaml(input: PipelineData, head: Span) -> Result { + let concat_string = input.collect_string(""); + + match from_yaml_string_to_value(concat_string, head) { + Ok(x) => Ok(x.into_pipeline_data()), + Err(other) => Err(other), + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_problematic_yaml() { + struct TestCase { + description: &'static str, + input: &'static str, + expected: Result, + } + let tt: Vec = vec![ + TestCase { + description: "Double Curly Braces With Quotes", + input: r#"value: "{{ something }}""#, + expected: Ok(Value::Record { + cols: vec!["value".to_string()], + vals: vec![Value::String { + val: "{{ something }}".to_string(), + span: Span::unknown(), + }], + span: Span::unknown(), + }), + }, + TestCase { + description: "Double Curly Braces Without Quotes", + input: r#"value: {{ something }}"#, + expected: Ok(Value::Record { + cols: vec!["value".to_string()], + vals: vec![Value::String { + val: "{{ something }}".to_string(), + span: Span::unknown(), + }], + span: Span::unknown(), + }), + }, + ]; + for tc in tt { + let actual = from_yaml_string_to_value(tc.input.to_owned(), Span::unknown()); + if actual.is_err() { + assert!( + tc.expected.is_err(), + "actual is Err for test:\nTest Description {}\nErr: {:?}", + tc.description, + actual + ); + } else { + assert_eq!( + actual.unwrap().into_string(""), + tc.expected.unwrap().into_string("") + ); + } + } + } + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(FromYaml {}) + } +}