from yaml and from yml (#322)

* MathEval Variance and Stddev

* Fix tests and linting

* Typo

* Deal with streams when they are not tables

* `from yaml` and `from yml`

`from yaml` and `from yml`

from yaml and from yml

* Fix collect_string

* Fix tests and linting

Cargo.lock

@@ -420,6 +420,12 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
 
+[[package]]
+name = "dtoa"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
+
 [[package]]
 name = "dunce"
 version = "1.0.2"
@@ -491,6 +497,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.19"
@@ -514,6 +526,16 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "indexmap"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -745,6 +767,7 @@ dependencies = [
  "csv",
  "dialoguer",
  "glob",
+ "itertools",
  "lscolors",
  "meval",
  "nu-engine",
@@ -756,6 +779,8 @@ dependencies = [
  "nu-term-grid",
  "rand",
  "rayon",
+ "serde",
+ "serde_yaml",
  "sysinfo",
  "terminal_size",
  "thiserror",
@@ -1285,6 +1310,18 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_yaml"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8c608a35705a5d3cdc9fbe403147647ff34b921f8e833e49306df898f9b20af"
+dependencies = [
+ "dtoa",
+ "indexmap",
+ "serde",
+ "yaml-rust",
+]
+
 [[package]]
 name = "signal-hook"
 version = "0.3.10"
@@ -1631,6 +1668,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "yaml-rust"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
+dependencies = [
+ "linked-hash-map",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.4.3"

crates/nu-command/Cargo.toml

@@ -33,6 +33,9 @@ dialoguer = "0.9.0"
 rayon = "1.5.1"
 titlecase = "1.1.0"
 meval = "0.2.0"
+serde = { version="1.0.123", features=["derive"] }
+serde_yaml = "0.8.16"
+itertools = "0.10.0"
 rand = "0.8"
 
 [features]

crates/nu-command/src/default_context.rs

@@ -47,6 +47,8 @@ pub fn create_default_context() -> EngineState {
             From,
             FromCsv,
             FromJson,
+            FromYaml,
+            FromYml,
             FromTsv,
             Get,
             Griddle,

crates/nu-command/src/formats/from/mod.rs

@@ -3,8 +3,11 @@ mod csv;
 mod delimited;
 mod json;
 mod tsv;
+mod yaml;
 
 pub use self::csv::FromCsv;
 pub use command::From;
 pub use json::FromJson;
 pub use tsv::FromTsv;
+pub use yaml::FromYaml;
+pub use yaml::FromYml;

crates/nu-command/src/formats/from/yaml.rs

@@ -0,0 +1,275 @@
+use itertools::Itertools;
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Spanned, Value,
+};
+use serde::de::Deserialize;
+use std::collections::HashMap;
+
+#[derive(Clone)]
+pub struct FromYaml;
+
+impl Command for FromYaml {
+    fn name(&self) -> &str {
+        "from yaml"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from yaml")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .yaml/.yml and create table."
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                example: "'a: 1' | from yaml",
+                description: "Converts yaml formatted string to table",
+                result: Some(Value::Record {
+                    cols: vec!["a".to_string()],
+                    vals: vec![Value::Int {
+                        val: 1,
+                        span: Span::unknown(),
+                    }],
+                    span: Span::unknown(),
+                }),
+            },
+            Example {
+                example: "'[ a: 1, b: [1, 2] ]' | from yaml",
+                description: "Converts yaml formatted string to table",
+                result: Some(Value::List {
+                    vals: vec![
+                        Value::Record {
+                            cols: vec!["a".to_string()],
+                            vals: vec![Value::test_int(1)],
+                            span: Span::unknown(),
+                        },
+                        Value::Record {
+                            cols: vec!["b".to_string()],
+                            vals: vec![Value::List {
+                                vals: vec![Value::test_int(1), Value::test_int(2)],
+                                span: Span::unknown(),
+                            }],
+                            span: Span::unknown(),
+                        },
+                    ],
+                    span: Span::unknown(),
+                }),
+            },
+        ]
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<nu_protocol::PipelineData, ShellError> {
+        let head = call.head;
+        from_yaml(input, head)
+    }
+}
+
+#[derive(Clone)]
+pub struct FromYml;
+
+impl Command for FromYml {
+    fn name(&self) -> &str {
+        "from yml"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("from yml")
+    }
+
+    fn usage(&self) -> &str {
+        "Parse text as .yaml/.yml and create table."
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<nu_protocol::PipelineData, ShellError> {
+        let head = call.head;
+        from_yaml(input, head)
+    }
+}
+
+fn convert_yaml_value_to_nu_value(v: &serde_yaml::Value, span: Span) -> Result<Value, ShellError> {
+    let err_not_compatible_number =
+        ShellError::UnsupportedInput("Expected a compatible number".to_string(), span);
+    Ok(match v {
+        serde_yaml::Value::Bool(b) => Value::Bool { val: *b, span },
+        serde_yaml::Value::Number(n) if n.is_i64() => Value::Int {
+            val: n.as_i64().ok_or(err_not_compatible_number)?,
+            span,
+        },
+        serde_yaml::Value::Number(n) if n.is_f64() => Value::Float {
+            val: n.as_f64().ok_or(err_not_compatible_number)?,
+            span,
+        },
+        serde_yaml::Value::String(s) => Value::String {
+            val: s.to_string(),
+            span,
+        },
+        serde_yaml::Value::Sequence(a) => {
+            let result: Result<Vec<Value>, ShellError> = a
+                .iter()
+                .map(|x| convert_yaml_value_to_nu_value(x, span))
+                .collect();
+            Value::List {
+                vals: result?,
+                span,
+            }
+        }
+        serde_yaml::Value::Mapping(t) => {
+            let mut collected = Spanned {
+                item: HashMap::new(),
+                span,
+            };
+
+            for (k, v) in t {
+                // A ShellError that we re-use multiple times in the Mapping scenario
+                let err_unexpected_map = ShellError::UnsupportedInput(
+                    format!("Unexpected YAML:\nKey: {:?}\nValue: {:?}", k, v),
+                    span,
+                );
+                match (k, v) {
+                    (serde_yaml::Value::String(k), _) => {
+                        collected
+                            .item
+                            .insert(k.clone(), convert_yaml_value_to_nu_value(v, span)?);
+                    }
+                    // Hard-code fix for cases where "v" is a string without quotations with double curly braces
+                    // e.g. k = value
+                    // value: {{ something }}
+                    // Strangely, serde_yaml returns
+                    // "value" -> Mapping(Mapping { map: {Mapping(Mapping { map: {String("something"): Null} }): Null} })
+                    (serde_yaml::Value::Mapping(m), serde_yaml::Value::Null) => {
+                        return m
+                            .iter()
+                            .take(1)
+                            .collect_vec()
+                            .first()
+                            .and_then(|e| match e {
+                                (serde_yaml::Value::String(s), serde_yaml::Value::Null) => {
+                                    Some(Value::String {
+                                        val: "{{ ".to_owned() + s + " }}",
+                                        span,
+                                    })
+                                }
+                                _ => None,
+                            })
+                            .ok_or(err_unexpected_map);
+                    }
+                    (_, _) => {
+                        return Err(err_unexpected_map);
+                    }
+                }
+            }
+
+            Value::from(collected)
+        }
+        serde_yaml::Value::Null => Value::nothing(span),
+        x => unimplemented!("Unsupported yaml case: {:?}", x),
+    })
+}
+
+pub fn from_yaml_string_to_value(s: String, span: Span) -> Result<Value, ShellError> {
+    let mut documents = vec![];
+
+    for document in serde_yaml::Deserializer::from_str(&s) {
+        let v: serde_yaml::Value = serde_yaml::Value::deserialize(document).map_err(|x| {
+            ShellError::UnsupportedInput(format!("Could not load yaml: {}", x), span)
+        })?;
+        documents.push(convert_yaml_value_to_nu_value(&v, span)?);
+    }
+
+    match documents.len() {
+        0 => Ok(Value::nothing(span)),
+        1 => Ok(documents.remove(0)),
+        _ => Ok(Value::List {
+            vals: documents,
+            span,
+        }),
+    }
+}
+
+fn from_yaml(input: PipelineData, head: Span) -> Result<PipelineData, ShellError> {
+    let concat_string = input.collect_string("");
+
+    match from_yaml_string_to_value(concat_string, head) {
+        Ok(x) => Ok(x.into_pipeline_data()),
+        Err(other) => Err(other),
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_problematic_yaml() {
+        struct TestCase {
+            description: &'static str,
+            input: &'static str,
+            expected: Result<Value, ShellError>,
+        }
+        let tt: Vec<TestCase> = vec![
+            TestCase {
+                description: "Double Curly Braces With Quotes",
+                input: r#"value: "{{ something }}""#,
+                expected: Ok(Value::Record {
+                    cols: vec!["value".to_string()],
+                    vals: vec![Value::String {
+                        val: "{{ something }}".to_string(),
+                        span: Span::unknown(),
+                    }],
+                    span: Span::unknown(),
+                }),
+            },
+            TestCase {
+                description: "Double Curly Braces Without Quotes",
+                input: r#"value: {{ something }}"#,
+                expected: Ok(Value::Record {
+                    cols: vec!["value".to_string()],
+                    vals: vec![Value::String {
+                        val: "{{ something }}".to_string(),
+                        span: Span::unknown(),
+                    }],
+                    span: Span::unknown(),
+                }),
+            },
+        ];
+        for tc in tt {
+            let actual = from_yaml_string_to_value(tc.input.to_owned(), Span::unknown());
+            if actual.is_err() {
+                assert!(
+                    tc.expected.is_err(),
+                    "actual is Err for test:\nTest Description {}\nErr: {:?}",
+                    tc.description,
+                    actual
+                );
+            } else {
+                assert_eq!(
+                    actual.unwrap().into_string(""),
+                    tc.expected.unwrap().into_string("")
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_examples() {
+        use crate::test_examples;
+
+        test_examples(FromYaml {})
+    }
+}