forked from extern/nushell
from yaml and from yml (#322)
* MathEval Variance and Stddev * Fix tests and linting * Typo * Deal with streams when they are not tables * `from yaml` and `from yml` `from yaml` and `from yml` from yaml and from yml * Fix collect_string * Fix tests and linting
This commit is contained in:
Luccas Mateus
GitHub
parent
d094f654c3
commit
75cfee28b2
46
Cargo.lock
generated
46
Cargo.lock
generated
@ -420,6 +420,12 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
|
||||
|
||||
[[package]]
|
||||
name = "dtoa"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
|
||||
|
||||
[[package]]
|
||||
name = "dunce"
|
||||
version = "1.0.2"
|
||||
@ -491,6 +497,12 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.19"
|
||||
@ -514,6 +526,16 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.12"
|
||||
@ -745,6 +767,7 @@ dependencies = [
|
||||
"csv",
|
||||
"dialoguer",
|
||||
"glob",
|
||||
"itertools",
|
||||
"lscolors",
|
||||
"meval",
|
||||
"nu-engine",
|
||||
@ -756,6 +779,8 @@ dependencies = [
|
||||
"nu-term-grid",
|
||||
"rand",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_yaml",
|
||||
"sysinfo",
|
||||
"terminal_size",
|
||||
"thiserror",
|
||||
@ -1285,6 +1310,18 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_yaml"
|
||||
version = "0.8.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8c608a35705a5d3cdc9fbe403147647ff34b921f8e833e49306df898f9b20af"
|
||||
dependencies = [
|
||||
"dtoa",
|
||||
"indexmap",
|
||||
"serde",
|
||||
"yaml-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook"
|
||||
version = "0.3.10"
|
||||
@ -1631,6 +1668,15 @@ version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "yaml-rust"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
|
||||
dependencies = [
|
||||
"linked-hash-map",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.4.3"
|
||||
|
||||
@ -33,6 +33,9 @@ dialoguer = "0.9.0"
|
||||
rayon = "1.5.1"
|
||||
titlecase = "1.1.0"
|
||||
meval = "0.2.0"
|
||||
serde = { version="1.0.123", features=["derive"] }
|
||||
serde_yaml = "0.8.16"
|
||||
itertools = "0.10.0"
|
||||
rand = "0.8"
|
||||
|
||||
[features]
|
||||
|
||||
@ -47,6 +47,8 @@ pub fn create_default_context() -> EngineState {
|
||||
From,
|
||||
FromCsv,
|
||||
FromJson,
|
||||
FromYaml,
|
||||
FromYml,
|
||||
FromTsv,
|
||||
Get,
|
||||
Griddle,
|
||||
|
||||
@ -3,8 +3,11 @@ mod csv;
|
||||
mod delimited;
|
||||
mod json;
|
||||
mod tsv;
|
||||
mod yaml;
|
||||
|
||||
pub use self::csv::FromCsv;
|
||||
pub use command::From;
|
||||
pub use json::FromJson;
|
||||
pub use tsv::FromTsv;
|
||||
pub use yaml::FromYaml;
|
||||
pub use yaml::FromYml;
|
||||
|
||||
275
crates/nu-command/src/formats/from/yaml.rs
Normal file
275
crates/nu-command/src/formats/from/yaml.rs
Normal file
@ -0,0 +1,275 @@
|
||||
use itertools::Itertools;
|
||||
use nu_protocol::ast::Call;
|
||||
use nu_protocol::engine::{Command, EngineState, Stack};
|
||||
use nu_protocol::{
|
||||
Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Spanned, Value,
|
||||
};
|
||||
use serde::de::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct FromYaml;
|
||||
|
||||
impl Command for FromYaml {
|
||||
fn name(&self) -> &str {
|
||||
"from yaml"
|
||||
}
|
||||
|
||||
fn signature(&self) -> Signature {
|
||||
Signature::build("from yaml")
|
||||
}
|
||||
|
||||
fn usage(&self) -> &str {
|
||||
"Parse text as .yaml/.yml and create table."
|
||||
}
|
||||
|
||||
fn examples(&self) -> Vec<Example> {
|
||||
vec![
|
||||
Example {
|
||||
example: "'a: 1' | from yaml",
|
||||
description: "Converts yaml formatted string to table",
|
||||
result: Some(Value::Record {
|
||||
cols: vec!["a".to_string()],
|
||||
vals: vec![Value::Int {
|
||||
val: 1,
|
||||
span: Span::unknown(),
|
||||
}],
|
||||
span: Span::unknown(),
|
||||
}),
|
||||
},
|
||||
Example {
|
||||
example: "'[ a: 1, b: [1, 2] ]' | from yaml",
|
||||
description: "Converts yaml formatted string to table",
|
||||
result: Some(Value::List {
|
||||
vals: vec![
|
||||
Value::Record {
|
||||
cols: vec!["a".to_string()],
|
||||
vals: vec![Value::test_int(1)],
|
||||
span: Span::unknown(),
|
||||
},
|
||||
Value::Record {
|
||||
cols: vec!["b".to_string()],
|
||||
vals: vec![Value::List {
|
||||
vals: vec![Value::test_int(1), Value::test_int(2)],
|
||||
span: Span::unknown(),
|
||||
}],
|
||||
span: Span::unknown(),
|
||||
},
|
||||
],
|
||||
span: Span::unknown(),
|
||||
}),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
_engine_state: &EngineState,
|
||||
_stack: &mut Stack,
|
||||
call: &Call,
|
||||
input: PipelineData,
|
||||
) -> Result<nu_protocol::PipelineData, ShellError> {
|
||||
let head = call.head;
|
||||
from_yaml(input, head)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct FromYml;
|
||||
|
||||
impl Command for FromYml {
|
||||
fn name(&self) -> &str {
|
||||
"from yml"
|
||||
}
|
||||
|
||||
fn signature(&self) -> Signature {
|
||||
Signature::build("from yml")
|
||||
}
|
||||
|
||||
fn usage(&self) -> &str {
|
||||
"Parse text as .yaml/.yml and create table."
|
||||
}
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
_engine_state: &EngineState,
|
||||
_stack: &mut Stack,
|
||||
call: &Call,
|
||||
input: PipelineData,
|
||||
) -> Result<nu_protocol::PipelineData, ShellError> {
|
||||
let head = call.head;
|
||||
from_yaml(input, head)
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_yaml_value_to_nu_value(v: &serde_yaml::Value, span: Span) -> Result<Value, ShellError> {
|
||||
let err_not_compatible_number =
|
||||
ShellError::UnsupportedInput("Expected a compatible number".to_string(), span);
|
||||
Ok(match v {
|
||||
serde_yaml::Value::Bool(b) => Value::Bool { val: *b, span },
|
||||
serde_yaml::Value::Number(n) if n.is_i64() => Value::Int {
|
||||
val: n.as_i64().ok_or(err_not_compatible_number)?,
|
||||
span,
|
||||
},
|
||||
serde_yaml::Value::Number(n) if n.is_f64() => Value::Float {
|
||||
val: n.as_f64().ok_or(err_not_compatible_number)?,
|
||||
span,
|
||||
},
|
||||
serde_yaml::Value::String(s) => Value::String {
|
||||
val: s.to_string(),
|
||||
span,
|
||||
},
|
||||
serde_yaml::Value::Sequence(a) => {
|
||||
let result: Result<Vec<Value>, ShellError> = a
|
||||
.iter()
|
||||
.map(|x| convert_yaml_value_to_nu_value(x, span))
|
||||
.collect();
|
||||
Value::List {
|
||||
vals: result?,
|
||||
span,
|
||||
}
|
||||
}
|
||||
serde_yaml::Value::Mapping(t) => {
|
||||
let mut collected = Spanned {
|
||||
item: HashMap::new(),
|
||||
span,
|
||||
};
|
||||
|
||||
for (k, v) in t {
|
||||
// A ShellError that we re-use multiple times in the Mapping scenario
|
||||
let err_unexpected_map = ShellError::UnsupportedInput(
|
||||
format!("Unexpected YAML:\nKey: {:?}\nValue: {:?}", k, v),
|
||||
span,
|
||||
);
|
||||
match (k, v) {
|
||||
(serde_yaml::Value::String(k), _) => {
|
||||
collected
|
||||
.item
|
||||
.insert(k.clone(), convert_yaml_value_to_nu_value(v, span)?);
|
||||
}
|
||||
// Hard-code fix for cases where "v" is a string without quotations with double curly braces
|
||||
// e.g. k = value
|
||||
// value: {{ something }}
|
||||
// Strangely, serde_yaml returns
|
||||
// "value" -> Mapping(Mapping { map: {Mapping(Mapping { map: {String("something"): Null} }): Null} })
|
||||
(serde_yaml::Value::Mapping(m), serde_yaml::Value::Null) => {
|
||||
return m
|
||||
.iter()
|
||||
.take(1)
|
||||
.collect_vec()
|
||||
.first()
|
||||
.and_then(|e| match e {
|
||||
(serde_yaml::Value::String(s), serde_yaml::Value::Null) => {
|
||||
Some(Value::String {
|
||||
val: "{{ ".to_owned() + s + " }}",
|
||||
span,
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.ok_or(err_unexpected_map);
|
||||
}
|
||||
(_, _) => {
|
||||
return Err(err_unexpected_map);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Value::from(collected)
|
||||
}
|
||||
serde_yaml::Value::Null => Value::nothing(span),
|
||||
x => unimplemented!("Unsupported yaml case: {:?}", x),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn from_yaml_string_to_value(s: String, span: Span) -> Result<Value, ShellError> {
|
||||
let mut documents = vec![];
|
||||
|
||||
for document in serde_yaml::Deserializer::from_str(&s) {
|
||||
let v: serde_yaml::Value = serde_yaml::Value::deserialize(document).map_err(|x| {
|
||||
ShellError::UnsupportedInput(format!("Could not load yaml: {}", x), span)
|
||||
})?;
|
||||
documents.push(convert_yaml_value_to_nu_value(&v, span)?);
|
||||
}
|
||||
|
||||
match documents.len() {
|
||||
0 => Ok(Value::nothing(span)),
|
||||
1 => Ok(documents.remove(0)),
|
||||
_ => Ok(Value::List {
|
||||
vals: documents,
|
||||
span,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn from_yaml(input: PipelineData, head: Span) -> Result<PipelineData, ShellError> {
|
||||
let concat_string = input.collect_string("");
|
||||
|
||||
match from_yaml_string_to_value(concat_string, head) {
|
||||
Ok(x) => Ok(x.into_pipeline_data()),
|
||||
Err(other) => Err(other),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_problematic_yaml() {
|
||||
struct TestCase {
|
||||
description: &'static str,
|
||||
input: &'static str,
|
||||
expected: Result<Value, ShellError>,
|
||||
}
|
||||
let tt: Vec<TestCase> = vec![
|
||||
TestCase {
|
||||
description: "Double Curly Braces With Quotes",
|
||||
input: r#"value: "{{ something }}""#,
|
||||
expected: Ok(Value::Record {
|
||||
cols: vec!["value".to_string()],
|
||||
vals: vec![Value::String {
|
||||
val: "{{ something }}".to_string(),
|
||||
span: Span::unknown(),
|
||||
}],
|
||||
span: Span::unknown(),
|
||||
}),
|
||||
},
|
||||
TestCase {
|
||||
description: "Double Curly Braces Without Quotes",
|
||||
input: r#"value: {{ something }}"#,
|
||||
expected: Ok(Value::Record {
|
||||
cols: vec!["value".to_string()],
|
||||
vals: vec![Value::String {
|
||||
val: "{{ something }}".to_string(),
|
||||
span: Span::unknown(),
|
||||
}],
|
||||
span: Span::unknown(),
|
||||
}),
|
||||
},
|
||||
];
|
||||
for tc in tt {
|
||||
let actual = from_yaml_string_to_value(tc.input.to_owned(), Span::unknown());
|
||||
if actual.is_err() {
|
||||
assert!(
|
||||
tc.expected.is_err(),
|
||||
"actual is Err for test:\nTest Description {}\nErr: {:?}",
|
||||
tc.description,
|
||||
actual
|
||||
);
|
||||
} else {
|
||||
assert_eq!(
|
||||
actual.unwrap().into_string(""),
|
||||
tc.expected.unwrap().into_string("")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_examples() {
|
||||
use crate::test_examples;
|
||||
|
||||
test_examples(FromYaml {})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user