from url and from eml (#324)

* MathEval Variance and Stddev

* Fix tests and linting

* Typo

* Deal with streams when they are not tables

* FromEml and FromUrl

Added tests for from eml
This commit is contained in:
Luccas Mateus
2021-11-12 17:46:39 -03:00
committed by GitHub
parent e756a9ea04
commit db2bca56c9
7 changed files with 403 additions and 0 deletions

View File

@ -27,6 +27,7 @@ chrono = { version = "0.4.19", features = ["serde"] }
chrono-humanize = "0.2.1"
chrono-tz = "0.6.0"
terminal_size = "0.1.17"
indexmap = { version="1.7", features=["serde-1"] }
lscolors = { version = "0.8.0", features = ["crossterm"] }
bytesize = "1.1.0"
dialoguer = "0.9.0"
@ -36,6 +37,8 @@ titlecase = "1.1.0"
meval = "0.2.0"
serde = { version="1.0.123", features=["derive"] }
serde_yaml = "0.8.16"
serde_urlencoded = "0.7.0"
eml-parser = "0.1.0"
itertools = "0.10.0"
rand = "0.8"

View File

@ -50,6 +50,8 @@ pub fn create_default_context() -> EngineState {
FromYaml,
FromYml,
FromTsv,
FromUrl,
FromEml,
Get,
Griddle,
Help,

View File

@ -0,0 +1,254 @@
use ::eml_parser::eml::*;
use ::eml_parser::EmlParser;
use indexmap::map::IndexMap;
use nu_engine::CallExt;
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Value,
};
#[derive(Clone)]
pub struct FromEml;
const DEFAULT_BODY_PREVIEW: usize = 50;
impl Command for FromEml {
fn name(&self) -> &str {
"from eml"
}
fn signature(&self) -> Signature {
Signature::build("from eml").named(
"preview-body",
SyntaxShape::Int,
"How many bytes of the body to preview",
Some('b'),
)
}
fn usage(&self) -> &str {
"Parse text as .eml and create table."
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<nu_protocol::PipelineData, ShellError> {
let head = call.head;
let preview_body: Option<Spanned<i64>> =
call.get_flag(engine_state, stack, "preview-body")?;
from_eml(input, preview_body, head)
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Convert eml structured data into table",
example: "'From: test@email.com
Subject: Welcome
To: someone@somewhere.com
Test' | from eml",
result: Some(Value::Record {
cols: vec![
"Subject".to_string(),
"From".to_string(),
"To".to_string(),
"Body".to_string(),
],
vals: vec![
Value::test_string("Welcome"),
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::unknown()),
Value::test_string("test@email.com"),
],
span: Span::unknown(),
},
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::unknown()),
Value::test_string("someone@somewhere.com"),
],
span: Span::unknown(),
},
Value::test_string("Test"),
],
span: Span::unknown(),
}),
},
Example {
description: "Convert eml structured data into table",
example: "'From: test@email.com
Subject: Welcome
To: someone@somewhere.com
Test' | from eml -b 1",
result: Some(Value::Record {
cols: vec![
"Subject".to_string(),
"From".to_string(),
"To".to_string(),
"Body".to_string(),
],
vals: vec![
Value::test_string("Welcome"),
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::unknown()),
Value::test_string("test@email.com"),
],
span: Span::unknown(),
},
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![
Value::nothing(Span::unknown()),
Value::test_string("someone@somewhere.com"),
],
span: Span::unknown(),
},
Value::test_string("T"),
],
span: Span::unknown(),
}),
},
]
}
}
fn emailaddress_to_value(span: Span, email_address: &EmailAddress) -> Value {
let (n, a) = match email_address {
EmailAddress::AddressOnly { address } => (
Value::nothing(span),
Value::String {
val: address.to_string(),
span,
},
),
EmailAddress::NameAndEmailAddress { name, address } => (
Value::String {
val: name.to_string(),
span,
},
Value::String {
val: address.to_string(),
span,
},
),
};
Value::Record {
cols: vec!["Name".to_string(), "Address".to_string()],
vals: vec![n, a],
span,
}
}
fn headerfieldvalue_to_value(head: Span, value: &HeaderFieldValue) -> Value {
use HeaderFieldValue::*;
match value {
SingleEmailAddress(address) => emailaddress_to_value(head, address),
MultipleEmailAddresses(addresses) => Value::List {
vals: addresses
.iter()
.map(|a| emailaddress_to_value(head, a))
.collect(),
span: head,
},
Unstructured(s) => Value::String {
val: s.to_string(),
span: head,
},
Empty => Value::nothing(head),
}
}
fn from_eml(
input: PipelineData,
preview_body: Option<Spanned<i64>>,
head: Span,
) -> Result<PipelineData, ShellError> {
let value = input.collect_string("");
let body_preview = preview_body
.map(|b| b.item as usize)
.unwrap_or(DEFAULT_BODY_PREVIEW);
let eml = EmlParser::from_string(value)
.with_body_preview(body_preview)
.parse()
.map_err(|_| {
ShellError::CantConvert("structured data from eml".into(), "string".into(), head)
})?;
let mut collected = IndexMap::new();
if let Some(subj) = eml.subject {
collected.insert(
"Subject".to_string(),
Value::String {
val: subj,
span: head,
},
);
}
if let Some(from) = eml.from {
collected.insert("From".to_string(), headerfieldvalue_to_value(head, &from));
}
if let Some(to) = eml.to {
collected.insert("To".to_string(), headerfieldvalue_to_value(head, &to));
}
for HeaderField { name, value } in &eml.headers {
collected.insert(name.to_string(), headerfieldvalue_to_value(head, value));
}
if let Some(body) = eml.body {
collected.insert(
"Body".to_string(),
Value::String {
val: body,
span: head,
},
);
}
let (cols, vals) = collected
.into_iter()
.fold((vec![], vec![]), |mut acc, (k, v)| {
acc.0.push(k);
acc.1.push(v);
acc
});
let record = Value::Record {
cols,
vals,
span: head,
};
Ok(PipelineData::Value(record))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(FromEml {})
}
}

View File

@ -1,13 +1,17 @@
mod command;
mod csv;
mod delimited;
mod eml;
mod json;
mod tsv;
mod url;
mod yaml;
pub use self::csv::FromCsv;
pub use command::From;
pub use eml::FromEml;
pub use json::FromJson;
pub use tsv::FromTsv;
pub use url::FromUrl;
pub use yaml::FromYaml;
pub use yaml::FromYml;

View File

@ -0,0 +1,92 @@
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{Example, PipelineData, ShellError, Signature, Span, Value};
#[derive(Clone)]
pub struct FromUrl;
impl Command for FromUrl {
fn name(&self) -> &str {
"from url"
}
fn signature(&self) -> Signature {
Signature::build("from url")
}
fn usage(&self) -> &str {
"Parse url-encoded string as a table."
}
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<nu_protocol::PipelineData, ShellError> {
let head = call.head;
from_url(input, head)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
example: "'bread=baguette&cheese=comt%C3%A9&meat=ham&fat=butter' | from url",
description: "Convert url encoded string into a table",
result: Some(Value::Record {
cols: vec![
"bread".to_string(),
"cheese".to_string(),
"meat".to_string(),
"fat".to_string(),
],
vals: vec![
Value::test_string("baguette"),
Value::test_string("comté"),
Value::test_string("ham"),
Value::test_string("butter"),
],
span: Span::unknown(),
}),
}]
}
}
fn from_url(input: PipelineData, head: Span) -> Result<PipelineData, ShellError> {
let concat_string = input.collect_string("");
let result = serde_urlencoded::from_str::<Vec<(String, String)>>(&concat_string);
match result {
Ok(result) => {
let mut cols = vec![];
let mut vals = vec![];
for (k, v) in result {
cols.push(k);
vals.push(Value::String { val: v, span: head })
}
Ok(PipelineData::Value(Value::Record {
cols,
vals,
span: head,
}))
}
_ => Err(ShellError::UnsupportedInput(
"String not compatible with url-encoding".to_string(),
head,
)),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(FromUrl {})
}
}

View File

@ -531,6 +531,7 @@ impl PartialOrd for Value {
(Value::Binary { val: lhs, .. }, Value::Binary { val: rhs, .. }) => {
lhs.partial_cmp(rhs)
}
(Value::Nothing { .. }, Value::Nothing { .. }) => Some(Ordering::Equal),
(_, _) => None,
}
}