holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

from vcf from ics and from ini (#367)

Browse Source 
* MathEval Variance and Stddev

* Fix tests and linting

* Typo

* Deal with streams when they are not tables

* `from toml` command

* From ods

* From XLSX

* From ics

* From ini

* From vcf

* Forgot a eprintln!
This commit is contained in:
Luccas Mateus 2021-11-25 14:10:56 -03:00 committed by GitHub
parent 6a1942b18f
commit 8043516d75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 722 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
Cargo.lock generated
View File

@ -992,6 +992,15 @@ dependencies = [
"libc",
]
[[package]]
name = "ical"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a9f7215ad0d77e69644570dee000c7678a47ba7441062c1b5f918adde0d73cf"
dependencies = [
"thiserror",
]
[[package]]
name = "im"
version = "15.0.0"
@ -1429,6 +1438,7 @@ dependencies = [
"dialoguer",
"eml-parser",
"glob",
"ical",
"indexmap",
"itertools",
"lscolors",
@ -1447,6 +1457,7 @@ dependencies = [
"rayon",
"regex",
"serde",
"serde_ini",
"serde_urlencoded",
"serde_yaml",
"sysinfo",
@ -1532,6 +1543,7 @@ dependencies = [
"chrono",
"chrono-humanize",
"im",
"indexmap",
"miette",
"nu-json",
"serde",
@ -2164,6 +2176,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "result"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "194d8e591e405d1eecf28819740abed6d719d1a2db87fc0bcdedee9a26d55560"
[[package]]
name = "rust-argon2"
version = "0.8.3"
@ -2232,6 +2250,17 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_ini"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb236687e2bb073a7521c021949be944641e671b8505a94069ca37b656c81139"
dependencies = [
"result",
"serde",
"void",
]
[[package]]
name = "serde_json"
version = "1.0.71"
@ -2672,6 +2701,12 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
[[package]]
name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
[[package]]
name = "vte"
version = "0.10.1"

2
crates/nu-command/Cargo.toml
View File

@ -39,9 +39,11 @@ meval = "0.2.0"
serde = { version="1.0.123", features=["derive"] }
serde_yaml = "0.8.16"
serde_urlencoded = "0.7.0"
serde_ini = "0.2.0"
eml-parser = "0.1.0"
toml = "0.5.8"
itertools = "0.10.0"
ical = "0.7.0"
calamine = "0.18.0"
rand = "0.8"

3
crates/nu-command/src/default_context.rs
View File

@ -60,6 +60,9 @@ pub fn create_default_context() -> EngineState {
FromUrl,
FromEml,
FromOds,
FromIcs,
FromIni,
FromVcf,
FromXlsx,
Get,
Griddle,

17
crates/nu-command/src/formats/from/eml.rs
View File

@ -230,21 +230,10 @@ fn from_eml(
);
}
let (cols, vals) = collected
.into_iter()
.fold((vec![], vec![]), |mut acc, (k, v)| {
acc.0.push(k);
acc.1.push(v);
acc
});
let record = Value::Record {
cols,
vals,
Ok(PipelineData::Value(Value::from(Spanned {
item: collected,
span: head,
};
Ok(PipelineData::Value(record))
})))
}
#[cfg(test)]

327
crates/nu-command/src/formats/from/ics.rs Normal file
View File

@ -0,0 +1,327 @@
extern crate ical;
use ical::parser::ical::component::*;
use ical::property::Property;
use indexmap::map::IndexMap;
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
Category, Config, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span,
Spanned, Value,
};
use std::io::BufReader;
#[derive(Clone)]
pub struct FromIcs;
impl Command for FromIcs {
fn name(&self) -> &str {
"from ics"
}
fn signature(&self) -> Signature {
Signature::build("from ics").category(Category::Formats)
}
fn usage(&self) -> &str {
"Parse text as .ics and create table."
}
fn run(
&self,
_engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<nu_protocol::PipelineData, ShellError> {
let head = call.head;
let config = stack.get_config()?;
from_ics(input, head, &config)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
example: "'BEGIN:VCALENDAR
END:VCALENDAR' | from ics",
description: "Converts ics formatted string to table",
result: Some(Value::List {
vals: vec![Value::Record {
cols: vec![
"properties".to_string(),
"events".to_string(),
"alarms".to_string(),
"to-Dos".to_string(),
"journals".to_string(),
"free-busys".to_string(),
"timezones".to_string(),
],
vals: vec![
Value::List {
vals: vec![],
span: Span::unknown(),
},
Value::List {
vals: vec![],
span: Span::unknown(),
},
Value::List {
vals: vec![],
span: Span::unknown(),
},
Value::List {
vals: vec![],
span: Span::unknown(),
},
Value::List {
vals: vec![],
span: Span::unknown(),
},
Value::List {
vals: vec![],
span: Span::unknown(),
},
Value::List {
vals: vec![],
span: Span::unknown(),
},
],
span: Span::unknown(),
}],
span: Span::unknown(),
}),
}]
}
}
fn from_ics(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
let input_string = input.collect_string("", config);
let input_bytes = input_string.as_bytes();
let buf_reader = BufReader::new(input_bytes);
let parser = ical::IcalParser::new(buf_reader);
let mut output = vec![];
for calendar in parser {
match calendar {
Ok(c) => output.push(calendar_to_value(c, head)),
Err(_) => output.push(Value::Error {
error: ShellError::UnsupportedInput(
"input cannot be parsed as .ics".to_string(),
head,
),
}),
}
}
Ok(Value::List {
vals: output,
span: head,
}
.into_pipeline_data())
}
fn calendar_to_value(calendar: IcalCalendar, span: Span) -> Value {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(calendar.properties, span),
);
row.insert("events".to_string(), events_to_value(calendar.events, span));
row.insert("alarms".to_string(), alarms_to_value(calendar.alarms, span));
row.insert("to-Dos".to_string(), todos_to_value(calendar.todos, span));
row.insert(
"journals".to_string(),
journals_to_value(calendar.journals, span),
);
row.insert(
"free-busys".to_string(),
free_busys_to_value(calendar.free_busys, span),
);
row.insert(
"timezones".to_string(),
timezones_to_value(calendar.timezones, span),
);
Value::from(Spanned { item: row, span })
}
fn events_to_value(events: Vec<IcalEvent>, span: Span) -> Value {
Value::List {
vals: events
.into_iter()
.map(|event| {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(event.properties, span),
);
row.insert("alarms".to_string(), alarms_to_value(event.alarms, span));
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn alarms_to_value(alarms: Vec<IcalAlarm>, span: Span) -> Value {
Value::List {
vals: alarms
.into_iter()
.map(|alarm| {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(alarm.properties, span),
);
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn todos_to_value(todos: Vec<IcalTodo>, span: Span) -> Value {
Value::List {
vals: todos
.into_iter()
.map(|todo| {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(todo.properties, span),
);
row.insert("alarms".to_string(), alarms_to_value(todo.alarms, span));
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn journals_to_value(journals: Vec<IcalJournal>, span: Span) -> Value {
Value::List {
vals: journals
.into_iter()
.map(|journal| {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(journal.properties, span),
);
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn free_busys_to_value(free_busys: Vec<IcalFreeBusy>, span: Span) -> Value {
Value::List {
vals: free_busys
.into_iter()
.map(|free_busy| {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(free_busy.properties, span),
);
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn timezones_to_value(timezones: Vec<IcalTimeZone>, span: Span) -> Value {
Value::List {
vals: timezones
.into_iter()
.map(|timezone| {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(timezone.properties, span),
);
row.insert(
"transitions".to_string(),
timezone_transitions_to_value(timezone.transitions, span),
);
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn timezone_transitions_to_value(transitions: Vec<IcalTimeZoneTransition>, span: Span) -> Value {
Value::List {
vals: transitions
.into_iter()
.map(|transition| {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(transition.properties, span),
);
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn properties_to_value(properties: Vec<Property>, span: Span) -> Value {
Value::List {
vals: properties
.into_iter()
.map(|prop| {
let mut row = IndexMap::new();
let name = Value::String {
val: prop.name,
span,
};
let value = match prop.value {
Some(val) => Value::String { val, span },
None => Value::nothing(span),
};
let params = match prop.params {
Some(param_list) => params_to_value(param_list, span),
None => Value::nothing(span),
};
row.insert("name".to_string(), name);
row.insert("value".to_string(), value);
row.insert("params".to_string(), params);
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn params_to_value(params: Vec<(String, Vec<String>)>, span: Span) -> Value {
let mut row = IndexMap::new();
for (param_name, param_values) in params {
let values: Vec<Value> = param_values
.into_iter()
.map(|val| Value::string(val, span))
.collect();
let values = Value::List { vals: values, span };
row.insert(param_name, values);
}
Value::from(Spanned { item: row, span })
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(FromIcs {})
}
}

109
crates/nu-command/src/formats/from/ini.rs Normal file
View File

@ -0,0 +1,109 @@
use indexmap::map::IndexMap;
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
Category, Config, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Value,
};
#[derive(Clone)]
pub struct FromIni;
impl Command for FromIni {
fn name(&self) -> &str {
"from ini"
}
fn signature(&self) -> Signature {
Signature::build("from ini").category(Category::Formats)
}
fn usage(&self) -> &str {
"Parse text as .ini and create table"
}
fn examples(&self) -> Vec<Example> {
vec![Example {
example: "'[foo]
a=1
b=2' | from ini",
description: "Converts ini formatted string to table",
result: Some(Value::Record {
cols: vec!["foo".to_string()],
vals: vec![Value::Record {
cols: vec!["a".to_string(), "b".to_string()],
vals: vec![
Value::String {
val: "1".to_string(),
span: Span::unknown(),
},
Value::String {
val: "2".to_string(),
span: Span::unknown(),
},
],
span: Span::unknown(),
}],
span: Span::unknown(),
}),
}]
}
fn run(
&self,
_engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<nu_protocol::PipelineData, ShellError> {
let head = call.head;
let config = stack.get_config()?;
from_ini(input, head, &config)
}
}
pub fn from_ini_string_to_value(s: String, span: Span) -> Result<Value, ShellError> {
let v: Result<IndexMap<String, IndexMap<String, String>>, serde_ini::de::Error> =
serde_ini::from_str(&s);
match v {
Ok(index_map) => {
let (cols, vals) = index_map
.into_iter()
.fold((vec![], vec![]), |mut acc, (k, v)| {
let (cols, vals) = v.into_iter().fold((vec![], vec![]), |mut acc, (k, v)| {
acc.0.push(k);
acc.1.push(Value::String { val: v, span });
acc
});
acc.0.push(k);
acc.1.push(Value::Record { cols, vals, span });
acc
});
Ok(Value::Record { cols, vals, span })
}
Err(err) => Err(ShellError::UnsupportedInput(
format!("Could not load ini: {}", err),
span,
)),
}
}
fn from_ini(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
let concat_string = input.collect_string("", config);
match from_ini_string_to_value(concat_string, head) {
Ok(x) => Ok(x.into_pipeline_data()),
Err(other) => Err(other),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(FromIni {})
}
}

6
crates/nu-command/src/formats/from/mod.rs
View File

@ -2,11 +2,14 @@ mod command;
mod csv;
mod delimited;
mod eml;
mod ics;
mod ini;
mod json;
mod ods;
mod toml;
mod tsv;
mod url;
mod vcf;
mod xlsx;
mod yaml;
@ -14,10 +17,13 @@ pub use self::csv::FromCsv;
pub use self::toml::FromToml;
pub use command::From;
pub use eml::FromEml;
pub use ics::FromIcs;
pub use ini::FromIni;
pub use json::FromJson;
pub use ods::FromOds;
pub use tsv::FromTsv;
pub use url::FromUrl;
pub use vcf::FromVcf;
pub use xlsx::FromXlsx;
pub use yaml::FromYaml;
pub use yaml::FromYml;

211
crates/nu-command/src/formats/from/vcf.rs Normal file
View File

@ -0,0 +1,211 @@
use ical::parser::vcard::component::*;
use ical::property::Property;
use indexmap::map::IndexMap;
use nu_protocol::ast::Call;
use nu_protocol::engine::{Command, EngineState, Stack};
use nu_protocol::{
Category, Config, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span,
Spanned, Value,
};
#[derive(Clone)]
pub struct FromVcf;
impl Command for FromVcf {
fn name(&self) -> &str {
"from vcf"
}
fn signature(&self) -> Signature {
Signature::build("from vcf").category(Category::Formats)
}
fn usage(&self) -> &str {
"Parse text as .vcf and create table."
}
fn run(
&self,
_engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<nu_protocol::PipelineData, ShellError> {
let head = call.head;
let config = stack.get_config()?;
from_vcf(input, head, &config)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
example: "'BEGIN:VCARD
N:Foo
FN:Bar
EMAIL:foo@bar.com
END:VCARD' | from vcf",
description: "Converts ics formatted string to table",
result: Some(Value::List {
vals: vec![Value::Record {
cols: vec!["properties".to_string()],
vals: vec![Value::List {
vals: vec![
Value::Record {
cols: vec![
"name".to_string(),
"value".to_string(),
"params".to_string(),
],
vals: vec![
Value::String {
val: "N".to_string(),
span: Span::unknown(),
},
Value::String {
val: "Foo".to_string(),
span: Span::unknown(),
},
Value::Nothing {
span: Span::unknown(),
},
],
span: Span::unknown(),
},
Value::Record {
cols: vec![
"name".to_string(),
"value".to_string(),
"params".to_string(),
],
vals: vec![
Value::String {
val: "FN".to_string(),
span: Span::unknown(),
},
Value::String {
val: "Bar".to_string(),
span: Span::unknown(),
},
Value::Nothing {
span: Span::unknown(),
},
],
span: Span::unknown(),
},
Value::Record {
cols: vec![
"name".to_string(),
"value".to_string(),
"params".to_string(),
],
vals: vec![
Value::String {
val: "EMAIL".to_string(),
span: Span::unknown(),
},
Value::String {
val: "foo@bar.com".to_string(),
span: Span::unknown(),
},
Value::Nothing {
span: Span::unknown(),
},
],
span: Span::unknown(),
},
],
span: Span::unknown(),
}],
span: Span::unknown(),
}],
span: Span::unknown(),
}),
}]
}
}
fn from_vcf(input: PipelineData, head: Span, config: &Config) -> Result<PipelineData, ShellError> {
let input_string = input.collect_string("", config);
let input_bytes = input_string.as_bytes();
let cursor = std::io::Cursor::new(input_bytes);
let parser = ical::VcardParser::new(cursor);
let iter = parser.map(move |contact| match contact {
Ok(c) => contact_to_value(c, head),
Err(_) => Value::Error {
error: ShellError::UnsupportedInput("input cannot be parsed as .vcf".to_string(), head),
},
});
let collected: Vec<_> = iter.collect();
Ok(Value::List {
vals: collected,
span: head,
}
.into_pipeline_data())
}
fn contact_to_value(contact: VcardContact, span: Span) -> Value {
let mut row = IndexMap::new();
row.insert(
"properties".to_string(),
properties_to_value(contact.properties, span),
);
Value::from(Spanned { item: row, span })
}
fn properties_to_value(properties: Vec<Property>, span: Span) -> Value {
Value::List {
vals: properties
.into_iter()
.map(|prop| {
let mut row = IndexMap::new();
let name = Value::String {
val: prop.name,
span,
};
let value = match prop.value {
Some(val) => Value::String { val, span },
None => Value::Nothing { span },
};
let params = match prop.params {
Some(param_list) => params_to_value(param_list, span),
None => Value::Nothing { span },
};
row.insert("name".to_string(), name);
row.insert("value".to_string(), value);
row.insert("params".to_string(), params);
Value::from(Spanned { item: row, span })
})
.collect::<Vec<Value>>(),
span,
}
}
fn params_to_value(params: Vec<(String, Vec<String>)>, span: Span) -> Value {
let mut row = IndexMap::new();
for (param_name, param_values) in params {
let values: Vec<Value> = param_values
.into_iter()
.map(|val| Value::string(val, span))
.collect();
let values = Value::List { vals: values, span };
row.insert(param_name, values);
}
Value::from(Spanned { item: row, span })
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples() {
use crate::test_examples;
test_examples(FromVcf {})
}
}

22
crates/nu-command/src/math/utils.rs
View File

@ -1,6 +1,6 @@
use indexmap::map::IndexMap;
use nu_protocol::ast::Call;
use nu_protocol::{IntoPipelineData, PipelineData, ShellError, Span, Value};
use std::collections::HashMap;
use nu_protocol::{IntoPipelineData, PipelineData, ShellError, Span, Spanned, Value};
pub fn run_with_function(
call: &Call,
@ -22,7 +22,7 @@ fn helper_for_tables(
) -> Result<Value, ShellError> {
// If we are not dealing with Primitives, then perhaps we are dealing with a table
// Create a key for each column name
let mut column_values = HashMap::new();
let mut column_values = IndexMap::new();
for val in values {
if let Value::Record { cols, vals, .. } = val {
for (key, value) in cols.iter().zip(vals.iter()) {
@ -37,7 +37,7 @@ fn helper_for_tables(
}
}
// The mathematical function operates over the columns of the table
let mut column_totals = HashMap::new();
let mut column_totals = IndexMap::new();
for (col_name, col_vals) in column_values {
if let Ok(out) = mf(&col_vals, &name) {
column_totals.insert(col_name, out);
@ -49,19 +49,11 @@ fn helper_for_tables(
name,
));
}
let (cols, vals) = column_totals
.into_iter()
.fold((vec![], vec![]), |mut acc, (k, v)| {
acc.0.push(k);
acc.1.push(v);
acc
});
Ok(Value::Record {
cols,
vals,
Ok(Value::from(Spanned {
item: column_totals,
span: name,
})
}))
}
pub fn calculate(

1
crates/nu-protocol/Cargo.toml
View File

@ -10,6 +10,7 @@ thiserror = "1.0.29"
miette = "3.0.0"
serde = {version = "1.0.130", features = ["derive"]}
chrono = { version="0.4.19", features=["serde"] }
indexmap = { version="1.7", features=["serde-1"] }
chrono-humanize = "0.2.1"
byte-unit = "4.0.9"
im = "15.0.0"

18
crates/nu-protocol/src/value/mod.rs
View File

@ -6,6 +6,7 @@ mod unit;
use chrono::{DateTime, FixedOffset};
use chrono_humanize::HumanTime;
use indexmap::map::IndexMap;
pub use range::*;
use serde::{Deserialize, Serialize};
pub use stream::*;
@ -1320,6 +1321,23 @@ impl From<Spanned<HashMap<String, Value>>> for Value {
}
}
/// Create a Value::Record from a spanned indexmap
impl From<Spanned<IndexMap<String, Value>>> for Value {
fn from(input: Spanned<IndexMap<String, Value>>) -> Self {
let span = input.span;
let (cols, vals) = input
.item
.into_iter()
.fold((vec![], vec![]), |mut acc, (k, v)| {
acc.0.push(k);
acc.1.push(v);
acc
});
Value::Record { cols, vals, span }
}
}
/// Format a duration in nanoseconds into a string
pub fn format_duration(duration: i64) -> String {
let (sign, duration) = if duration >= 0 {