diff --git a/Cargo.lock b/Cargo.lock index 6ef0a31b51..208abba75e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -992,6 +992,15 @@ dependencies = [ "libc", ] +[[package]] +name = "ical" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9f7215ad0d77e69644570dee000c7678a47ba7441062c1b5f918adde0d73cf" +dependencies = [ + "thiserror", +] + [[package]] name = "im" version = "15.0.0" @@ -1429,6 +1438,7 @@ dependencies = [ "dialoguer", "eml-parser", "glob", + "ical", "indexmap", "itertools", "lscolors", @@ -1447,6 +1457,7 @@ dependencies = [ "rayon", "regex", "serde", + "serde_ini", "serde_urlencoded", "serde_yaml", "sysinfo", @@ -1532,6 +1543,7 @@ dependencies = [ "chrono", "chrono-humanize", "im", + "indexmap", "miette", "nu-json", "serde", @@ -2164,6 +2176,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "result" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194d8e591e405d1eecf28819740abed6d719d1a2db87fc0bcdedee9a26d55560" + [[package]] name = "rust-argon2" version = "0.8.3" @@ -2232,6 +2250,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_ini" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb236687e2bb073a7521c021949be944641e671b8505a94069ca37b656c81139" +dependencies = [ + "result", + "serde", + "void", +] + [[package]] name = "serde_json" version = "1.0.71" @@ -2672,6 +2701,12 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "vte" version = "0.10.1" diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 73708e2171..e9924b35a7 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -39,9 +39,11 @@ meval = "0.2.0" serde = { version="1.0.123", features=["derive"] } serde_yaml = "0.8.16" serde_urlencoded = "0.7.0" +serde_ini = "0.2.0" eml-parser = "0.1.0" toml = "0.5.8" itertools = "0.10.0" +ical = "0.7.0" calamine = "0.18.0" rand = "0.8" diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index e5bc4a0a44..b634339684 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -60,6 +60,9 @@ pub fn create_default_context() -> EngineState { FromUrl, FromEml, FromOds, + FromIcs, + FromIni, + FromVcf, FromXlsx, Get, Griddle, diff --git a/crates/nu-command/src/formats/from/eml.rs b/crates/nu-command/src/formats/from/eml.rs index 629db00880..7a4d1a9cb3 100644 --- a/crates/nu-command/src/formats/from/eml.rs +++ b/crates/nu-command/src/formats/from/eml.rs @@ -230,21 +230,10 @@ fn from_eml( ); } - let (cols, vals) = collected - .into_iter() - .fold((vec![], vec![]), |mut acc, (k, v)| { - acc.0.push(k); - acc.1.push(v); - acc - }); - - let record = Value::Record { - cols, - vals, + Ok(PipelineData::Value(Value::from(Spanned { + item: collected, span: head, - }; - - Ok(PipelineData::Value(record)) + }))) } #[cfg(test)] diff --git a/crates/nu-command/src/formats/from/ics.rs b/crates/nu-command/src/formats/from/ics.rs new file mode 100644 index 0000000000..55c4f463c2 --- /dev/null +++ b/crates/nu-command/src/formats/from/ics.rs @@ -0,0 +1,327 @@ +extern crate ical; +use ical::parser::ical::component::*; +use ical::property::Property; +use indexmap::map::IndexMap; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{ + Category, Config, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, + Spanned, Value, +}; +use std::io::BufReader; + +#[derive(Clone)] +pub struct FromIcs; + +impl Command for FromIcs { + fn name(&self) -> &str { + "from ics" + } + + fn signature(&self) -> Signature { + Signature::build("from ics").category(Category::Formats) + } + + fn usage(&self) -> &str { + "Parse text as .ics and create table." + } + + fn run( + &self, + _engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let head = call.head; + let config = stack.get_config()?; + from_ics(input, head, &config) + } + + fn examples(&self) -> Vec { + vec![Example { + example: "'BEGIN:VCALENDAR +END:VCALENDAR' | from ics", + description: "Converts ics formatted string to table", + result: Some(Value::List { + vals: vec![Value::Record { + cols: vec![ + "properties".to_string(), + "events".to_string(), + "alarms".to_string(), + "to-Dos".to_string(), + "journals".to_string(), + "free-busys".to_string(), + "timezones".to_string(), + ], + vals: vec![ + Value::List { + vals: vec![], + span: Span::unknown(), + }, + Value::List { + vals: vec![], + span: Span::unknown(), + }, + Value::List { + vals: vec![], + span: Span::unknown(), + }, + Value::List { + vals: vec![], + span: Span::unknown(), + }, + Value::List { + vals: vec![], + span: Span::unknown(), + }, + Value::List { + vals: vec![], + span: Span::unknown(), + }, + Value::List { + vals: vec![], + span: Span::unknown(), + }, + ], + span: Span::unknown(), + }], + span: Span::unknown(), + }), + }] + } +} + +fn from_ics(input: PipelineData, head: Span, config: &Config) -> Result { + let input_string = input.collect_string("", config); + let input_bytes = input_string.as_bytes(); + let buf_reader = BufReader::new(input_bytes); + let parser = ical::IcalParser::new(buf_reader); + + let mut output = vec![]; + + for calendar in parser { + match calendar { + Ok(c) => output.push(calendar_to_value(c, head)), + Err(_) => output.push(Value::Error { + error: ShellError::UnsupportedInput( + "input cannot be parsed as .ics".to_string(), + head, + ), + }), + } + } + Ok(Value::List { + vals: output, + span: head, + } + .into_pipeline_data()) +} + +fn calendar_to_value(calendar: IcalCalendar, span: Span) -> Value { + let mut row = IndexMap::new(); + + row.insert( + "properties".to_string(), + properties_to_value(calendar.properties, span), + ); + row.insert("events".to_string(), events_to_value(calendar.events, span)); + row.insert("alarms".to_string(), alarms_to_value(calendar.alarms, span)); + row.insert("to-Dos".to_string(), todos_to_value(calendar.todos, span)); + row.insert( + "journals".to_string(), + journals_to_value(calendar.journals, span), + ); + row.insert( + "free-busys".to_string(), + free_busys_to_value(calendar.free_busys, span), + ); + row.insert( + "timezones".to_string(), + timezones_to_value(calendar.timezones, span), + ); + + Value::from(Spanned { item: row, span }) +} + +fn events_to_value(events: Vec, span: Span) -> Value { + Value::List { + vals: events + .into_iter() + .map(|event| { + let mut row = IndexMap::new(); + row.insert( + "properties".to_string(), + properties_to_value(event.properties, span), + ); + row.insert("alarms".to_string(), alarms_to_value(event.alarms, span)); + Value::from(Spanned { item: row, span }) + }) + .collect::>(), + span, + } +} + +fn alarms_to_value(alarms: Vec, span: Span) -> Value { + Value::List { + vals: alarms + .into_iter() + .map(|alarm| { + let mut row = IndexMap::new(); + row.insert( + "properties".to_string(), + properties_to_value(alarm.properties, span), + ); + Value::from(Spanned { item: row, span }) + }) + .collect::>(), + span, + } +} + +fn todos_to_value(todos: Vec, span: Span) -> Value { + Value::List { + vals: todos + .into_iter() + .map(|todo| { + let mut row = IndexMap::new(); + row.insert( + "properties".to_string(), + properties_to_value(todo.properties, span), + ); + row.insert("alarms".to_string(), alarms_to_value(todo.alarms, span)); + Value::from(Spanned { item: row, span }) + }) + .collect::>(), + span, + } +} + +fn journals_to_value(journals: Vec, span: Span) -> Value { + Value::List { + vals: journals + .into_iter() + .map(|journal| { + let mut row = IndexMap::new(); + row.insert( + "properties".to_string(), + properties_to_value(journal.properties, span), + ); + Value::from(Spanned { item: row, span }) + }) + .collect::>(), + span, + } +} + +fn free_busys_to_value(free_busys: Vec, span: Span) -> Value { + Value::List { + vals: free_busys + .into_iter() + .map(|free_busy| { + let mut row = IndexMap::new(); + row.insert( + "properties".to_string(), + properties_to_value(free_busy.properties, span), + ); + Value::from(Spanned { item: row, span }) + }) + .collect::>(), + span, + } +} + +fn timezones_to_value(timezones: Vec, span: Span) -> Value { + Value::List { + vals: timezones + .into_iter() + .map(|timezone| { + let mut row = IndexMap::new(); + row.insert( + "properties".to_string(), + properties_to_value(timezone.properties, span), + ); + row.insert( + "transitions".to_string(), + timezone_transitions_to_value(timezone.transitions, span), + ); + Value::from(Spanned { item: row, span }) + }) + .collect::>(), + span, + } +} + +fn timezone_transitions_to_value(transitions: Vec, span: Span) -> Value { + Value::List { + vals: transitions + .into_iter() + .map(|transition| { + let mut row = IndexMap::new(); + row.insert( + "properties".to_string(), + properties_to_value(transition.properties, span), + ); + Value::from(Spanned { item: row, span }) + }) + .collect::>(), + span, + } +} + +fn properties_to_value(properties: Vec, span: Span) -> Value { + Value::List { + vals: properties + .into_iter() + .map(|prop| { + let mut row = IndexMap::new(); + + let name = Value::String { + val: prop.name, + span, + }; + let value = match prop.value { + Some(val) => Value::String { val, span }, + None => Value::nothing(span), + }; + let params = match prop.params { + Some(param_list) => params_to_value(param_list, span), + None => Value::nothing(span), + }; + + row.insert("name".to_string(), name); + row.insert("value".to_string(), value); + row.insert("params".to_string(), params); + Value::from(Spanned { item: row, span }) + }) + .collect::>(), + span, + } +} + +fn params_to_value(params: Vec<(String, Vec)>, span: Span) -> Value { + let mut row = IndexMap::new(); + + for (param_name, param_values) in params { + let values: Vec = param_values + .into_iter() + .map(|val| Value::string(val, span)) + .collect(); + let values = Value::List { vals: values, span }; + row.insert(param_name, values); + } + + Value::from(Spanned { item: row, span }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(FromIcs {}) + } +} diff --git a/crates/nu-command/src/formats/from/ini.rs b/crates/nu-command/src/formats/from/ini.rs new file mode 100644 index 0000000000..40666dc0ef --- /dev/null +++ b/crates/nu-command/src/formats/from/ini.rs @@ -0,0 +1,109 @@ +use indexmap::map::IndexMap; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{ + Category, Config, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Value, +}; + +#[derive(Clone)] +pub struct FromIni; + +impl Command for FromIni { + fn name(&self) -> &str { + "from ini" + } + + fn signature(&self) -> Signature { + Signature::build("from ini").category(Category::Formats) + } + + fn usage(&self) -> &str { + "Parse text as .ini and create table" + } + + fn examples(&self) -> Vec { + vec![Example { + example: "'[foo] +a=1 +b=2' | from ini", + description: "Converts ini formatted string to table", + result: Some(Value::Record { + cols: vec!["foo".to_string()], + vals: vec![Value::Record { + cols: vec!["a".to_string(), "b".to_string()], + vals: vec![ + Value::String { + val: "1".to_string(), + span: Span::unknown(), + }, + Value::String { + val: "2".to_string(), + span: Span::unknown(), + }, + ], + span: Span::unknown(), + }], + span: Span::unknown(), + }), + }] + } + + fn run( + &self, + _engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let head = call.head; + let config = stack.get_config()?; + from_ini(input, head, &config) + } +} + +pub fn from_ini_string_to_value(s: String, span: Span) -> Result { + let v: Result>, serde_ini::de::Error> = + serde_ini::from_str(&s); + match v { + Ok(index_map) => { + let (cols, vals) = index_map + .into_iter() + .fold((vec![], vec![]), |mut acc, (k, v)| { + let (cols, vals) = v.into_iter().fold((vec![], vec![]), |mut acc, (k, v)| { + acc.0.push(k); + acc.1.push(Value::String { val: v, span }); + acc + }); + acc.0.push(k); + acc.1.push(Value::Record { cols, vals, span }); + acc + }); + Ok(Value::Record { cols, vals, span }) + } + Err(err) => Err(ShellError::UnsupportedInput( + format!("Could not load ini: {}", err), + span, + )), + } +} + +fn from_ini(input: PipelineData, head: Span, config: &Config) -> Result { + let concat_string = input.collect_string("", config); + + match from_ini_string_to_value(concat_string, head) { + Ok(x) => Ok(x.into_pipeline_data()), + Err(other) => Err(other), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(FromIni {}) + } +} diff --git a/crates/nu-command/src/formats/from/mod.rs b/crates/nu-command/src/formats/from/mod.rs index 9fab2206b6..4698e7556d 100644 --- a/crates/nu-command/src/formats/from/mod.rs +++ b/crates/nu-command/src/formats/from/mod.rs @@ -2,11 +2,14 @@ mod command; mod csv; mod delimited; mod eml; +mod ics; +mod ini; mod json; mod ods; mod toml; mod tsv; mod url; +mod vcf; mod xlsx; mod yaml; @@ -14,10 +17,13 @@ pub use self::csv::FromCsv; pub use self::toml::FromToml; pub use command::From; pub use eml::FromEml; +pub use ics::FromIcs; +pub use ini::FromIni; pub use json::FromJson; pub use ods::FromOds; pub use tsv::FromTsv; pub use url::FromUrl; +pub use vcf::FromVcf; pub use xlsx::FromXlsx; pub use yaml::FromYaml; pub use yaml::FromYml; diff --git a/crates/nu-command/src/formats/from/vcf.rs b/crates/nu-command/src/formats/from/vcf.rs new file mode 100644 index 0000000000..cf60f7ac65 --- /dev/null +++ b/crates/nu-command/src/formats/from/vcf.rs @@ -0,0 +1,211 @@ +use ical::parser::vcard::component::*; +use ical::property::Property; +use indexmap::map::IndexMap; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{ + Category, Config, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, + Spanned, Value, +}; + +#[derive(Clone)] +pub struct FromVcf; + +impl Command for FromVcf { + fn name(&self) -> &str { + "from vcf" + } + + fn signature(&self) -> Signature { + Signature::build("from vcf").category(Category::Formats) + } + + fn usage(&self) -> &str { + "Parse text as .vcf and create table." + } + + fn run( + &self, + _engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let head = call.head; + let config = stack.get_config()?; + from_vcf(input, head, &config) + } + + fn examples(&self) -> Vec { + vec![Example { + example: "'BEGIN:VCARD +N:Foo +FN:Bar +EMAIL:foo@bar.com +END:VCARD' | from vcf", + description: "Converts ics formatted string to table", + result: Some(Value::List { + vals: vec![Value::Record { + cols: vec!["properties".to_string()], + vals: vec![Value::List { + vals: vec![ + Value::Record { + cols: vec![ + "name".to_string(), + "value".to_string(), + "params".to_string(), + ], + vals: vec![ + Value::String { + val: "N".to_string(), + span: Span::unknown(), + }, + Value::String { + val: "Foo".to_string(), + span: Span::unknown(), + }, + Value::Nothing { + span: Span::unknown(), + }, + ], + span: Span::unknown(), + }, + Value::Record { + cols: vec![ + "name".to_string(), + "value".to_string(), + "params".to_string(), + ], + vals: vec![ + Value::String { + val: "FN".to_string(), + span: Span::unknown(), + }, + Value::String { + val: "Bar".to_string(), + span: Span::unknown(), + }, + Value::Nothing { + span: Span::unknown(), + }, + ], + span: Span::unknown(), + }, + Value::Record { + cols: vec![ + "name".to_string(), + "value".to_string(), + "params".to_string(), + ], + vals: vec![ + Value::String { + val: "EMAIL".to_string(), + span: Span::unknown(), + }, + Value::String { + val: "foo@bar.com".to_string(), + span: Span::unknown(), + }, + Value::Nothing { + span: Span::unknown(), + }, + ], + span: Span::unknown(), + }, + ], + span: Span::unknown(), + }], + span: Span::unknown(), + }], + span: Span::unknown(), + }), + }] + } +} + +fn from_vcf(input: PipelineData, head: Span, config: &Config) -> Result { + let input_string = input.collect_string("", config); + let input_bytes = input_string.as_bytes(); + let cursor = std::io::Cursor::new(input_bytes); + let parser = ical::VcardParser::new(cursor); + + let iter = parser.map(move |contact| match contact { + Ok(c) => contact_to_value(c, head), + Err(_) => Value::Error { + error: ShellError::UnsupportedInput("input cannot be parsed as .vcf".to_string(), head), + }, + }); + + let collected: Vec<_> = iter.collect(); + Ok(Value::List { + vals: collected, + span: head, + } + .into_pipeline_data()) +} + +fn contact_to_value(contact: VcardContact, span: Span) -> Value { + let mut row = IndexMap::new(); + row.insert( + "properties".to_string(), + properties_to_value(contact.properties, span), + ); + Value::from(Spanned { item: row, span }) +} + +fn properties_to_value(properties: Vec, span: Span) -> Value { + Value::List { + vals: properties + .into_iter() + .map(|prop| { + let mut row = IndexMap::new(); + + let name = Value::String { + val: prop.name, + span, + }; + let value = match prop.value { + Some(val) => Value::String { val, span }, + None => Value::Nothing { span }, + }; + let params = match prop.params { + Some(param_list) => params_to_value(param_list, span), + None => Value::Nothing { span }, + }; + + row.insert("name".to_string(), name); + row.insert("value".to_string(), value); + row.insert("params".to_string(), params); + Value::from(Spanned { item: row, span }) + }) + .collect::>(), + span, + } +} + +fn params_to_value(params: Vec<(String, Vec)>, span: Span) -> Value { + let mut row = IndexMap::new(); + + for (param_name, param_values) in params { + let values: Vec = param_values + .into_iter() + .map(|val| Value::string(val, span)) + .collect(); + let values = Value::List { vals: values, span }; + row.insert(param_name, values); + } + + Value::from(Spanned { item: row, span }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(FromVcf {}) + } +} diff --git a/crates/nu-command/src/math/utils.rs b/crates/nu-command/src/math/utils.rs index aef8f2ce21..13228b7857 100644 --- a/crates/nu-command/src/math/utils.rs +++ b/crates/nu-command/src/math/utils.rs @@ -1,6 +1,6 @@ +use indexmap::map::IndexMap; use nu_protocol::ast::Call; -use nu_protocol::{IntoPipelineData, PipelineData, ShellError, Span, Value}; -use std::collections::HashMap; +use nu_protocol::{IntoPipelineData, PipelineData, ShellError, Span, Spanned, Value}; pub fn run_with_function( call: &Call, @@ -22,7 +22,7 @@ fn helper_for_tables( ) -> Result { // If we are not dealing with Primitives, then perhaps we are dealing with a table // Create a key for each column name - let mut column_values = HashMap::new(); + let mut column_values = IndexMap::new(); for val in values { if let Value::Record { cols, vals, .. } = val { for (key, value) in cols.iter().zip(vals.iter()) { @@ -37,7 +37,7 @@ fn helper_for_tables( } } // The mathematical function operates over the columns of the table - let mut column_totals = HashMap::new(); + let mut column_totals = IndexMap::new(); for (col_name, col_vals) in column_values { if let Ok(out) = mf(&col_vals, &name) { column_totals.insert(col_name, out); @@ -49,19 +49,11 @@ fn helper_for_tables( name, )); } - let (cols, vals) = column_totals - .into_iter() - .fold((vec![], vec![]), |mut acc, (k, v)| { - acc.0.push(k); - acc.1.push(v); - acc - }); - Ok(Value::Record { - cols, - vals, + Ok(Value::from(Spanned { + item: column_totals, span: name, - }) + })) } pub fn calculate( diff --git a/crates/nu-protocol/Cargo.toml b/crates/nu-protocol/Cargo.toml index 84767c5bc7..e5aa910c51 100644 --- a/crates/nu-protocol/Cargo.toml +++ b/crates/nu-protocol/Cargo.toml @@ -10,6 +10,7 @@ thiserror = "1.0.29" miette = "3.0.0" serde = {version = "1.0.130", features = ["derive"]} chrono = { version="0.4.19", features=["serde"] } +indexmap = { version="1.7", features=["serde-1"] } chrono-humanize = "0.2.1" byte-unit = "4.0.9" im = "15.0.0" diff --git a/crates/nu-protocol/src/value/mod.rs b/crates/nu-protocol/src/value/mod.rs index 720581699d..d4c53156f2 100644 --- a/crates/nu-protocol/src/value/mod.rs +++ b/crates/nu-protocol/src/value/mod.rs @@ -6,6 +6,7 @@ mod unit; use chrono::{DateTime, FixedOffset}; use chrono_humanize::HumanTime; +use indexmap::map::IndexMap; pub use range::*; use serde::{Deserialize, Serialize}; pub use stream::*; @@ -1320,6 +1321,23 @@ impl From>> for Value { } } +/// Create a Value::Record from a spanned indexmap +impl From>> for Value { + fn from(input: Spanned>) -> Self { + let span = input.span; + let (cols, vals) = input + .item + .into_iter() + .fold((vec![], vec![]), |mut acc, (k, v)| { + acc.0.push(k); + acc.1.push(v); + acc + }); + + Value::Record { cols, vals, span } + } +} + /// Format a duration in nanoseconds into a string pub fn format_duration(duration: i64) -> String { let (sign, duration) = if duration >= 0 {