From 191dacdd8b8d5c0da9ae8d8debbd413218fc8392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20N=2E=20Robalino?= Date: Sun, 21 Jul 2019 02:08:05 -0500 Subject: [PATCH] Introduced conversion to csv command. --- src/cli.rs | 1 + src/commands.rs | 1 + src/commands/from_csv.rs | 1 + src/commands/open.rs | 2 +- src/commands/save.rs | 9 ++++ src/commands/to_csv.rs | 65 +++++++++++++++++++++++ src/commands/to_json.rs | 1 + tests/commands_test.rs | 56 ++++++++++--------- tests/filters_test.rs | 10 ++++ tests/fixtures/formats/caco3_plastics.csv | 2 +- tests/fixtures/nuplayground/.gitignore | 2 +- tests/helpers/mod.rs | 22 ++++++++ 12 files changed, 145 insertions(+), 27 deletions(-) create mode 100644 src/commands/to_csv.rs diff --git a/src/cli.rs b/src/cli.rs index 55b02a11a..15b6a50b8 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -172,6 +172,7 @@ pub async fn cli() -> Result<(), Box> { command("reject", Box::new(reject::reject)), command("trim", Box::new(trim::trim)), command("to-array", Box::new(to_array::to_array)), + command("to-csv", Box::new(to_csv::to_csv)), command("to-json", Box::new(to_json::to_json)), command("to-toml", Box::new(to_toml::to_toml)), command("to-yaml", Box::new(to_yaml::to_yaml)), diff --git a/src/commands.rs b/src/commands.rs index dc1d9704a..e1ba9b793 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -34,6 +34,7 @@ crate mod split_row; crate mod sysinfo; crate mod table; crate mod to_array; +crate mod to_csv; crate mod to_json; crate mod to_toml; crate mod to_yaml; diff --git a/src/commands/from_csv.rs b/src/commands/from_csv.rs index b59013772..22381a35f 100644 --- a/src/commands/from_csv.rs +++ b/src/commands/from_csv.rs @@ -6,6 +6,7 @@ pub fn from_csv_string_to_value( s: String, span: impl Into, ) -> Result, Box> { + let mut reader = ReaderBuilder::new() .has_headers(false) .from_reader(s.as_bytes()); diff --git a/src/commands/open.rs b/src/commands/open.rs index 87411e9d3..5a4fa1474 100644 --- a/src/commands/open.rs +++ b/src/commands/open.rs @@ -189,7 +189,7 @@ pub fn fetch( }, Err(_) => { return Err(ShellError::labeled_error( - "File cound not be opened", + "File could not be opened", "file not found", span, )); diff --git a/src/commands/save.rs b/src/commands/save.rs index a9545651a..866f539f0 100644 --- a/src/commands/save.rs +++ b/src/commands/save.rs @@ -1,4 +1,5 @@ use crate::commands::command::SinkCommandArgs; +use crate::commands::to_csv::{to_string as to_csv_to_string, value_to_csv_value}; use crate::commands::to_json::value_to_json_value; use crate::commands::to_toml::value_to_toml_value; use crate::commands::to_yaml::value_to_yaml_value; @@ -37,6 +38,14 @@ pub fn save(args: SinkCommandArgs) -> Result<(), ShellError> { }; let contents = match full_path.extension() { + Some(x) if x == "csv" && !save_raw => { + if args.input.len() != 1 { + return Err(ShellError::string( + "saving to csv requires a single object (or use --raw)", + )); + } + to_csv_to_string(&value_to_csv_value(&args.input[0])).unwrap() + } Some(x) if x == "toml" && !save_raw => { if args.input.len() != 1 { return Err(ShellError::string( diff --git a/src/commands/to_csv.rs b/src/commands/to_csv.rs new file mode 100644 index 000000000..04a60590d --- /dev/null +++ b/src/commands/to_csv.rs @@ -0,0 +1,65 @@ +use crate::object::{Primitive, Value}; +use crate::prelude::*; +use log::debug; +use csv::WriterBuilder; + +pub fn value_to_csv_value(v: &Value) -> Value { + + debug!("value_to_csv_value(Value::Object(v)) where v = {:?}", v); + + match v { + Value::Primitive(Primitive::String(s)) => Value::Primitive(Primitive::String(s.clone())), + Value::Primitive(Primitive::Nothing) => Value::Primitive(Primitive::Nothing), + Value::Object(o) => Value::Object(o.clone()), + Value::List(l) => Value::List(l.clone()), + Value::Block(_) => Value::Primitive(Primitive::Nothing), + _ => Value::Primitive(Primitive::Nothing) + } +} + +pub fn to_string(v: &Value) -> Result> { + match v { + Value::List(_l) => return Ok(String::from("[list list]")), + Value::Object(o) => { + + debug!("to_csv:to_string(Value::Object(v)) where v = {:?}", v); + + let mut wtr = WriterBuilder::new().from_writer(vec![]); + let mut fields: VecDeque = VecDeque::new(); + let mut values: VecDeque = VecDeque::new(); + + for (k, v) in o.entries.iter() { + fields.push_back(k.clone()); + values.push_back(to_string(&v)?); + } + + wtr.write_record(fields).expect("can not write."); + wtr.write_record(values).expect("can not write."); + + return Ok(String::from_utf8(wtr.into_inner()?)?) + }, + Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()), + _ => return Err("Bad input".into()) + } +} + +pub fn to_csv(args: CommandArgs) -> Result { + let out = args.input; + let name_span = args.call_info.name_span; + Ok(out + .values + .map( + move |a| match to_string(&value_to_csv_value(&a.item)) { + + Ok(x) => { + ReturnSuccess::value(Value::Primitive(Primitive::String(x)).spanned(name_span)) + } + Err(_) => Err(ShellError::maybe_labeled_error( + "Can not convert to CSV string", + "can not convert piped data to CSV string", + name_span, + )), + }, + ) + .to_output_stream()) +} diff --git a/src/commands/to_json.rs b/src/commands/to_json.rs index 5cd4c913c..25f1c7187 100644 --- a/src/commands/to_json.rs +++ b/src/commands/to_json.rs @@ -1,5 +1,6 @@ use crate::object::{Primitive, Value}; use crate::prelude::*; +use log::trace; pub fn value_to_json_value(v: &Value) -> serde_json::Value { match v { diff --git a/tests/commands_test.rs b/tests/commands_test.rs index c25a02c58..dd9863e05 100644 --- a/tests/commands_test.rs +++ b/tests/commands_test.rs @@ -13,7 +13,7 @@ fn lines() { } #[test] -fn open_csv() { +fn open_can_parse_csv() { nu!( output, cwd("tests/fixtures/formats"), @@ -24,7 +24,7 @@ fn open_csv() { } #[test] -fn open_toml() { +fn open_can_parse_toml() { nu!( output, cwd("tests/fixtures/formats"), @@ -35,7 +35,7 @@ fn open_toml() { } #[test] -fn open_json() { +fn open_can_parse_json() { nu!(output, cwd("tests/fixtures/formats"), "open sgml_description.json | get glossary.GlossDiv.GlossList.GlossEntry.GlossSee | echo $it"); @@ -44,7 +44,7 @@ fn open_json() { } #[test] -fn open_xml() { +fn open_can_parse_xml() { nu!( output, cwd("tests/fixtures/formats"), @@ -58,7 +58,7 @@ fn open_xml() { } #[test] -fn open_ini() { +fn open_can_parse_ini() { nu!( output, cwd("tests/fixtures/formats"), @@ -76,11 +76,28 @@ fn open_error_if_file_not_found() { "open i_dont_exist.txt | echo $it" ); - assert!(output.contains("File cound not be opened")); + assert!(output.contains("File could not be opened")); } #[test] -fn rm() { +fn save_can_write_out_csv() { + let (playground_path, tests_dir) = h::setup_playground_for("save_test"); + + let full_path = format!("{}/{}", playground_path, tests_dir ); + let expected_file = format!("{}/{}", full_path , "cargo_sample.csv"); + + nu!( + _output, + cwd(&playground_path), + "open ../formats/cargo_sample.toml | inc package.version --minor | get package | save save_test/cargo_sample.csv" + ); + + let actual = h::file_contents(&expected_file); + assert!(actual.contains("[list list],A shell for the GitHub era,2018,ISC,nu,0.2.0")); +} + +#[test] +fn rm_can_remove_a_file() { let directory = "tests/fixtures/nuplayground"; let file = format!("{}/rm_test.txt", directory); @@ -92,16 +109,11 @@ fn rm() { } #[test] -fn can_remove_directory_contents_with_recursive_flag() { - let path = "tests/fixtures/nuplayground/rm_test"; - - if h::file_exists_at(&path) { - h::delete_directory_at(path) - } - h::create_directory_at(path); +fn rm_can_remove_directory_contents_with_recursive_flag() { + let (playground_path, tests_dir) = h::setup_playground_for("rm_test"); for f in ["yehuda.txt", "jonathan.txt", "andres.txt"].iter() { - h::create_file_at(&format!("{}/{}", path, f)); + h::create_file_at(&format!("{}/{}/{}", playground_path, tests_dir, f)); } nu!( @@ -110,23 +122,19 @@ fn can_remove_directory_contents_with_recursive_flag() { "rm rm_test --recursive" ); - assert!(!h::file_exists_at(&path)); + assert!(!h::file_exists_at(&format!("{}/{}", playground_path, tests_dir))); } #[test] fn rm_error_if_attempting_to_delete_a_directory_without_recursive_flag() { - let path = "tests/fixtures/nuplayground/rm_test_2"; - - if h::file_exists_at(&path) { - h::delete_directory_at(path) - } - h::create_directory_at(path); + let (playground_path, tests_dir) = h::setup_playground_for("rm_test_2"); + let full_path = format!("{}/{}", playground_path, tests_dir); nu_error!(output, cwd("tests/fixtures/nuplayground"), "rm rm_test_2"); - assert!(h::file_exists_at(&path)); + assert!(h::file_exists_at(&full_path)); assert!(output.contains("is a directory")); - h::delete_directory_at(path); + h::delete_directory_at(&full_path); } #[test] diff --git a/tests/filters_test.rs b/tests/filters_test.rs index 55a2d9ef8..d7552c583 100644 --- a/tests/filters_test.rs +++ b/tests/filters_test.rs @@ -2,6 +2,16 @@ mod helpers; use helpers::in_directory as cwd; + +#[test] +fn can_convert_table_to_csv_text_and_from_csv_text_back_into_table() { + nu!(output, + cwd("tests/fixtures/formats"), + "open caco3_plastics.csv | to-csv | from-csv | first 1 | get origin | echo $it"); + + assert_eq!(output, "SPAIN"); +} + #[test] fn can_convert_table_to_json_text_and_from_json_text_back_into_table() { nu!(output, diff --git a/tests/fixtures/formats/caco3_plastics.csv b/tests/fixtures/formats/caco3_plastics.csv index c4f2fdfc6..8b04b00c0 100644 --- a/tests/fixtures/formats/caco3_plastics.csv +++ b/tests/fixtures/formats/caco3_plastics.csv @@ -7,4 +7,4 @@ QUIMICA COMERCIAL QUIMICIAL CIA. LTDA.,SA REVERTE,2836500000,CARBONATO DE CALCIO PICA PLASTICOS INDUSTRIALES C.A.,OMYA ANDINA S.A,3824909999,CARBONATO DE CALCIO,COLOMBIA,01/01/1900,18/01/2016,"66,500.00","12,635.00","18,670.52",0.28 PLASTIQUIM S.A.,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYA CARB 1T CG BBS 1000,COLOMBIA,01/01/1900,25/10/2016,"33,000.00","6,270.00","9,999.00",0.30 QUIMICOS ANDINOS QUIMANDI S.A.,SIBELCO COLOMBIA SAS,3824909999,CARBONATO DE CALCIO RECUBIERTO,COLOMBIA,01/11/2016,03/11/2016,"52,000.00","8,944.00","13,039.05",0.25 -TIGRE ECUADOR S.A. ECUATIGRE,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYACARB 1T CG BPA 25 NO,COLOMBIA,01/01/1900,28/10/2016,"66,000.00","11,748.00","18,216.00",0.28 \ No newline at end of file +TIGRE ECUADOR S.A. ECUATIGRE,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYACARB 1T CG BPA 25 NO,COLOMBIA,01/01/1900,28/10/2016,"66,000.00","11,748.00","18,216.00",0.28 diff --git a/tests/fixtures/nuplayground/.gitignore b/tests/fixtures/nuplayground/.gitignore index 67db37f04..09fb65d85 100644 --- a/tests/fixtures/nuplayground/.gitignore +++ b/tests/fixtures/nuplayground/.gitignore @@ -1,2 +1,2 @@ -rm_test +*_test *.txt diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index e61dab7ec..ed54a69dd 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -2,6 +2,8 @@ pub use std::path::PathBuf; +use std::io::Read; + #[macro_export] macro_rules! nu { ($out:ident, $cwd:expr, $commands:expr) => { @@ -77,6 +79,26 @@ macro_rules! nu_error { }; } +pub fn setup_playground_for(topic: &str) -> (String, String) { + let home = "tests/fixtures/nuplayground"; + let full_path = format!("{}/{}", home, topic); + + if file_exists_at(&full_path) { + delete_directory_at(&full_path); + } + + create_directory_at(&full_path); + + (home.to_string(), topic.to_string()) +} + +pub fn file_contents(full_path: &str) -> String { + let mut file = std::fs::File::open(full_path).expect("can not open file"); + let mut contents = String::new(); + file.read_to_string(&mut contents).expect("can not read file"); + contents +} + pub fn create_file_at(full_path: &str) { std::fs::write(PathBuf::from(full_path), "fake data".as_bytes()).expect("can not create file"); }