Merge pull request #200 from androbtech/to_csv_and_refactorings

Introduced conversion to csv command.
This commit is contained in:
Jonathan Turner 2019-07-21 19:23:58 +12:00 committed by GitHub
commit d1ebcaab77
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 145 additions and 27 deletions

View File

@ -172,6 +172,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
command("reject", Box::new(reject::reject)),
command("trim", Box::new(trim::trim)),
command("to-array", Box::new(to_array::to_array)),
command("to-csv", Box::new(to_csv::to_csv)),
command("to-json", Box::new(to_json::to_json)),
command("to-toml", Box::new(to_toml::to_toml)),
command("to-yaml", Box::new(to_yaml::to_yaml)),

View File

@ -34,6 +34,7 @@ crate mod split_row;
crate mod sysinfo;
crate mod table;
crate mod to_array;
crate mod to_csv;
crate mod to_json;
crate mod to_toml;
crate mod to_yaml;

View File

@ -6,6 +6,7 @@ pub fn from_csv_string_to_value(
s: String,
span: impl Into<Span>,
) -> Result<Spanned<Value>, Box<dyn std::error::Error>> {
let mut reader = ReaderBuilder::new()
.has_headers(false)
.from_reader(s.as_bytes());

View File

@ -189,7 +189,7 @@ pub fn fetch(
},
Err(_) => {
return Err(ShellError::labeled_error(
"File cound not be opened",
"File could not be opened",
"file not found",
span,
));

View File

@ -1,4 +1,5 @@
use crate::commands::command::SinkCommandArgs;
use crate::commands::to_csv::{to_string as to_csv_to_string, value_to_csv_value};
use crate::commands::to_json::value_to_json_value;
use crate::commands::to_toml::value_to_toml_value;
use crate::commands::to_yaml::value_to_yaml_value;
@ -37,6 +38,14 @@ pub fn save(args: SinkCommandArgs) -> Result<(), ShellError> {
};
let contents = match full_path.extension() {
Some(x) if x == "csv" && !save_raw => {
if args.input.len() != 1 {
return Err(ShellError::string(
"saving to csv requires a single object (or use --raw)",
));
}
to_csv_to_string(&value_to_csv_value(&args.input[0])).unwrap()
}
Some(x) if x == "toml" && !save_raw => {
if args.input.len() != 1 {
return Err(ShellError::string(

65
src/commands/to_csv.rs Normal file
View File

@ -0,0 +1,65 @@
use crate::object::{Primitive, Value};
use crate::prelude::*;
use log::debug;
use csv::WriterBuilder;
pub fn value_to_csv_value(v: &Value) -> Value {
debug!("value_to_csv_value(Value::Object(v)) where v = {:?}", v);
match v {
Value::Primitive(Primitive::String(s)) => Value::Primitive(Primitive::String(s.clone())),
Value::Primitive(Primitive::Nothing) => Value::Primitive(Primitive::Nothing),
Value::Object(o) => Value::Object(o.clone()),
Value::List(l) => Value::List(l.clone()),
Value::Block(_) => Value::Primitive(Primitive::Nothing),
_ => Value::Primitive(Primitive::Nothing)
}
}
pub fn to_string(v: &Value) -> Result<String, Box<dyn std::error::Error>> {
match v {
Value::List(_l) => return Ok(String::from("[list list]")),
Value::Object(o) => {
debug!("to_csv:to_string(Value::Object(v)) where v = {:?}", v);
let mut wtr = WriterBuilder::new().from_writer(vec![]);
let mut fields: VecDeque<String> = VecDeque::new();
let mut values: VecDeque<String> = VecDeque::new();
for (k, v) in o.entries.iter() {
fields.push_back(k.clone());
values.push_back(to_string(&v)?);
}
wtr.write_record(fields).expect("can not write.");
wtr.write_record(values).expect("can not write.");
return Ok(String::from_utf8(wtr.into_inner()?)?)
},
Value::Primitive(Primitive::String(s)) => return Ok(s.to_string()),
_ => return Err("Bad input".into())
}
}
pub fn to_csv(args: CommandArgs) -> Result<OutputStream, ShellError> {
let out = args.input;
let name_span = args.call_info.name_span;
Ok(out
.values
.map(
move |a| match to_string(&value_to_csv_value(&a.item)) {
Ok(x) => {
ReturnSuccess::value(Value::Primitive(Primitive::String(x)).spanned(name_span))
}
Err(_) => Err(ShellError::maybe_labeled_error(
"Can not convert to CSV string",
"can not convert piped data to CSV string",
name_span,
)),
},
)
.to_output_stream())
}

View File

@ -1,5 +1,6 @@
use crate::object::{Primitive, Value};
use crate::prelude::*;
use log::trace;
pub fn value_to_json_value(v: &Value) -> serde_json::Value {
match v {

View File

@ -13,7 +13,7 @@ fn lines() {
}
#[test]
fn open_csv() {
fn open_can_parse_csv() {
nu!(
output,
cwd("tests/fixtures/formats"),
@ -24,7 +24,7 @@ fn open_csv() {
}
#[test]
fn open_toml() {
fn open_can_parse_toml() {
nu!(
output,
cwd("tests/fixtures/formats"),
@ -35,7 +35,7 @@ fn open_toml() {
}
#[test]
fn open_json() {
fn open_can_parse_json() {
nu!(output,
cwd("tests/fixtures/formats"),
"open sgml_description.json | get glossary.GlossDiv.GlossList.GlossEntry.GlossSee | echo $it");
@ -44,7 +44,7 @@ fn open_json() {
}
#[test]
fn open_xml() {
fn open_can_parse_xml() {
nu!(
output,
cwd("tests/fixtures/formats"),
@ -58,7 +58,7 @@ fn open_xml() {
}
#[test]
fn open_ini() {
fn open_can_parse_ini() {
nu!(
output,
cwd("tests/fixtures/formats"),
@ -76,11 +76,28 @@ fn open_error_if_file_not_found() {
"open i_dont_exist.txt | echo $it"
);
assert!(output.contains("File cound not be opened"));
assert!(output.contains("File could not be opened"));
}
#[test]
fn rm() {
fn save_can_write_out_csv() {
let (playground_path, tests_dir) = h::setup_playground_for("save_test");
let full_path = format!("{}/{}", playground_path, tests_dir );
let expected_file = format!("{}/{}", full_path , "cargo_sample.csv");
nu!(
_output,
cwd(&playground_path),
"open ../formats/cargo_sample.toml | inc package.version --minor | get package | save save_test/cargo_sample.csv"
);
let actual = h::file_contents(&expected_file);
assert!(actual.contains("[list list],A shell for the GitHub era,2018,ISC,nu,0.2.0"));
}
#[test]
fn rm_can_remove_a_file() {
let directory = "tests/fixtures/nuplayground";
let file = format!("{}/rm_test.txt", directory);
@ -92,16 +109,11 @@ fn rm() {
}
#[test]
fn can_remove_directory_contents_with_recursive_flag() {
let path = "tests/fixtures/nuplayground/rm_test";
if h::file_exists_at(&path) {
h::delete_directory_at(path)
}
h::create_directory_at(path);
fn rm_can_remove_directory_contents_with_recursive_flag() {
let (playground_path, tests_dir) = h::setup_playground_for("rm_test");
for f in ["yehuda.txt", "jonathan.txt", "andres.txt"].iter() {
h::create_file_at(&format!("{}/{}", path, f));
h::create_file_at(&format!("{}/{}/{}", playground_path, tests_dir, f));
}
nu!(
@ -110,23 +122,19 @@ fn can_remove_directory_contents_with_recursive_flag() {
"rm rm_test --recursive"
);
assert!(!h::file_exists_at(&path));
assert!(!h::file_exists_at(&format!("{}/{}", playground_path, tests_dir)));
}
#[test]
fn rm_error_if_attempting_to_delete_a_directory_without_recursive_flag() {
let path = "tests/fixtures/nuplayground/rm_test_2";
if h::file_exists_at(&path) {
h::delete_directory_at(path)
}
h::create_directory_at(path);
let (playground_path, tests_dir) = h::setup_playground_for("rm_test_2");
let full_path = format!("{}/{}", playground_path, tests_dir);
nu_error!(output, cwd("tests/fixtures/nuplayground"), "rm rm_test_2");
assert!(h::file_exists_at(&path));
assert!(h::file_exists_at(&full_path));
assert!(output.contains("is a directory"));
h::delete_directory_at(path);
h::delete_directory_at(&full_path);
}
#[test]

View File

@ -2,6 +2,16 @@ mod helpers;
use helpers::in_directory as cwd;
#[test]
fn can_convert_table_to_csv_text_and_from_csv_text_back_into_table() {
nu!(output,
cwd("tests/fixtures/formats"),
"open caco3_plastics.csv | to-csv | from-csv | first 1 | get origin | echo $it");
assert_eq!(output, "SPAIN");
}
#[test]
fn can_convert_table_to_json_text_and_from_json_text_back_into_table() {
nu!(output,

View File

@ -7,4 +7,4 @@ QUIMICA COMERCIAL QUIMICIAL CIA. LTDA.,SA REVERTE,2836500000,CARBONATO DE CALCIO
PICA PLASTICOS INDUSTRIALES C.A.,OMYA ANDINA S.A,3824909999,CARBONATO DE CALCIO,COLOMBIA,01/01/1900,18/01/2016,"66,500.00","12,635.00","18,670.52",0.28
PLASTIQUIM S.A.,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYA CARB 1T CG BBS 1000,COLOMBIA,01/01/1900,25/10/2016,"33,000.00","6,270.00","9,999.00",0.30
QUIMICOS ANDINOS QUIMANDI S.A.,SIBELCO COLOMBIA SAS,3824909999,CARBONATO DE CALCIO RECUBIERTO,COLOMBIA,01/11/2016,03/11/2016,"52,000.00","8,944.00","13,039.05",0.25
TIGRE ECUADOR S.A. ECUATIGRE,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYACARB 1T CG BPA 25 NO,COLOMBIA,01/01/1900,28/10/2016,"66,000.00","11,748.00","18,216.00",0.28
TIGRE ECUADOR S.A. ECUATIGRE,OMYA ANDINA S.A NIT 830.027.386-6,3824909999,CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYACARB 1T CG BPA 25 NO,COLOMBIA,01/01/1900,28/10/2016,"66,000.00","11,748.00","18,216.00",0.28

1 importer shipper tariff_item name origin shipped_at arrived_at net_weight fob_price cif_price cif_per_net_weight
7 PICA PLASTICOS INDUSTRIALES C.A. OMYA ANDINA S.A 3824909999 CARBONATO DE CALCIO COLOMBIA 01/01/1900 18/01/2016 66,500.00 12,635.00 18,670.52 0.28
8 PLASTIQUIM S.A. OMYA ANDINA S.A NIT 830.027.386-6 3824909999 CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYA CARB 1T CG BBS 1000 COLOMBIA 01/01/1900 25/10/2016 33,000.00 6,270.00 9,999.00 0.30
9 QUIMICOS ANDINOS QUIMANDI S.A. SIBELCO COLOMBIA SAS 3824909999 CARBONATO DE CALCIO RECUBIERTO COLOMBIA 01/11/2016 03/11/2016 52,000.00 8,944.00 13,039.05 0.25
10 TIGRE ECUADOR S.A. ECUATIGRE OMYA ANDINA S.A NIT 830.027.386-6 3824909999 CARBONATO DE CALCIO RECUBIERTO CON ACIDO ESTEARICO OMYACARB 1T CG BPA 25 NO COLOMBIA 01/01/1900 28/10/2016 66,000.00 11,748.00 18,216.00 0.28

View File

@ -1,2 +1,2 @@
rm_test
*_test
*.txt

View File

@ -2,6 +2,8 @@
pub use std::path::PathBuf;
use std::io::Read;
#[macro_export]
macro_rules! nu {
($out:ident, $cwd:expr, $commands:expr) => {
@ -77,6 +79,26 @@ macro_rules! nu_error {
};
}
pub fn setup_playground_for(topic: &str) -> (String, String) {
let home = "tests/fixtures/nuplayground";
let full_path = format!("{}/{}", home, topic);
if file_exists_at(&full_path) {
delete_directory_at(&full_path);
}
create_directory_at(&full_path);
(home.to_string(), topic.to_string())
}
pub fn file_contents(full_path: &str) -> String {
let mut file = std::fs::File::open(full_path).expect("can not open file");
let mut contents = String::new();
file.read_to_string(&mut contents).expect("can not read file");
contents
}
pub fn create_file_at(full_path: &str) {
std::fs::write(PathBuf::from(full_path), "fake data".as_bytes()).expect("can not create file");
}