2019-08-19 07:16:39 +02:00
|
|
|
use crate::commands::WholeStreamCommand;
|
2019-09-05 18:23:42 +02:00
|
|
|
use crate::data::{Primitive, TaggedDictBuilder, Value};
|
2019-07-19 22:11:49 +02:00
|
|
|
use crate::prelude::*;
|
|
|
|
use csv::ReaderBuilder;
|
|
|
|
|
2019-08-19 07:16:39 +02:00
|
|
|
pub struct FromCSV;
|
|
|
|
|
2019-08-25 14:59:46 +02:00
|
|
|
#[derive(Deserialize)]
|
|
|
|
pub struct FromCSVArgs {
|
|
|
|
headerless: bool,
|
2019-11-08 14:11:04 +01:00
|
|
|
separator: Option<Tagged<Value>>,
|
2019-08-25 14:59:46 +02:00
|
|
|
}
|
2019-08-19 07:16:39 +02:00
|
|
|
|
2019-08-25 14:59:46 +02:00
|
|
|
impl WholeStreamCommand for FromCSV {
|
2019-08-19 07:16:39 +02:00
|
|
|
fn name(&self) -> &str {
|
|
|
|
"from-csv"
|
|
|
|
}
|
|
|
|
|
|
|
|
fn signature(&self) -> Signature {
|
2019-10-28 06:15:35 +01:00
|
|
|
Signature::build("from-csv")
|
2019-11-08 16:27:29 +01:00
|
|
|
.named(
|
|
|
|
"separator",
|
|
|
|
SyntaxShape::String,
|
|
|
|
"a character to separate columns, defaults to ','",
|
|
|
|
)
|
2019-10-28 06:15:35 +01:00
|
|
|
.switch("headerless", "don't treat the first row as column names")
|
2019-08-30 00:52:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
fn usage(&self) -> &str {
|
2019-11-11 12:01:21 +01:00
|
|
|
"Parse text as .csv and create table."
|
2019-08-25 14:59:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
fn run(
|
|
|
|
&self,
|
|
|
|
args: CommandArgs,
|
|
|
|
registry: &CommandRegistry,
|
|
|
|
) -> Result<OutputStream, ShellError> {
|
|
|
|
args.process(registry, from_csv)?.run()
|
2019-08-19 07:16:39 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-07-19 22:11:49 +02:00
|
|
|
pub fn from_csv_string_to_value(
|
|
|
|
s: String,
|
2019-08-25 14:59:46 +02:00
|
|
|
headerless: bool,
|
2019-11-08 14:11:04 +01:00
|
|
|
separator: char,
|
2019-08-05 10:54:29 +02:00
|
|
|
tag: impl Into<Tag>,
|
2019-08-21 08:39:57 +02:00
|
|
|
) -> Result<Tagged<Value>, csv::Error> {
|
2019-07-20 04:27:10 +02:00
|
|
|
let mut reader = ReaderBuilder::new()
|
2019-11-11 12:01:21 +01:00
|
|
|
.has_headers(!headerless)
|
2019-11-08 14:11:04 +01:00
|
|
|
.delimiter(separator as u8)
|
2019-07-20 04:27:10 +02:00
|
|
|
.from_reader(s.as_bytes());
|
2019-08-05 10:54:29 +02:00
|
|
|
let tag = tag.into();
|
2019-07-19 22:11:49 +02:00
|
|
|
|
2019-11-11 12:01:21 +01:00
|
|
|
let headers = if headerless {
|
|
|
|
(1..=reader.headers()?.len())
|
|
|
|
.map(|i| format!("Column{}", i))
|
|
|
|
.collect::<Vec<String>>()
|
|
|
|
} else {
|
|
|
|
reader.headers()?.iter().map(String::from).collect()
|
|
|
|
};
|
2019-07-19 22:11:49 +02:00
|
|
|
|
2019-11-11 12:01:21 +01:00
|
|
|
let mut rows = vec![];
|
|
|
|
for row in reader.records() {
|
|
|
|
let mut tagged_row = TaggedDictBuilder::new(&tag);
|
|
|
|
for (value, header) in row?.iter().zip(headers.iter()) {
|
|
|
|
tagged_row.insert_tagged(
|
|
|
|
header,
|
|
|
|
Value::Primitive(Primitive::String(String::from(value))).tagged(&tag),
|
|
|
|
)
|
2019-07-19 22:11:49 +02:00
|
|
|
}
|
2019-11-11 12:01:21 +01:00
|
|
|
rows.push(tagged_row.into_tagged_value());
|
2019-07-19 22:11:49 +02:00
|
|
|
}
|
|
|
|
|
2019-10-13 06:12:43 +02:00
|
|
|
Ok(Value::Table(rows).tagged(&tag))
|
2019-07-19 22:11:49 +02:00
|
|
|
}
|
|
|
|
|
2019-08-25 14:59:46 +02:00
|
|
|
fn from_csv(
|
|
|
|
FromCSVArgs {
|
2019-11-11 12:01:21 +01:00
|
|
|
headerless,
|
2019-11-08 14:11:04 +01:00
|
|
|
separator,
|
2019-08-25 14:59:46 +02:00
|
|
|
}: FromCSVArgs,
|
|
|
|
RunnableContext { input, name, .. }: RunnableContext,
|
|
|
|
) -> Result<OutputStream, ShellError> {
|
2019-09-14 18:30:24 +02:00
|
|
|
let name_tag = name;
|
2019-11-08 14:11:04 +01:00
|
|
|
let sep = match separator {
|
2019-11-08 16:27:29 +01:00
|
|
|
Some(Tagged {
|
|
|
|
item: Value::Primitive(Primitive::String(s)),
|
|
|
|
tag,
|
|
|
|
..
|
|
|
|
}) => {
|
2019-11-08 14:11:04 +01:00
|
|
|
let vec_s: Vec<char> = s.chars().collect();
|
|
|
|
if vec_s.len() != 1 {
|
|
|
|
return Err(ShellError::labeled_error(
|
|
|
|
"Expected a single separator char from --separator",
|
|
|
|
"requires a single character string input",
|
|
|
|
tag,
|
2019-11-08 16:27:29 +01:00
|
|
|
));
|
2019-11-08 14:11:04 +01:00
|
|
|
};
|
|
|
|
vec_s[0]
|
|
|
|
}
|
2019-11-08 16:27:29 +01:00
|
|
|
_ => ',',
|
2019-11-08 14:11:04 +01:00
|
|
|
};
|
2019-07-19 22:11:49 +02:00
|
|
|
|
2019-09-26 02:22:17 +02:00
|
|
|
let stream = async_stream! {
|
2019-08-21 08:39:57 +02:00
|
|
|
let values: Vec<Tagged<Value>> = input.values.collect().await;
|
|
|
|
|
|
|
|
let mut concat_string = String::new();
|
|
|
|
let mut latest_tag: Option<Tag> = None;
|
|
|
|
|
|
|
|
for value in values {
|
|
|
|
let value_tag = value.tag();
|
2019-10-13 06:12:43 +02:00
|
|
|
latest_tag = Some(value_tag.clone());
|
2019-08-21 08:39:57 +02:00
|
|
|
match value.item {
|
2019-08-01 03:58:42 +02:00
|
|
|
Value::Primitive(Primitive::String(s)) => {
|
2019-08-21 08:39:57 +02:00
|
|
|
concat_string.push_str(&s);
|
|
|
|
concat_string.push_str("\n");
|
2019-08-01 03:58:42 +02:00
|
|
|
}
|
2019-08-21 08:39:57 +02:00
|
|
|
_ => yield Err(ShellError::labeled_error_with_secondary(
|
2019-08-05 10:54:29 +02:00
|
|
|
"Expected a string from pipeline",
|
|
|
|
"requires string input",
|
2019-10-13 06:12:43 +02:00
|
|
|
name_tag.clone(),
|
2019-08-05 10:54:29 +02:00
|
|
|
"value originates from here",
|
2019-10-13 06:12:43 +02:00
|
|
|
value_tag.clone(),
|
2019-07-19 22:11:49 +02:00
|
|
|
)),
|
2019-08-21 08:39:57 +02:00
|
|
|
|
2019-08-01 03:58:42 +02:00
|
|
|
}
|
2019-08-21 08:39:57 +02:00
|
|
|
}
|
|
|
|
|
2019-11-11 12:01:21 +01:00
|
|
|
match from_csv_string_to_value(concat_string, headerless, sep, name_tag.clone()) {
|
2019-08-24 09:38:38 +02:00
|
|
|
Ok(x) => match x {
|
2019-09-05 18:23:42 +02:00
|
|
|
Tagged { item: Value::Table(list), .. } => {
|
2019-08-24 09:38:38 +02:00
|
|
|
for l in list {
|
|
|
|
yield ReturnSuccess::value(l);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
x => yield ReturnSuccess::value(x),
|
|
|
|
},
|
2019-08-21 08:39:57 +02:00
|
|
|
Err(_) => if let Some(last_tag) = latest_tag {
|
|
|
|
yield Err(ShellError::labeled_error_with_secondary(
|
|
|
|
"Could not parse as CSV",
|
|
|
|
"input cannot be parsed as CSV",
|
2019-10-13 06:12:43 +02:00
|
|
|
name_tag.clone(),
|
2019-08-21 08:39:57 +02:00
|
|
|
"value originates from here",
|
2019-10-13 06:12:43 +02:00
|
|
|
last_tag.clone(),
|
2019-08-21 08:39:57 +02:00
|
|
|
))
|
|
|
|
} ,
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
Ok(stream.to_output_stream())
|
2019-07-19 22:11:49 +02:00
|
|
|
}
|