extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2024-12-23 23:49:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

This is part of on-going work with capabilities when working with

Browse Source 
tables and able to work with them for data processing & viewing
purposes. At the moment, certain ways to process said tables we
are able to view a histogram of a given column.

As usage matures, we may find certain core commands that could
be used ergonomically when working with tables on Nu.
This commit is contained in:
Andrés N. Robalino 2019-11-12 03:38:55 -05:00
parent 3163b0d362
commit 00b3c2036a
7 changed files with 147 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -256,6 +256,7 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
| format pattern | Format table row data as a string following the given pattern |
| get column-or-column-path | Open column and get data from the corresponding cells |
| group-by column | Creates a new table with the data from the table rows grouped by the column given |
| histogram column ...column-names | Creates a new table with a histogram based on the column name passed in, optionally give the frequency column name
| inc (column-or-column-path) | Increment a value or version. Optionally use the column of a table |
| insert column-or-column-path value | Insert a new column to the table |
| last amount | Show only the last number of rows |
@ -267,6 +268,7 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
| reverse | Reverses the table. |
| skip amount | Skip a number of rows |
| skip-while condition | Skips rows while the condition matches. |
| split-by column | Creates a new table with the data from the inner tables splitted by the column given |
| sort-by ...columns | Sort by the given columns |
| str (column) | Apply string function. Optionally use the column of a table |
| sum | Sum a column of values |

4
src/cli.rs
View File

@ -301,6 +301,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
whole_stream_command(FromYML),
whole_stream_command(Pick),
whole_stream_command(Get),
whole_stream_command(Histogram),
per_item_command(Remove),
per_item_command(Fetch),
per_item_command(Open),
@ -320,6 +321,7 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
per_item_command(Mkdir),
per_item_command(Move),
whole_stream_command(Save),
whole_stream_command(SplitBy),
whole_stream_command(Table),
whole_stream_command(Version),
whole_stream_command(Which),
@ -328,12 +330,10 @@ pub async fn cli() -> Result<(), Box<dyn Error>> {
cfg_if::cfg_if! {
if #[cfg(data_processing_primitives)] {
context.add_commands(vec![
whole_stream_command(SplitBy),
whole_stream_command(ReduceBy),
whole_stream_command(EvaluateBy),
whole_stream_command(TSortBy),
whole_stream_command(MapMaxBy),
whole_stream_command(Histogram),
]);
}
}

44
src/commands.rs
View File

@ -16,6 +16,8 @@ pub(crate) mod debug;
pub(crate) mod echo;
pub(crate) mod enter;
pub(crate) mod env;
#[allow(unused)]
pub(crate) mod evaluate_by;
pub(crate) mod exit;
pub(crate) mod fetch;
pub(crate) mod first;
@ -33,10 +35,13 @@ pub(crate) mod from_yaml;
pub(crate) mod get;
pub(crate) mod group_by;
pub(crate) mod help;
pub(crate) mod histogram;
pub(crate) mod history;
pub(crate) mod last;
pub(crate) mod lines;
pub(crate) mod ls;
#[allow(unused)]
pub(crate) mod map_max_by;
pub(crate) mod mkdir;
pub(crate) mod mv;
pub(crate) mod next;
@ -49,6 +54,8 @@ pub(crate) mod post;
pub(crate) mod prepend;
pub(crate) mod prev;
pub(crate) mod pwd;
#[allow(unused)]
pub(crate) mod reduce_by;
pub(crate) mod reject;
pub(crate) mod reverse;
pub(crate) mod rm;
@ -57,20 +64,11 @@ pub(crate) mod shells;
pub(crate) mod size;
pub(crate) mod skip_while;
pub(crate) mod sort_by;
cfg_if::cfg_if! {
if #[cfg(data_processing_primitives)] {
pub(crate) mod split_by;
pub(crate) mod reduce_by;
pub(crate) mod evaluate_by;
pub(crate) mod t_sort_by;
pub(crate) mod map_max_by;
pub(crate) mod histogram;
}
}
pub(crate) mod split_by;
pub(crate) mod split_column;
pub(crate) mod split_row;
#[allow(unused)]
pub(crate) mod t_sort_by;
pub(crate) mod table;
pub(crate) mod tags;
pub(crate) mod to_bson;
@ -103,6 +101,8 @@ pub(crate) use debug::Debug;
pub(crate) use echo::Echo;
pub(crate) use enter::Enter;
pub(crate) use env::Env;
#[allow(unused)]
pub(crate) use evaluate_by::EvaluateBy;
pub(crate) use exit::Exit;
pub(crate) use fetch::Fetch;
pub(crate) use first::First;
@ -122,10 +122,13 @@ pub(crate) use from_yaml::FromYML;
pub(crate) use get::Get;
pub(crate) use group_by::GroupBy;
pub(crate) use help::Help;
pub(crate) use histogram::Histogram;
pub(crate) use history::History;
pub(crate) use last::Last;
pub(crate) use lines::Lines;
pub(crate) use ls::LS;
#[allow(unused)]
pub(crate) use map_max_by::MapMaxBy;
pub(crate) use mkdir::Mkdir;
pub(crate) use mv::Move;
pub(crate) use next::Next;
@ -137,6 +140,8 @@ pub(crate) use post::Post;
pub(crate) use prepend::Prepend;
pub(crate) use prev::Previous;
pub(crate) use pwd::PWD;
#[allow(unused)]
pub(crate) use reduce_by::ReduceBy;
pub(crate) use reject::Reject;
pub(crate) use reverse::Reverse;
pub(crate) use rm::Remove;
@ -145,20 +150,11 @@ pub(crate) use shells::Shells;
pub(crate) use size::Size;
pub(crate) use skip_while::SkipWhile;
pub(crate) use sort_by::SortBy;
cfg_if::cfg_if! {
if #[cfg(data_processing_primitives)] {
pub(crate) use split_by::SplitBy;
pub(crate) use reduce_by::ReduceBy;
pub(crate) use evaluate_by::EvaluateBy;
pub(crate) use t_sort_by::TSortBy;
pub(crate) use map_max_by::MapMaxBy;
pub(crate) use histogram::Histogram;
}
}
pub(crate) use split_by::SplitBy;
pub(crate) use split_column::SplitColumn;
pub(crate) use split_row::SplitRow;
#[allow(unused)]
pub(crate) use t_sort_by::TSortBy;
pub(crate) use table::Table;
pub(crate) use tags::Tags;
pub(crate) use to_bson::ToBSON;

97
src/commands/histogram.rs
View File

@ -1,10 +1,10 @@
use crate::commands::WholeStreamCommand;
use crate::commands::evaluate_by::evaluate;
use crate::commands::group_by::group;
use crate::commands::map_max_by::map_max;
use crate::commands::reduce_by::reduce;
use crate::commands::t_sort_by::columns_sorted;
use crate::commands::t_sort_by::t_sort;
use crate::commands::evaluate_by::evaluate;
use crate::commands::reduce_by::reduce;
use crate::commands::map_max_by::map_max;
use crate::commands::WholeStreamCommand;
use crate::data::TaggedDictBuilder;
use crate::errors::ShellError;
use crate::prelude::*;
@ -15,6 +15,7 @@ pub struct Histogram;
#[derive(Deserialize)]
pub struct HistogramArgs {
column_name: Tagged<String>,
rest: Vec<Tagged<String>>,
}
impl WholeStreamCommand for Histogram {
@ -23,11 +24,16 @@ impl WholeStreamCommand for Histogram {
}
fn signature(&self) -> Signature {
Signature::build("histogram").required(
"column_name",
SyntaxShape::String,
"the name of the column to graph by",
)
Signature::build("histogram")
.required(
"column_name",
SyntaxShape::String,
"the name of the column to graph by",
)
.rest(
SyntaxShape::Member,
"column name to give the histogram's frequency column",
)
}
fn usage(&self) -> &str {
@ -44,7 +50,7 @@ impl WholeStreamCommand for Histogram {
}
pub fn histogram(
HistogramArgs { column_name }: HistogramArgs,
HistogramArgs { column_name, rest }: HistogramArgs,
RunnableContext { input, name, .. }: RunnableContext,
) -> Result<OutputStream, ShellError> {
let stream = async_stream! {
@ -68,13 +74,24 @@ pub fn histogram(
let mut idx = 0;
let column_names_supplied: Vec<_> = rest.iter().map(|f| f.item.clone()).collect();
let frequency_column_name = if column_names_supplied.is_empty() {
"frecuency".to_string()
} else {
column_names_supplied[0].clone()
};
let column = (*column_name).clone();
if let Tagged { item: Value::Table(start), .. } = datasets.get(0).unwrap() {
for percentage in start.into_iter() {
let mut fact = TaggedDictBuilder::new(&name);
fact.insert_tagged("committer", group_labels.get(idx).unwrap().clone());
fact.insert_tagged(&column, group_labels.get(idx).unwrap().clone());
if let Tagged { item: Value::Primitive(Primitive::Int(ref num)), .. } = percentage.clone() {
fact.insert("activity", std::iter::repeat("*").take(num.to_i32().unwrap() as usize).collect::<String>());
fact.insert(&frequency_column_name, std::iter::repeat("*").take(num.to_i32().unwrap() as usize).collect::<String>());
}
idx = idx + 1;
@ -104,38 +121,36 @@ fn percentages(
} => {
let datasets: Vec<_> = datasets
.into_iter()
.map(|subsets| {
match subsets {
Tagged {
item: Value::Table(data),
..
} => {
let data = data
.into_iter()
.map(|d| match d {
Tagged {
item: Value::Primitive(Primitive::Int(n)),
..
} => {
let max = match max {
Tagged {
item: Value::Primitive(Primitive::Int(ref maxima)),
..
} => maxima.to_i32().unwrap(),
_ => 0,
};
.map(|subsets| match subsets {
Tagged {
item: Value::Table(data),
..
} => {
let data = data
.into_iter()
.map(|d| match d {
Tagged {
item: Value::Primitive(Primitive::Int(n)),
..
} => {
let max = match max {
Tagged {
item: Value::Primitive(Primitive::Int(ref maxima)),
..
} => maxima.to_i32().unwrap(),
_ => 0,
};
let n = { n.to_i32().unwrap() * 100 / max };
let n = { n.to_i32().unwrap() * 100 / max };
Value::number(n).tagged(&tag)
}
_ => Value::number(0).tagged(&tag),
})
.collect::<Vec<_>>();
Value::Table(data).tagged(&tag)
}
_ => Value::Table(vec![]).tagged(&tag),
Value::number(n).tagged(&tag)
}
_ => Value::number(0).tagged(&tag),
})
.collect::<Vec<_>>();
Value::Table(data).tagged(&tag)
}
_ => Value::Table(vec![]).tagged(&tag),
})
.collect();

40
src/commands/map_max_by.rs
View File

@ -81,29 +81,27 @@ pub fn map_max(
} => {
let datasets: Vec<_> = datasets
.into_iter()
.map(|subsets| {
match subsets {
Tagged {
item: Value::Table(data),
..
} => {
let data = data.into_iter().fold(0, |acc, value| match value {
Tagged {
item: Value::Primitive(Primitive::Int(n)),
..
} => {
if n.to_i32().unwrap() > acc {
n.to_i32().unwrap()
} else {
acc
}
.map(|subsets| match subsets {
Tagged {
item: Value::Table(data),
..
} => {
let data = data.into_iter().fold(0, |acc, value| match value {
Tagged {
item: Value::Primitive(Primitive::Int(n)),
..
} => {
if n.to_i32().unwrap() > acc {
n.to_i32().unwrap()
} else {
acc
}
_ => acc,
});
Value::number(data).tagged(&tag)
}
_ => Value::number(0).tagged(&tag),
}
_ => acc,
});
Value::number(data).tagged(&tag)
}
_ => Value::number(0).tagged(&tag),
})
.collect();

38
src/commands/t_sort_by.rs
View File

@ -57,25 +57,25 @@ fn t_sort_by(
RunnableContext { input, name, .. }: RunnableContext,
) -> Result<OutputStream, ShellError> {
Ok(OutputStream::new(async_stream! {
let values: Vec<Tagged<Value>> = input.values.collect().await;
let values: Vec<Tagged<Value>> = input.values.collect().await;
let column_grouped_by_name = if let Some(grouped_by) = group_by {
Some(grouped_by.item().clone())
} else {
None
};
let column_grouped_by_name = if let Some(grouped_by) = group_by {
Some(grouped_by.item().clone())
} else {
None
};
if show_columns {
for label in columns_sorted(column_grouped_by_name, &values[0], &name).iter() {
yield ReturnSuccess::value(label.clone());
}
} else {
match t_sort(column_grouped_by_name, None, &values[0], name) {
Ok(sorted) => yield ReturnSuccess::value(sorted),
Err(err) => yield Err(err)
}
if show_columns {
for label in columns_sorted(column_grouped_by_name, &values[0], &name).iter() {
yield ReturnSuccess::value(label.clone());
}
}))
} else {
match t_sort(column_grouped_by_name, None, &values[0], name) {
Ok(sorted) => yield ReturnSuccess::value(sorted),
Err(err) => yield Err(err)
}
}
}))
}
pub fn columns_sorted(
@ -125,7 +125,7 @@ pub fn columns_sorted(
keys.into_iter().map(|k| k.tagged(&origin_tag)).collect()
}
_ => vec![Value::string("default").tagged(&origin_tag)]
_ => vec![Value::string("default").tagged(&origin_tag)],
}
}
@ -238,7 +238,7 @@ mod tests {
use crate::data::meta::*;
use crate::Value;
use indexmap::IndexMap;
fn string(input: impl Into<String>) -> Tagged<Value> {
Value::string(input.into()).tagged_unknown()
}
@ -305,7 +305,7 @@ mod tests {
]
)
}
#[test]
fn sorts_the_tables() {
let group_by = String::from("date");

31
tests/commands_test.rs
View File

@ -31,6 +31,35 @@ fn group_by() {
})
}
#[test]
fn histogram() {
Playground::setup("histogram_test_1", |dirs, sandbox| {
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"los_tres_caballeros.csv",
r#"
first_name,last_name,rusty_at
Andrés,Robalino,Ecuador
Jonathan,Turner,Estados Unidos
Yehuda,Katz,Estados Unidos
"#,
)]);
let actual = nu!(
cwd: dirs.test(), h::pipeline(
r#"
open los_tres_caballeros.csv
| histogram rusty_at countries
| where rusty_at == "Ecuador"
| get countries
| echo $it
"#
));
assert_eq!(actual, "**************************************************");
// 50%
})
}
#[test]
fn group_by_errors_if_unknown_column_name() {
Playground::setup("group_by_test_2", |dirs, sandbox| {
@ -56,7 +85,6 @@ fn group_by_errors_if_unknown_column_name() {
})
}
#[cfg(data_processing_primitives)]
#[test]
fn split_by() {
Playground::setup("split_by_test_1", |dirs, sandbox| {
@ -86,7 +114,6 @@ fn split_by() {
})
}
#[cfg(data_processing_primitives)]
#[test]
fn split_by_errors_if_no_table_given_as_input() {
Playground::setup("split_by_test_2", |dirs, sandbox| {