holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Data summarize reporting overhaul. (#2299)

Browse Source 
Refactored out most of internal work for summarizing data opening
the door for generating charts from it. A model is introduced
to hold information needed for a summary, Histogram command is
an example of a partial usage. This is the beginning.

Removed implicit arithmetic traits on Value and Primitive to avoid
mixed types panics. The std operations traits can't fail and we
can't guarantee that. We can handle gracefully now since compute_values
was introduced after the parser changes four months ago. The handling
logic should be taken care of either explicitly or in compute_values.

The zero identity trait was also removed (and implementing this forced
us to also implement Add, Mult, etc)

Also: the `math` operations now remove in the output if a given column is not computable:

```
> ls | math sum
──────┬──────────
 size │ 150.9 KB
──────┴──────────
```
This commit is contained in:
Andrés N. Robalino 2020-08-03 17:47:19 -05:00 committed by GitHub
parent eeb9b4edcb
commit 028fc9b9cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1396 additions and 1512 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.azure/azure-pipelines.yml
View File

@ -55,10 +55,10 @@ steps:
- bash: RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used - bash: RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
condition: eq(variables['style'], 'unflagged') condition: eq(variables['style'], 'unflagged')
displayName: Check clippy lints displayName: Check clippy lints
- bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --features stable - bash: RUSTFLAGS="-D warnings" cargo test --all --features stable
condition: eq(variables['style'], 'canary') condition: eq(variables['style'], 'canary')
displayName: Run tests displayName: Run tests
- bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used - bash: RUSTFLAGS="-D warnings" cargo clippy --all --features=stable -- -D clippy::result_unwrap_used -D clippy::option_unwrap_used
condition: eq(variables['style'], 'canary') condition: eq(variables['style'], 'canary')
displayName: Check clippy lints displayName: Check clippy lints
- bash: RUSTFLAGS="-D warnings" cargo test --all --no-default-features - bash: RUSTFLAGS="-D warnings" cargo test --all --no-default-features

1
.cargo/config
View File

@ -1 +0,0 @@
[build]

623
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

1
Cargo.toml
View File

@ -25,6 +25,7 @@ nu-plugin = {version = "0.17.0", path = "./crates/nu-plugin"}
nu-protocol = {version = "0.17.0", path = "./crates/nu-protocol"} nu-protocol = {version = "0.17.0", path = "./crates/nu-protocol"}
nu-source = {version = "0.17.0", path = "./crates/nu-source"} nu-source = {version = "0.17.0", path = "./crates/nu-source"}
nu-value-ext = {version = "0.17.0", path = "./crates/nu-value-ext"} nu-value-ext = {version = "0.17.0", path = "./crates/nu-value-ext"}
nu_plugin_binaryview = {version = "0.17.0", path = "./crates/nu_plugin_binaryview", optional = true} nu_plugin_binaryview = {version = "0.17.0", path = "./crates/nu_plugin_binaryview", optional = true}
nu_plugin_fetch = {version = "0.17.0", path = "./crates/nu_plugin_fetch", optional = true} nu_plugin_fetch = {version = "0.17.0", path = "./crates/nu_plugin_fetch", optional = true}
nu_plugin_from_bson = {version = "0.17.0", path = "./crates/nu_plugin_from_bson", optional = true} nu_plugin_from_bson = {version = "0.17.0", path = "./crates/nu_plugin_from_bson", optional = true}

1
crates/nu-cli/src/commands/from_xml.rs
View File

@ -135,7 +135,6 @@ async fn from_xml(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::commands::from_xml; use crate::commands::from_xml;
use indexmap::IndexMap; use indexmap::IndexMap;
use nu_protocol::{UntaggedValue, Value}; use nu_protocol::{UntaggedValue, Value};

93
crates/nu-cli/src/commands/group_by.rs
View File

@ -250,77 +250,32 @@ pub fn group(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::group; use super::group;
use indexmap::IndexMap; use crate::utils::data::helpers::{committers, date, int, row, string, table};
use nu_errors::ShellError; use nu_errors::ShellError;
use nu_protocol::{UntaggedValue, Value};
use nu_source::*; use nu_source::*;
fn string(input: impl Into<String>) -> Value {
UntaggedValue::string(input.into()).into_untagged_value()
}
fn row(entries: IndexMap<String, Value>) -> Value {
UntaggedValue::row(entries).into_untagged_value()
}
fn table(list: &[Value]) -> Value {
UntaggedValue::table(list).into_untagged_value()
}
fn nu_releases_committers() -> Vec<Value> {
vec![
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")},
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")},
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")},
),
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")},
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")},
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")},
),
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")},
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")},
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")},
),
]
}
#[test] #[test]
fn groups_table_by_date_column() -> Result<(), ShellError> { fn groups_table_by_date_column() -> Result<(), ShellError> {
let for_key = Some(String::from("date").tagged_unknown()); let for_key = Some(String::from("date").tagged_unknown());
let sample = table(&nu_releases_committers()); let sample = table(&committers());
assert_eq!( assert_eq!(
group(&for_key, &sample, Tag::unknown())?, group(&for_key, &sample, Tag::unknown())?,
row(indexmap! { row(indexmap! {
"August 23-2019".into() => table(&[ "2019-07-23".into() => table(&[
row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}), row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-07-23"), "chickens".into() => int(10) }),
row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}), row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-07-23"), "chickens".into() => int(5) }),
row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}) row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-07-23"), "chickens".into() => int(2) })
]), ]),
"October 10-2019".into() => table(&[ "2019-10-10".into() => table(&[
row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}), row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-10-10"), "chickens".into() => int(6) }),
row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}), row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-10-10"), "chickens".into() => int(15) }),
row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")}) row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-10-10"), "chickens".into() => int(30) })
]), ]),
"Sept 24-2019".into() => table(&[ "2019-09-24".into() => table(&[
row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}), row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-09-24"), "chickens".into() => int(20) }),
row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}), row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-09-24"), "chickens".into() => int(4) }),
row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")}) row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-09-24"), "chickens".into() => int(10) })
]), ]),
}) })
); );
@ -331,25 +286,25 @@ mod tests {
#[test] #[test]
fn groups_table_by_country_column() -> Result<(), ShellError> { fn groups_table_by_country_column() -> Result<(), ShellError> {
let for_key = Some(String::from("country").tagged_unknown()); let for_key = Some(String::from("country").tagged_unknown());
let sample = table(&nu_releases_committers()); let sample = table(&committers());
assert_eq!( assert_eq!(
group(&for_key, &sample, Tag::unknown())?, group(&for_key, &sample, Tag::unknown())?,
row(indexmap! { row(indexmap! {
"EC".into() => table(&[ "EC".into() => table(&[
row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}), row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-07-23"), "chickens".into() => int(10) }),
row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}), row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-09-24"), "chickens".into() => int(20) }),
row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")}) row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-10-10"), "chickens".into() => int(30) })
]), ]),
"NZ".into() => table(&[ "NZ".into() => table(&[
row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}), row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-07-23"), "chickens".into() => int(5) }),
row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}), row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-10-10"), "chickens".into() => int(15) }),
row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")}) row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-09-24"), "chickens".into() => int(10) })
]), ]),
"US".into() => table(&[ "US".into() => table(&[
row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}), row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-10-10"), "chickens".into() => int(6) }),
row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}), row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-09-24"), "chickens".into() => int(4) }),
row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}), row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-07-23"), "chickens".into() => int(2) }),
]), ]),
}) })
); );

4
crates/nu-cli/src/commands/group_by_date.rs
View File

@ -100,7 +100,7 @@ pub async fn group_by_date(
let value_result = match (grouper_date, grouper_column) { let value_result = match (grouper_date, grouper_column) {
(Grouper::ByDate(None), GroupByColumn::Name(None)) => { (Grouper::ByDate(None), GroupByColumn::Name(None)) => {
let block = Box::new(move |_, row: &Value| row.format("%Y-%b-%d")); let block = Box::new(move |_, row: &Value| row.format("%Y-%m-%d"));
crate::utils::data::group(&values, &Some(block), &name) crate::utils::data::group(&values, &Some(block), &name)
} }
@ -110,7 +110,7 @@ pub async fn group_by_date(
.get_data_by_key(column_name.borrow_spanned()) .get_data_by_key(column_name.borrow_spanned())
.ok_or_else(|| suggestions(column_name.borrow_tagged(), &row)); .ok_or_else(|| suggestions(column_name.borrow_tagged(), &row));
group_key?.format("%Y-%b-%d") group_key?.format("%Y-%m-%d")
}); });
crate::utils::data::group(&values, &Some(block), &name) crate::utils::data::group(&values, &Some(block), &name)

190
crates/nu-cli/src/commands/histogram.rs
View File

@ -1,13 +1,8 @@
use crate::commands::group_by::group;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::prelude::*; use crate::prelude::*;
use crate::utils::data_processing::{columns_sorted, evaluate, map_max, reduce, t_sort};
use nu_errors::ShellError; use nu_errors::ShellError;
use nu_protocol::{ use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue, Value};
Primitive, ReturnSuccess, Signature, SyntaxShape, TaggedDictBuilder, UntaggedValue, Value,
};
use nu_source::Tagged; use nu_source::Tagged;
use num_traits::{ToPrimitive, Zero};
pub struct Histogram; pub struct Histogram;
@ -79,23 +74,32 @@ pub async fn histogram(
let (HistogramArgs { column_name, rest }, input) = args.process(&registry).await?; let (HistogramArgs { column_name, rest }, input) = args.process(&registry).await?;
let values: Vec<Value> = input.collect().await; let values: Vec<Value> = input.collect().await;
let values = UntaggedValue::table(&values).into_value(&name);
let groups = group(&Some(column_name.clone()), &values, &name)?; let column_grouper = column_name.clone();
let group_labels = columns_sorted(Some(column_name.clone()), &groups, &name);
let sorted = t_sort(Some(column_name.clone()), None, &groups, &name)?;
let evaled = evaluate(&sorted, None, &name)?;
let reduced = reduce(&evaled, None, &name)?;
let maxima = map_max(&reduced, None, &name)?;
let percents = percentages(&reduced, maxima, &name)?;
match percents { let results = crate::utils::data::report(
Value { &UntaggedValue::table(&values).into_value(&name),
value: UntaggedValue::Table(datasets), crate::utils::data::Operation {
.. grouper: Some(Box::new(move |_, _| Ok(String::from("frequencies")))),
} => { splitter: Some(Box::new(move |_, row: &Value| {
let mut idx = 0; let key = &column_grouper;
match row.get_data_by_key(key.borrow_spanned()) {
Some(key) => nu_value_ext::as_string(&key),
None => Err(ShellError::labeled_error(
"unknown column",
"unknown column",
key.tag(),
)),
}
})),
format: None,
eval: &None,
},
&name,
)?;
let labels = results.labels.y.clone();
let column_names_supplied: Vec<_> = rest.iter().map(|f| f.item.clone()).collect(); let column_names_supplied: Vec<_> = rest.iter().map(|f| f.item.clone()).collect();
let frequency_column_name = if column_names_supplied.is_empty() { let frequency_column_name = if column_names_supplied.is_empty() {
@ -105,50 +109,23 @@ pub async fn histogram(
}; };
let column = (*column_name).clone(); let column = (*column_name).clone();
let mut idx = 0;
let count_column_name = "count".to_string(); Ok(futures::stream::iter(
let count_shell_error = ShellError::labeled_error( results
"Unable to load group count", .percentages
"unabled to load group count", .table_entries()
&name, .map(move |value| {
); let values = value.table_entries().cloned().collect::<Vec<_>>();
let mut count_values: Vec<u64> = Vec::new(); let count = values.len();
for table_entry in reduced.table_entries() { (count, values[count - 1].clone())
match table_entry { })
Value { .collect::<Vec<_>>()
value: UntaggedValue::Table(list), .into_iter()
.. .map(move |(count, value)| {
} => {
for i in list {
if let Ok(count) = i.value.clone().into_value(&name).as_u64() {
count_values.push(count);
} else {
return Err(count_shell_error);
}
}
}
_ => {
return Err(count_shell_error);
}
}
}
if let Value {
value: UntaggedValue::Table(start),
..
} = datasets.get(0).ok_or_else(|| {
ShellError::labeled_error(
"Unable to load dataset",
"unabled to load dataset",
&name,
)
})? {
let start = start.clone();
Ok(
futures::stream::iter(start.into_iter().map(move |percentage| {
let mut fact = TaggedDictBuilder::new(&name); let mut fact = TaggedDictBuilder::new(&name);
let value: Tagged<String> = group_labels let column_value = labels
.get(idx) .get(idx)
.ok_or_else(|| { .ok_or_else(|| {
ShellError::labeled_error( ShellError::labeled_error(
@ -158,99 +135,24 @@ pub async fn histogram(
) )
})? })?
.clone(); .clone();
fact.insert_value(
&column,
UntaggedValue::string(value.item).into_value(value.tag),
);
fact.insert_untagged( fact.insert_value(&column, column_value);
&count_column_name, fact.insert_untagged("count", UntaggedValue::int(count));
UntaggedValue::int(count_values[idx]),
);
if let Value {
value: UntaggedValue::Primitive(Primitive::Int(ref num)),
ref tag,
} = percentage
{
let string = std::iter::repeat("*") let string = std::iter::repeat("*")
.take(num.to_i32().ok_or_else(|| { .take(value.as_u64().map_err(|_| {
ShellError::labeled_error( ShellError::labeled_error("expected a number", "expected a number", &name)
"Expected a number",
"expected a number",
tag,
)
})? as usize) })? as usize)
.collect::<String>(); .collect::<String>();
fact.insert_untagged(
&frequency_column_name, fact.insert_untagged(&frequency_column_name, UntaggedValue::string(string));
UntaggedValue::string(string),
);
}
idx += 1; idx += 1;
ReturnSuccess::value(fact.into_value()) ReturnSuccess::value(fact.into_value())
})) }),
.to_output_stream(),
) )
} else { .to_output_stream())
Ok(OutputStream::empty())
}
}
_ => Ok(OutputStream::empty()),
}
}
fn percentages(values: &Value, max: Value, tag: impl Into<Tag>) -> Result<Value, ShellError> {
let tag = tag.into();
let results: Value = match values {
Value {
value: UntaggedValue::Table(datasets),
..
} => {
let datasets: Vec<_> = datasets
.iter()
.map(|subsets| match subsets {
Value {
value: UntaggedValue::Table(data),
..
} => {
let data = data
.iter()
.map(|d| match d {
Value {
value: UntaggedValue::Primitive(Primitive::Int(n)),
..
} => {
let max = match &max {
Value {
value: UntaggedValue::Primitive(Primitive::Int(maxima)),
..
} => maxima.clone(),
_ => Zero::zero(),
};
let n = (n * 100) / max;
UntaggedValue::int(n).into_value(&tag)
}
_ => UntaggedValue::int(0).into_value(&tag),
})
.collect::<Vec<_>>();
UntaggedValue::Table(data).into_value(&tag)
}
_ => UntaggedValue::Table(vec![]).into_value(&tag),
})
.collect();
UntaggedValue::Table(datasets).into_value(&tag)
}
other => other.clone(),
};
Ok(results)
} }
#[cfg(test)] #[cfg(test)]

57
crates/nu-cli/src/commands/math/avg.rs
View File

@ -1,14 +1,17 @@
use crate::prelude::*;
use crate::commands::math::reducers::{reducer_for, Reduce};
use crate::commands::math::utils::run_with_function; use crate::commands::math::utils::run_with_function;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::prelude::*;
use crate::utils::data_processing::{reducer_for, Reduce};
use bigdecimal::{FromPrimitive, Zero};
use nu_errors::ShellError; use nu_errors::ShellError;
use nu_protocol::{ use nu_protocol::{
hir::{convert_number_to_u64, Number, Operator}, hir::{convert_number_to_u64, Number, Operator},
Primitive, Signature, UntaggedValue, Value, Primitive, Signature, UntaggedValue, Value,
}; };
use bigdecimal::FromPrimitive;
pub struct SubCommand; pub struct SubCommand;
#[async_trait] #[async_trait]
@ -55,19 +58,59 @@ impl WholeStreamCommand for SubCommand {
} }
} }
fn to_byte(value: &Value) -> Option<Value> {
match &value.value {
UntaggedValue::Primitive(Primitive::Int(num)) => Some(
UntaggedValue::Primitive(Primitive::Filesize(convert_number_to_u64(&Number::Int(
num.clone(),
))))
.into_untagged_value(),
),
_ => None,
}
}
pub fn average(values: &[Value], name: &Tag) -> Result<Value, ShellError> { pub fn average(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
let sum = reducer_for(Reduce::Summation); let sum = reducer_for(Reduce::Summation);
let number = BigDecimal::from_usize(values.len()).ok_or_else(|| { let number = BigDecimal::from_usize(values.len()).ok_or_else(|| {
ShellError::labeled_error("nothing to average", "nothing to average", &name.span)
})?;
let total_rows = UntaggedValue::decimal(number);
let are_bytes = values
.get(0)
.ok_or_else(|| {
ShellError::unexpected("Cannot perform aggregate math operation on empty data")
})?
.is_filesize();
let total = if are_bytes {
to_byte(&sum(
UntaggedValue::int(0).into_untagged_value(),
values
.to_vec()
.iter()
.map(|v| match v {
Value {
value: UntaggedValue::Primitive(Primitive::Filesize(num)),
..
} => UntaggedValue::int(*num as usize).into_untagged_value(),
other => other.clone(),
})
.collect::<Vec<_>>(),
)?)
.ok_or_else(|| {
ShellError::labeled_error( ShellError::labeled_error(
"could not convert to big decimal", "could not convert to big decimal",
"could not convert to big decimal", "could not convert to big decimal",
&name.span, &name.span,
) )
})?; })
} else {
let total_rows = UntaggedValue::decimal(number); sum(UntaggedValue::int(0).into_untagged_value(), values.to_vec())
let total = sum(Value::zero(), values.to_vec())?; }?;
match total { match total {
Value { Value {

2
crates/nu-cli/src/commands/math/max.rs
View File

@ -1,7 +1,7 @@
use crate::commands::math::reducers::{reducer_for, Reduce};
use crate::commands::math::utils::run_with_function; use crate::commands::math::utils::run_with_function;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::prelude::*; use crate::prelude::*;
use crate::utils::data_processing::{reducer_for, Reduce};
use nu_errors::ShellError; use nu_errors::ShellError;
use nu_protocol::{Signature, UntaggedValue, Value}; use nu_protocol::{Signature, UntaggedValue, Value};

6
crates/nu-cli/src/commands/math/median.rs
View File

@ -1,8 +1,8 @@
use crate::commands::math::reducers::{reducer_for, Reduce};
use crate::commands::math::utils::run_with_function; use crate::commands::math::utils::run_with_function;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::prelude::*; use crate::prelude::*;
use crate::utils::data_processing::{reducer_for, Reduce}; use bigdecimal::FromPrimitive;
use bigdecimal::{FromPrimitive, Zero};
use nu_errors::ShellError; use nu_errors::ShellError;
use nu_protocol::{ use nu_protocol::{
hir::{convert_number_to_u64, Number, Operator}, hir::{convert_number_to_u64, Number, Operator},
@ -130,7 +130,7 @@ fn compute_average(values: &[Value], name: impl Into<Tag>) -> Result<Value, Shel
) )
})?; })?;
let total_rows = UntaggedValue::decimal(number); let total_rows = UntaggedValue::decimal(number);
let total = sum(Value::zero(), values.to_vec())?; let total = sum(UntaggedValue::int(0).into_untagged_value(), values.to_vec())?;
match total { match total {
Value { Value {

2
crates/nu-cli/src/commands/math/min.rs
View File

@ -1,7 +1,7 @@
use crate::commands::math::reducers::{reducer_for, Reduce};
use crate::commands::math::utils::run_with_function; use crate::commands::math::utils::run_with_function;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::prelude::*; use crate::prelude::*;
use crate::utils::data_processing::{reducer_for, Reduce};
use nu_errors::ShellError; use nu_errors::ShellError;
use nu_protocol::{Signature, UntaggedValue, Value}; use nu_protocol::{Signature, UntaggedValue, Value};

4
crates/nu-cli/src/commands/math/mod.rs
View File

@ -7,9 +7,11 @@ pub mod min;
pub mod mode; pub mod mode;
pub mod stddev; pub mod stddev;
pub mod sum; pub mod sum;
pub mod utils;
pub mod variance; pub mod variance;
mod reducers;
mod utils;
pub use avg::SubCommand as MathAverage; pub use avg::SubCommand as MathAverage;
pub use command::Command as Math; pub use command::Command as Math;
pub use eval::SubCommand as MathEval; pub use eval::SubCommand as MathEval;

135
crates/nu-cli/src/commands/math/reducers.rs Normal file
View File

@ -0,0 +1,135 @@
use crate::data::value::{compare_values, compute_values};
use nu_errors::ShellError;
use nu_protocol::hir::Operator;
use nu_protocol::{UntaggedValue, Value};
use nu_source::{SpannedItem, Tag};
// Re-usable error messages
const ERR_EMPTY_DATA: &str = "Cannot perform aggregate math operation on empty data";
fn formula(
acc_begin: Value,
calculator: Box<dyn Fn(Vec<Value>) -> Result<Value, ShellError> + Send + Sync + 'static>,
) -> Box<dyn Fn(Value, Vec<Value>) -> Result<Value, ShellError> + Send + Sync + 'static> {
Box::new(move |acc, datax| -> Result<Value, ShellError> {
let result = match compute_values(Operator::Multiply, &acc, &acc_begin) {
Ok(v) => v.into_untagged_value(),
Err((left_type, right_type)) => {
return Err(ShellError::coerce_error(
left_type.spanned_unknown(),
right_type.spanned_unknown(),
))
}
};
match calculator(datax) {
Ok(total) => Ok(match compute_values(Operator::Plus, &result, &total) {
Ok(v) => v.into_untagged_value(),
Err((left_type, right_type)) => {
return Err(ShellError::coerce_error(
left_type.spanned_unknown(),
right_type.spanned_unknown(),
))
}
}),
Err(reason) => Err(reason),
}
})
}
pub fn reducer_for(
command: Reduce,
) -> Box<dyn Fn(Value, Vec<Value>) -> Result<Value, ShellError> + Send + Sync + 'static> {
match command {
Reduce::Summation | Reduce::Default => Box::new(formula(
UntaggedValue::int(0).into_untagged_value(),
Box::new(sum),
)),
Reduce::Minimum => Box::new(|_, values| min(values)),
Reduce::Maximum => Box::new(|_, values| max(values)),
}
}
pub enum Reduce {
Summation,
Minimum,
Maximum,
Default,
}
pub fn sum(data: Vec<Value>) -> Result<Value, ShellError> {
let mut acc = UntaggedValue::int(0).into_untagged_value();
for value in data {
match value.value {
UntaggedValue::Primitive(_) => {
acc = match compute_values(Operator::Plus, &acc, &value) {
Ok(v) => v.into_untagged_value(),
Err((left_type, right_type)) => {
return Err(ShellError::coerce_error(
left_type.spanned_unknown(),
right_type.spanned_unknown(),
))
}
};
}
_ => {
return Err(ShellError::labeled_error(
"Attempted to compute the sum of a value that cannot be summed.",
"value appears here",
value.tag.span,
))
}
}
}
Ok(acc)
}
pub fn max(data: Vec<Value>) -> Result<Value, ShellError> {
let mut biggest = data
.first()
.ok_or_else(|| ShellError::unexpected(ERR_EMPTY_DATA))?
.value
.clone();
for value in data.iter() {
if let Ok(greater_than) = compare_values(Operator::GreaterThan, &value.value, &biggest) {
if greater_than {
biggest = value.value.clone();
}
} else {
return Err(ShellError::unexpected(format!(
"Could not compare\nleft: {:?}\nright: {:?}",
biggest, value.value
)));
}
}
Ok(Value {
value: biggest,
tag: Tag::unknown(),
})
}
pub fn min(data: Vec<Value>) -> Result<Value, ShellError> {
let mut smallest = data
.first()
.ok_or_else(|| ShellError::unexpected(ERR_EMPTY_DATA))?
.value
.clone();
for value in data.iter() {
if let Ok(greater_than) = compare_values(Operator::LessThan, &value.value, &smallest) {
if greater_than {
smallest = value.value.clone();
}
} else {
return Err(ShellError::unexpected(format!(
"Could not compare\nleft: {:?}\nright: {:?}",
smallest, value.value
)));
}
}
Ok(Value {
value: smallest,
tag: Tag::unknown(),
})
}

87
crates/nu-cli/src/commands/math/sum.rs
View File

@ -1,10 +1,13 @@
use crate::commands::math::reducers::{reducer_for, Reduce};
use crate::commands::math::utils::run_with_function; use crate::commands::math::utils::run_with_function;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::prelude::*; use crate::prelude::*;
use crate::utils::data_processing::{reducer_for, Reduce};
use nu_errors::ShellError; use nu_errors::ShellError;
use nu_protocol::{Dictionary, Signature, UntaggedValue, Value};
use num_traits::identities::Zero; use nu_protocol::{
hir::{convert_number_to_u64, Number},
Primitive, Signature, UntaggedValue, Value,
};
pub struct SubCommand; pub struct SubCommand;
@ -59,37 +62,63 @@ impl WholeStreamCommand for SubCommand {
} }
} }
fn to_byte(value: &Value) -> Option<Value> {
match &value.value {
UntaggedValue::Primitive(Primitive::Int(num)) => Some(
UntaggedValue::Primitive(Primitive::Filesize(convert_number_to_u64(&Number::Int(
num.clone(),
))))
.into_untagged_value(),
),
_ => None,
}
}
pub fn summation(values: &[Value], name: &Tag) -> Result<Value, ShellError> { pub fn summation(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
let sum = reducer_for(Reduce::Summation); let sum = reducer_for(Reduce::Summation);
if values.iter().all(|v| v.is_primitive()) { let first = values.get(0).ok_or_else(|| {
Ok(sum(Value::zero(), values.to_vec())?) ShellError::unexpected("Cannot perform aggregate math operation on empty data")
} else { })?;
let mut column_values = IndexMap::new();
for value in values { match first {
if let UntaggedValue::Row(row_dict) = value.value.clone() { v if v.is_filesize() => to_byte(&sum(
for (key, value) in row_dict.entries.iter() { UntaggedValue::int(0).into_untagged_value(),
column_values values
.entry(key.clone()) .to_vec()
.and_modify(|v: &mut Vec<Value>| v.push(value.clone())) .iter()
.or_insert(vec![value.clone()]); .map(|v| match v {
} Value {
}; value: UntaggedValue::Primitive(Primitive::Filesize(num)),
} ..
} => UntaggedValue::int(*num as usize).into_untagged_value(),
let mut column_totals = IndexMap::new(); other => other.clone(),
for (col_name, col_vals) in column_values {
let sum = sum(Value::zero(), col_vals)?;
column_totals.insert(col_name, sum);
}
Ok(UntaggedValue::Row(Dictionary {
entries: column_totals,
}) })
.into_value(name)) .collect::<Vec<_>>(),
)?)
.ok_or_else(|| {
ShellError::labeled_error(
"could not convert to big decimal",
"could not convert to big decimal",
&name.span,
)
}),
// v is nothing primitive
v if v.is_none() => sum(
UntaggedValue::int(0).into_untagged_value(),
values
.to_vec()
.iter()
.map(|v| match v {
Value {
value: UntaggedValue::Primitive(Primitive::Nothing),
..
} => UntaggedValue::int(0).into_untagged_value(),
other => other.clone(),
})
.collect::<Vec<_>>(),
),
_ => sum(UntaggedValue::int(0).into_untagged_value(), values.to_vec()),
} }
} }

13
crates/nu-cli/src/commands/math/utils.rs
View File

@ -13,6 +13,7 @@ pub async fn run_with_function(
mf: MathFunction, mf: MathFunction,
) -> Result<OutputStream, ShellError> { ) -> Result<OutputStream, ShellError> {
let values: Vec<Value> = input.drain_vec().await; let values: Vec<Value> = input.drain_vec().await;
let res = calculate(&values, &name, mf); let res = calculate(&values, &name, mf);
match res { match res {
Ok(v) => { Ok(v) => {
@ -50,7 +51,17 @@ pub fn calculate(values: &[Value], name: &Tag, mf: MathFunction) -> Result<Value
// The mathematical function operates over the columns of the table // The mathematical function operates over the columns of the table
let mut column_totals = IndexMap::new(); let mut column_totals = IndexMap::new();
for (col_name, col_vals) in column_values { for (col_name, col_vals) in column_values {
column_totals.insert(col_name, mf(&col_vals, &name)?); if let Ok(out) = mf(&col_vals, &name) {
column_totals.insert(col_name, out);
}
}
if column_totals.keys().len() == 0 {
return Err(ShellError::labeled_error(
"Attempted to compute values that can't be operated on",
"value appears here",
name.span,
));
} }
Ok(UntaggedValue::Row(Dictionary { Ok(UntaggedValue::Row(Dictionary {

28
crates/nu-cli/src/commands/math/variance.rs
View File

@ -2,7 +2,7 @@ use crate::commands::math::utils::run_with_function;
use crate::commands::WholeStreamCommand; use crate::commands::WholeStreamCommand;
use crate::data::value::compute_values; use crate::data::value::compute_values;
use crate::prelude::*; use crate::prelude::*;
use bigdecimal::{FromPrimitive, Zero}; use bigdecimal::FromPrimitive;
use nu_errors::ShellError; use nu_errors::ShellError;
use nu_protocol::{hir::Operator, Primitive, Signature, UntaggedValue, Value}; use nu_protocol::{hir::Operator, Primitive, Signature, UntaggedValue, Value};
@ -60,8 +60,8 @@ fn sum_of_squares(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
&name.span, &name.span,
) )
})?; })?;
let mut sum_x = Value::zero(); let mut sum_x = UntaggedValue::int(0).into_untagged_value();
let mut sum_x2 = Value::zero(); let mut sum_x2 = UntaggedValue::int(0).into_untagged_value();
for value in values { for value in values {
let v = match value { let v = match value {
Value { Value {
@ -87,7 +87,17 @@ fn sum_of_squares(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
let v_squared = compute_values(Operator::Multiply, &v, &v); let v_squared = compute_values(Operator::Multiply, &v, &v);
match v_squared { match v_squared {
// X^2 // X^2
Ok(x2) => sum_x2 = sum_x2 + x2.into_untagged_value(), Ok(x2) => {
sum_x2 = match compute_values(Operator::Plus, &sum_x2, &x2) {
Ok(v) => v.into_untagged_value(),
Err((left_type, right_type)) => {
return Err(ShellError::coerce_error(
left_type.spanned(name.span),
right_type.spanned(name.span),
))
}
};
}
Err((left_type, right_type)) => { Err((left_type, right_type)) => {
return Err(ShellError::coerce_error( return Err(ShellError::coerce_error(
left_type.spanned(value.tag.span), left_type.spanned(value.tag.span),
@ -95,7 +105,15 @@ fn sum_of_squares(values: &[Value], name: &Tag) -> Result<Value, ShellError> {
)) ))
} }
}; };
sum_x = sum_x + v.into_untagged_value(); sum_x = match compute_values(Operator::Plus, &sum_x, &v) {
Ok(v) => v.into_untagged_value(),
Err((left_type, right_type)) => {
return Err(ShellError::coerce_error(
left_type.spanned(name.span),
right_type.spanned(name.span),
))
}
};
} }
let sum_x_squared = match compute_values(Operator::Multiply, &sum_x, &sum_x) { let sum_x_squared = match compute_values(Operator::Multiply, &sum_x, &sum_x) {

106
crates/nu-cli/src/commands/split_by.rs
View File

@ -124,106 +124,52 @@ pub fn suggestions(tried: Tagged<&str>, for_value: &Value) -> ShellError {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::split; use super::split;
use crate::commands::group_by::group; use crate::utils::data::helpers::{committers_grouped_by_date, date, int, row, string, table};
use indexmap::IndexMap; use nu_protocol::UntaggedValue;
use nu_errors::ShellError;
use nu_protocol::{UntaggedValue, Value};
use nu_source::*; use nu_source::*;
fn string(input: impl Into<String>) -> Value {
UntaggedValue::string(input.into()).into_untagged_value()
}
fn row(entries: IndexMap<String, Value>) -> Value {
UntaggedValue::row(entries).into_untagged_value()
}
fn table(list: &[Value]) -> Value {
UntaggedValue::table(list).into_untagged_value()
}
fn nu_releases_grouped_by_date() -> Result<Value, ShellError> {
let key = Some(String::from("date").tagged_unknown());
let sample = table(&nu_releases_committers());
group(&key, &sample, Tag::unknown())
}
fn nu_releases_committers() -> Vec<Value> {
vec![
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")},
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")},
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")},
),
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")},
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")},
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")},
),
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")},
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")},
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")},
),
]
}
#[test] #[test]
fn splits_inner_tables_by_key() -> Result<(), ShellError> { fn splits_inner_tables_by_key() {
let for_key = Some(String::from("country").tagged_unknown()); let for_key = Some(String::from("country").tagged_unknown());
assert_eq!( assert_eq!(
split(&for_key, &nu_releases_grouped_by_date()?, Tag::unknown())?, split(&for_key, &committers_grouped_by_date(), Tag::unknown()).unwrap(),
UntaggedValue::row(indexmap! { UntaggedValue::row(indexmap! {
"EC".into() => row(indexmap! { "EC".into() => row(indexmap! {
"August 23-2019".into() => table(&[ "2019-07-23".into() => table(&[
row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}) row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-07-23"), "chickens".into() => int(10)})
]), ]),
"Sept 24-2019".into() => table(&[ "2019-09-24".into() => table(&[
row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("Sept 24-2019")}) row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-09-24"), "chickens".into() => int(20)})
]), ]),
"October 10-2019".into() => table(&[ "2019-10-10".into() => table(&[
row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")}) row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => date("2019-10-10"), "chickens".into() => int(30)})
]) ])
}), }),
"NZ".into() => row(indexmap! { "NZ".into() => row(indexmap! {
"August 23-2019".into() => table(&[ "2019-07-23".into() => table(&[
row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}) row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-07-23"), "chickens".into() => int(5)})
]), ]),
"Sept 24-2019".into() => table(&[ "2019-09-24".into() => table(&[
row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("Sept 24-2019")}) row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-09-24"), "chickens".into() => int(10)})
]), ]),
"October 10-2019".into() => table(&[ "2019-10-10".into() => table(&[
row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}) row(indexmap!{"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => date("2019-10-10"), "chickens".into() => int(15)})
]) ])
}), }),
"US".into() => row(indexmap! { "US".into() => row(indexmap! {
"August 23-2019".into() => table(&[ "2019-07-23".into() => table(&[
row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}) row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-07-23"), "chickens".into() => int(2)})
]), ]),
"Sept 24-2019".into() => table(&[ "2019-09-24".into() => table(&[
row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("Sept 24-2019")}) row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-09-24"), "chickens".into() => int(4)})
]), ]),
"October 10-2019".into() => table(&[ "2019-10-10".into() => table(&[
row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}) row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => date("2019-10-10"), "chickens".into() => int(6)})
]) ])
}) })
}).into_untagged_value() }).into_untagged_value()
); );
Ok(())
} }
#[test] #[test]
@ -231,11 +177,11 @@ mod tests {
let for_key = Some(String::from("country").tagged_unknown()); let for_key = Some(String::from("country").tagged_unknown());
let nu_releases = row(indexmap! { let nu_releases = row(indexmap! {
"August 23-2019".into() => table(&[ "2019-07-23".into() => table(&[
row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}) row(indexmap!{"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("2019-07-23")})
]), ]),
"Sept 24-2019".into() => table(&[ "2019-09-24".into() => table(&[
row(indexmap!{"name".into() => UntaggedValue::string("JT").into_value(Tag::from(Span::new(5,10))), "date".into() => string("Sept 24-2019")}) row(indexmap!{"name".into() => UntaggedValue::string("JT").into_value(Tag::from(Span::new(5,10))), "date".into() => string("2019-09-24")})
]), ]),
"October 10-2019".into() => table(&[ "October 10-2019".into() => table(&[
row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}) row(indexmap!{"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")})

8
crates/nu-cli/src/data/base.rs
View File

@ -173,19 +173,19 @@ mod tests {
use num_bigint::BigInt; use num_bigint::BigInt;
fn string(input: impl Into<String>) -> Value { fn string(input: impl Into<String>) -> Value {
UntaggedValue::string(input.into()).into_untagged_value() crate::utils::data::helpers::string(input)
} }
fn int(input: impl Into<BigInt>) -> Value { fn int(input: impl Into<BigInt>) -> Value {
UntaggedValue::int(input.into()).into_untagged_value() crate::utils::data::helpers::int(input)
} }
fn row(entries: IndexMap<String, Value>) -> Value { fn row(entries: IndexMap<String, Value>) -> Value {
UntaggedValue::row(entries).into_untagged_value() crate::utils::data::helpers::row(entries)
} }
fn table(list: &[Value]) -> Value { fn table(list: &[Value]) -> Value {
UntaggedValue::table(list).into_untagged_value() crate::utils::data::helpers::table(list)
} }
fn error_callback( fn error_callback(

43
crates/nu-cli/src/data/value.rs
View File

@ -1,7 +1,7 @@
use crate::data::base::coerce_compare; use crate::data::base::coerce_compare;
use crate::data::base::shape::{Column, InlineShape}; use crate::data::base::shape::{Column, InlineShape};
use crate::data::primitive::style_primitive; use crate::data::primitive::style_primitive;
use chrono::DateTime; use chrono::{DateTime, NaiveDate, Utc};
use nu_errors::ShellError; use nu_errors::ShellError;
use nu_protocol::hir::Operator; use nu_protocol::hir::Operator;
use nu_protocol::ShellTypeName; use nu_protocol::ShellTypeName;
@ -10,7 +10,10 @@ use nu_source::{DebugDocBuilder, PrettyDebug, Span, Tagged};
use nu_table::TextStyle; use nu_table::TextStyle;
use num_traits::Zero; use num_traits::Zero;
pub fn date_from_str(s: Tagged<&str>) -> Result<UntaggedValue, ShellError> { pub struct Date;
impl Date {
pub fn from_regular_str(s: Tagged<&str>) -> Result<UntaggedValue, ShellError> {
let date = DateTime::parse_from_rfc3339(s.item).map_err(|err| { let date = DateTime::parse_from_rfc3339(s.item).map_err(|err| {
ShellError::labeled_error( ShellError::labeled_error(
&format!("Date parse error: {}", err), &format!("Date parse error: {}", err),
@ -24,6 +27,25 @@ pub fn date_from_str(s: Tagged<&str>) -> Result<UntaggedValue, ShellError> {
Ok(UntaggedValue::Primitive(Primitive::Date(date))) Ok(UntaggedValue::Primitive(Primitive::Date(date)))
} }
pub fn naive_from_str(s: Tagged<&str>) -> Result<UntaggedValue, ShellError> {
let date = NaiveDate::parse_from_str(s.item, "%Y-%m-%d").map_err(|reason| {
ShellError::labeled_error(
&format!("Date parse error: {}", reason),
"original value",
s.tag,
)
})?;
Ok(UntaggedValue::Primitive(Primitive::Date(
DateTime::<Utc>::from_utc(date.and_hms(12, 34, 56), Utc),
)))
}
}
pub fn date_from_str(s: Tagged<&str>) -> Result<UntaggedValue, ShellError> {
Date::from_regular_str(s)
}
pub fn merge_values( pub fn merge_values(
left: &UntaggedValue, left: &UntaggedValue,
right: &UntaggedValue, right: &UntaggedValue,
@ -204,23 +226,30 @@ pub fn format_for_column<'a>(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::UntaggedValue as v;
use indexmap::indexmap;
use super::merge_values; use super::merge_values;
use super::Date as d;
use super::UntaggedValue as v;
use nu_source::TaggedItem;
use indexmap::indexmap;
#[test] #[test]
fn merges_tables() { fn merges_tables() {
let (author_1_date, author_2_date) = (
"2020-04-29".to_string().tagged_unknown(),
"2019-10-10".to_string().tagged_unknown(),
);
let table_author_row = v::row(indexmap! { let table_author_row = v::row(indexmap! {
"name".into() => v::string("Andrés").into_untagged_value(), "name".into() => v::string("Andrés").into_untagged_value(),
"country".into() => v::string("EC").into_untagged_value(), "country".into() => v::string("EC").into_untagged_value(),
"date".into() => v::string("April 29-2020").into_untagged_value() "date".into() => d::naive_from_str(author_1_date.borrow_tagged()).unwrap().into_untagged_value()
}); });
let other_table_author_row = v::row(indexmap! { let other_table_author_row = v::row(indexmap! {
"name".into() => v::string("YK").into_untagged_value(), "name".into() => v::string("YK").into_untagged_value(),
"country".into() => v::string("US").into_untagged_value(), "country".into() => v::string("US").into_untagged_value(),
"date".into() => v::string("October 10-2019").into_untagged_value() "date".into() => d::naive_from_str(author_2_date.borrow_tagged()).unwrap().into_untagged_value()
}); });
assert_eq!( assert_eq!(

1
crates/nu-cli/src/utils.rs
View File

@ -1,5 +1,4 @@
pub mod data; pub mod data;
pub mod data_processing;
pub mod test_bins; pub mod test_bins;
use crate::path::canonicalize; use crate::path::canonicalize;

274
crates/nu-cli/src/utils/data/internal.rs Normal file
View File

@ -0,0 +1,274 @@
#![allow(clippy::type_complexity)]
use crate::data::value::compute_values;
use derive_new::new;
use nu_errors::ShellError;
use nu_protocol::hir::Operator;
use nu_protocol::{UntaggedValue, Value};
use nu_source::{SpannedItem, Tag, TaggedItem};
use nu_value_ext::ValueExt;
#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new)]
pub struct Labels {
pub x: Vec<String>,
pub y: Vec<String>,
}
impl Labels {
pub fn at(&self, idx: usize) -> Option<&str> {
if let Some(k) = self.x.get(idx) {
Some(&k[..])
} else {
None
}
}
pub fn grouped(&self) -> impl Iterator<Item = &String> {
self.x.iter()
}
pub fn grouping_total(&self) -> Value {
UntaggedValue::int(self.x.len()).into_untagged_value()
}
pub fn splits(&self) -> impl Iterator<Item = &String> {
self.y.iter()
}
pub fn splits_total(&self) -> Value {
UntaggedValue::int(self.y.len()).into_untagged_value()
}
}
#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new)]
pub struct Range {
pub start: Value,
pub end: Value,
}
fn formula(
acc_begin: Value,
calculator: Box<dyn Fn(Vec<&Value>) -> Result<Value, ShellError> + Send + Sync + 'static>,
) -> Box<dyn Fn(&Value, Vec<&Value>) -> Result<Value, ShellError> + Send + Sync + 'static> {
Box::new(move |acc, datax| -> Result<Value, ShellError> {
let result = match compute_values(Operator::Multiply, &acc, &acc_begin) {
Ok(v) => v.into_untagged_value(),
Err((left_type, right_type)) => {
return Err(ShellError::coerce_error(
left_type.spanned_unknown(),
right_type.spanned_unknown(),
))
}
};
match calculator(datax) {
Ok(total) => Ok(match compute_values(Operator::Plus, &result, &total) {
Ok(v) => v.into_untagged_value(),
Err((left_type, right_type)) => {
return Err(ShellError::coerce_error(
left_type.spanned_unknown(),
right_type.spanned_unknown(),
))
}
}),
Err(reason) => Err(reason),
}
})
}
pub fn reducer_for(
command: Reduction,
) -> Box<dyn Fn(&Value, Vec<&Value>) -> Result<Value, ShellError> + Send + Sync + 'static> {
match command {
Reduction::Accumulate => Box::new(formula(
UntaggedValue::int(1).into_untagged_value(),
Box::new(sum),
)),
_ => Box::new(formula(
UntaggedValue::int(0).into_untagged_value(),
Box::new(sum),
)),
}
}
pub fn max(values: &Value, tag: impl Into<Tag>) -> Result<&Value, ShellError> {
let tag = tag.into();
values
.table_entries()
.filter_map(|dataset| dataset.table_entries().max())
.max()
.ok_or_else(|| ShellError::labeled_error("err", "err", &tag))
}
pub fn sum(data: Vec<&Value>) -> Result<Value, ShellError> {
let mut acc = UntaggedValue::int(0);
for value in data {
match value.value {
UntaggedValue::Primitive(_) => {
acc = match compute_values(Operator::Plus, &acc, &value) {
Ok(v) => v,
Err((left_type, right_type)) => {
return Err(ShellError::coerce_error(
left_type.spanned_unknown(),
right_type.spanned_unknown(),
))
}
};
}
_ => {
return Err(ShellError::labeled_error(
"Attempted to compute the sum of a value that cannot be summed.",
"value appears here",
value.tag.span,
))
}
}
}
Ok(acc.into_untagged_value())
}
pub fn sort_columns(
values: &[String],
format: &Option<Box<dyn Fn(&Value, String) -> Result<String, ShellError>>>,
) -> Result<Vec<String>, ShellError> {
let mut keys = vec![];
if let Some(fmt) = format {
for k in values.iter() {
let k = k.clone().tagged_unknown();
let v =
crate::data::value::Date::naive_from_str(k.borrow_tagged())?.into_untagged_value();
keys.push(fmt(&v, k.to_string())?);
}
} else {
keys = values.to_vec();
}
keys.sort();
Ok(keys)
}
pub fn sort(planes: &Labels, values: &Value, tag: impl Into<Tag>) -> Result<Value, ShellError> {
let tag = tag.into();
let mut x = vec![];
for column in planes.splits() {
let key = column.clone().tagged_unknown();
let groups = values
.get_data_by_key(key.borrow_spanned())
.ok_or_else(|| {
ShellError::labeled_error("unknown column", "unknown column", key.span())
})?;
let mut y = vec![];
for inner_column in planes.grouped() {
let key = inner_column.clone().tagged_unknown();
let grouped = groups.get_data_by_key(key.borrow_spanned());
if let Some(grouped) = grouped {
y.push(grouped.table_entries().cloned().collect::<Vec<_>>());
} else {
let empty = UntaggedValue::table(&[]).into_value(&tag);
y.push(empty.table_entries().cloned().collect::<Vec<_>>());
}
}
x.push(
UntaggedValue::table(&y.iter().cloned().flatten().collect::<Vec<Value>>())
.into_value(&tag),
);
}
Ok(UntaggedValue::table(&x).into_value(&tag))
}
pub fn evaluate(
values: &Value,
evaluator: &Option<Box<dyn Fn(usize, &Value) -> Result<Value, ShellError> + Send>>,
tag: impl Into<Tag>,
) -> Result<Value, ShellError> {
let tag = tag.into();
let mut x = vec![];
for split in values.table_entries() {
let mut y = vec![];
for (idx, subset) in split.table_entries().enumerate() {
let mut set = vec![];
if let Some(ref evaluator) = evaluator {
let value = evaluator(idx, subset)?;
set.push(value);
} else {
set.push(UntaggedValue::int(1).into_value(&tag));
}
y.push(UntaggedValue::table(&set).into_value(&tag));
}
x.push(UntaggedValue::table(&y).into_value(&tag));
}
Ok(UntaggedValue::table(&x).into_value(&tag))
}
pub enum Reduction {
#[allow(dead_code)]
Count,
Accumulate,
}
pub fn reduce(values: &Value, tag: impl Into<Tag>) -> Result<Value, ShellError> {
let tag = tag.into();
let reduce_with = reducer_for(Reduction::Accumulate);
let mut datasets = vec![];
for dataset in values.table_entries() {
let mut acc = UntaggedValue::int(0).into_value(&tag);
let mut subsets = vec![];
for subset in dataset.table_entries() {
acc = reduce_with(&acc, subset.table_entries().collect::<Vec<_>>())?;
subsets.push(acc.clone());
}
datasets.push(UntaggedValue::table(&subsets).into_value(&tag));
}
Ok(UntaggedValue::table(&datasets).into_value(&tag))
}
pub fn percentages(
maxima: &Value,
values: &Value,
tag: impl Into<Tag>,
) -> Result<Value, ShellError> {
let tag = tag.into();
let mut x = vec![];
for split in values.table_entries() {
x.push(
UntaggedValue::table(
&split
.table_entries()
.filter_map(|s| {
let hundred = UntaggedValue::decimal(100);
match compute_values(Operator::Divide, &hundred, &maxima) {
Ok(v) => match compute_values(Operator::Multiply, &s, &v) {
Ok(v) => Some(v.into_untagged_value()),
Err(_) => None,
},
Err(_) => None,
}
})
.collect::<Vec<_>>(),
)
.into_value(&tag),
);
}
Ok(UntaggedValue::table(&x).into_value(&tag))
}

291
crates/nu-cli/src/utils/data/mod.rs
View File

@ -1,5 +1,296 @@
pub mod group; pub mod group;
pub mod split; pub mod split;
mod internal;
pub use crate::utils::data::group::group; pub use crate::utils::data::group::group;
pub use crate::utils::data::split::split; pub use crate::utils::data::split::split;
use crate::utils::data::internal::*;
use derive_new::new;
use getset::Getters;
use nu_errors::ShellError;
use nu_protocol::{UntaggedValue, Value};
use nu_source::Tag;
#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Getters, Clone, new)]
pub struct Model {
pub labels: Labels,
pub ranges: (Range, Range),
pub data: Value,
pub percentages: Value,
}
#[allow(clippy::type_complexity)]
pub struct Operation<'a> {
pub grouper: Option<Box<dyn Fn(usize, &Value) -> Result<String, ShellError> + Send>>,
pub splitter: Option<Box<dyn Fn(usize, &Value) -> Result<String, ShellError> + Send>>,
pub format: Option<Box<dyn Fn(&Value, String) -> Result<String, ShellError>>>,
pub eval: &'a Option<Box<dyn Fn(usize, &Value) -> Result<Value, ShellError> + Send>>,
}
pub fn report(
values: &Value,
options: Operation,
tag: impl Into<Tag>,
) -> Result<Model, ShellError> {
let tag = tag.into();
let grouped = group(&values, &options.grouper, &tag)?;
let splitted = split(&grouped, &options.splitter, &tag)?;
let x = grouped
.row_entries()
.map(|(key, _)| key.clone())
.collect::<Vec<_>>();
let x = if options.format.is_some() {
sort_columns(&x, &options.format)
} else {
sort_columns(&x, &None)
}?;
let mut y = splitted
.row_entries()
.map(|(key, _)| key.clone())
.collect::<Vec<_>>();
y.sort();
let planes = Labels { x, y };
let sorted = sort(&planes, &splitted, &tag)?;
let evaluated = evaluate(
&sorted,
if options.eval.is_some() {
options.eval
} else {
&None
},
&tag,
)?;
let group_labels = planes.grouping_total();
let reduced = reduce(&evaluated, &tag)?;
let max = max(&reduced, &tag)?.clone();
let maxima = max.clone();
let percents = percentages(&maxima, &reduced, &tag)?;
Ok(Model {
labels: planes,
ranges: (
Range {
start: UntaggedValue::int(0).into_untagged_value(),
end: group_labels,
},
Range {
start: UntaggedValue::int(0).into_untagged_value(),
end: max,
},
),
data: reduced,
percentages: percents,
})
}
#[cfg(test)]
pub mod helpers {
use super::{report, Labels, Model, Operation, Range};
use bigdecimal::BigDecimal;
use indexmap::indexmap;
use nu_errors::ShellError;
use nu_protocol::{UntaggedValue, Value};
use nu_source::{Tag, TaggedItem};
use nu_value_ext::ValueExt;
use num_bigint::BigInt;
use indexmap::IndexMap;
pub fn int(s: impl Into<BigInt>) -> Value {
UntaggedValue::int(s).into_untagged_value()
}
pub fn decimal(f: impl Into<BigDecimal>) -> Value {
UntaggedValue::decimal(f.into()).into_untagged_value()
}
pub fn string(input: impl Into<String>) -> Value {
UntaggedValue::string(input.into()).into_untagged_value()
}
pub fn row(entries: IndexMap<String, Value>) -> Value {
UntaggedValue::row(entries).into_untagged_value()
}
pub fn table(list: &[Value]) -> Value {
UntaggedValue::table(list).into_untagged_value()
}
pub fn date(input: impl Into<String>) -> Value {
let key = input.into().tagged_unknown();
crate::data::value::Date::naive_from_str(key.borrow_tagged())
.unwrap()
.into_untagged_value()
}
pub fn committers() -> Vec<Value> {
vec![
row(indexmap! {
"date".into() => date("2019-07-23"),
"name".into() => string("AR"),
"country".into() => string("EC"),
"chickens".into() => int(10),
}),
row(indexmap! {
"date".into() => date("2019-07-23"),
"name".into() => string("JT"),
"country".into() => string("NZ"),
"chickens".into() => int(5),
}),
row(indexmap! {
"date".into() => date("2019-10-10"),
"name".into() => string("YK"),
"country".into() => string("US"),
"chickens".into() => int(6),
}),
row(indexmap! {
"date".into() => date("2019-09-24"),
"name".into() => string("AR"),
"country".into() => string("EC"),
"chickens".into() => int(20),
}),
row(indexmap! {
"date".into() => date("2019-10-10"),
"name".into() => string("JT"),
"country".into() => string("NZ"),
"chickens".into() => int(15),
}),
row(indexmap! {
"date".into() => date("2019-09-24"),
"name".into() => string("YK"),
"country".into() => string("US"),
"chickens".into() => int(4),
}),
row(indexmap! {
"date".into() => date("2019-10-10"),
"name".into() => string("AR"),
"country".into() => string("EC"),
"chickens".into() => int(30),
}),
row(indexmap! {
"date".into() => date("2019-09-24"),
"name".into() => string("JT"),
"country".into() => string("NZ"),
"chickens".into() => int(10),
}),
row(indexmap! {
"date".into() => date("2019-07-23"),
"name".into() => string("YK"),
"country".into() => string("US"),
"chickens".into() => int(2),
}),
]
}
pub fn committers_grouped_by_date() -> Value {
let sample = table(&committers());
let grouper = Box::new(move |_, row: &Value| {
let key = String::from("date").tagged_unknown();
let group_key = row.get_data_by_key(key.borrow_spanned()).unwrap();
group_key.format("%Y-%m-%d")
});
crate::utils::data::group(&sample, &Some(grouper), Tag::unknown()).unwrap()
}
pub fn date_formatter(
fmt: &'static str,
) -> Box<dyn Fn(&Value, String) -> Result<String, ShellError>> {
Box::new(move |date: &Value, _: String| date.format(&fmt))
}
fn assert_without_checking_percentages(report_a: Model, report_b: Model) {
assert_eq!(report_a.labels.x, report_b.labels.x);
assert_eq!(report_a.labels.y, report_b.labels.y);
assert_eq!(report_a.ranges, report_b.ranges);
assert_eq!(report_a.data, report_b.data);
}
#[test]
fn prepares_report_using_accumulating_value() {
let committers = table(&committers());
let by_date = Box::new(move |_, row: &Value| {
let key = String::from("date").tagged_unknown();
let key = row.get_data_by_key(key.borrow_spanned()).unwrap();
let callback = date_formatter("%Y-%m-%d");
callback(&key, "nothing".to_string())
});
let by_country = Box::new(move |_, row: &Value| {
let key = String::from("country").tagged_unknown();
let key = row.get_data_by_key(key.borrow_spanned()).unwrap();
nu_value_ext::as_string(&key)
});
let options = Operation {
grouper: Some(by_date),
splitter: Some(by_country),
format: Some(date_formatter("%Y-%m-%d")),
eval: /* value to be used for accumulation */ &Some(Box::new(move |_, value: &Value| {
let chickens_key = String::from("chickens").tagged_unknown();
value
.get_data_by_key(chickens_key.borrow_spanned())
.ok_or_else(|| {
ShellError::labeled_error(
"unknown column",
"unknown column",
chickens_key.span(),
)
})
})),
};
assert_without_checking_percentages(
report(&committers, options, Tag::unknown()).unwrap(),
Model {
labels: Labels {
x: vec![
String::from("2019-07-23"),
String::from("2019-09-24"),
String::from("2019-10-10"),
],
y: vec![String::from("EC"), String::from("NZ"), String::from("US")],
},
ranges: (
Range {
start: int(0),
end: int(3),
},
Range {
start: int(0),
end: int(60),
},
),
data: table(&[
table(&[int(10), int(30), int(60)]),
table(&[int(5), int(15), int(30)]),
table(&[int(2), int(6), int(12)]),
]),
percentages: table(&[
table(&[decimal(16.66), decimal(50), decimal(100)]),
table(&[decimal(8.33), decimal(25), decimal(50)]),
table(&[decimal(3.33), decimal(10), decimal(20)]),
]),
},
);
}
}

6
crates/nu-cli/src/utils/data/split.rs
View File

@ -13,6 +13,12 @@ pub fn split(
let tag = tag.into(); let tag = tag.into();
let mut splits = indexmap::IndexMap::new(); let mut splits = indexmap::IndexMap::new();
let mut out = TaggedDictBuilder::new(&tag);
if splitter.is_none() {
out.insert_untagged("table", UntaggedValue::table(&[value.clone()]));
return Ok(out.into_value());
}
for (column, value) in value.row_entries() { for (column, value) in value.row_entries() {
if !&value.is_table() { if !&value.is_table() {

675
crates/nu-cli/src/utils/data_processing.rs
View File

@ -1,675 +0,0 @@
use crate::data::value::compare_values;
use crate::data::TaggedListBuilder;
use chrono::{DateTime, NaiveDate, Utc};
use nu_errors::ShellError;
use nu_protocol::hir::Operator;
use nu_protocol::{Primitive, TaggedDictBuilder, UntaggedValue, Value};
use nu_source::{SpannedItem, Tag, Tagged, TaggedItem};
use nu_value_ext::{get_data_by_key, ValueExt};
use num_traits::Zero;
// Re-usable error messages
const ERR_EMPTY_DATA: &str = "Cannot perform aggregate math operation on empty data";
pub fn columns_sorted(
_group_by_name: Option<Tagged<String>>,
value: &Value,
tag: impl Into<Tag>,
) -> Vec<Tagged<String>> {
let origin_tag = tag.into();
match value {
Value {
value: UntaggedValue::Row(rows),
..
} => {
let mut keys: Vec<Value> = rows
.entries
.keys()
.map(|s| s.as_ref())
.map(|k: &str| {
let date = NaiveDate::parse_from_str(k, "%B %d-%Y");
let date = match date {
Ok(parsed) => UntaggedValue::Primitive(Primitive::Date(
DateTime::<Utc>::from_utc(parsed.and_hms(12, 34, 56), Utc),
)),
Err(_) => UntaggedValue::string(k),
};
date.into_untagged_value()
})
.collect();
keys.sort();
let keys: Vec<String> = keys
.into_iter()
.map(|k| match k {
Value {
value: UntaggedValue::Primitive(Primitive::Date(d)),
..
} => format!("{}", d.format("%B %d-%Y")),
_ => k.as_string().unwrap_or_else(|_| String::from("<string>")),
})
.collect();
keys.into_iter().map(|k| k.tagged(&origin_tag)).collect()
}
_ => vec!["default".to_owned().tagged(&origin_tag)],
}
}
pub fn t_sort(
group_by_name: Option<Tagged<String>>,
split_by_name: Option<String>,
value: &Value,
tag: impl Into<Tag>,
) -> Result<Value, ShellError> {
let origin_tag = tag.into();
match group_by_name {
Some(column_name) => {
let sorted_labels: Vec<Tagged<String>> =
columns_sorted(Some(column_name), value, &origin_tag);
match split_by_name {
None => {
let mut dataset = TaggedDictBuilder::new(&origin_tag);
dataset.insert_value("default", value.clone());
let dataset = dataset.into_value();
let split_labels: Vec<Tagged<String>> = match &dataset {
Value {
value: UntaggedValue::Row(rows),
..
} => {
let mut keys: Vec<Tagged<String>> = rows
.entries
.keys()
.map(|k| k.clone().tagged_unknown())
.collect();
keys.sort();
keys
}
_ => vec![],
};
let results: Vec<Vec<Value>> = split_labels
.iter()
.map(|split| {
let groups = get_data_by_key(&dataset, split.borrow_spanned());
sorted_labels
.clone()
.into_iter()
.map(|label| match &groups {
Some(Value {
value: UntaggedValue::Row(dict),
..
}) => {
dict.get_data_by_key(label.borrow_spanned()).unwrap_or_else(
|| UntaggedValue::Table(vec![]).into_value(&origin_tag),
)
}
_ => UntaggedValue::Table(vec![]).into_value(&origin_tag),
})
.collect()
})
.collect();
let mut outer = TaggedListBuilder::new(&origin_tag);
for i in results {
outer.push_value(UntaggedValue::Table(i).into_value(&origin_tag));
}
Ok(UntaggedValue::Table(outer.list).into_value(&origin_tag))
}
Some(_) => Ok(UntaggedValue::nothing().into_value(&origin_tag)),
}
}
None => Ok(UntaggedValue::nothing().into_value(&origin_tag)),
}
}
pub fn fetch(key: Option<String>) -> Box<dyn Fn(Value, Tag) -> Option<Value> + 'static> {
Box::new(move |value: Value, tag| match &key {
Some(key_given) => value.get_data_by_key(key_given[..].spanned(tag.span)),
None => Some(UntaggedValue::int(1).into_value(tag)),
})
}
pub fn evaluate(
values: &Value,
evaluator: Option<String>,
tag: impl Into<Tag>,
) -> Result<Value, ShellError> {
let tag = tag.into();
let evaluate_with = match evaluator {
Some(keyfn) => fetch(Some(keyfn)),
None => fetch(None),
};
let results: Value = match values {
Value {
value: UntaggedValue::Table(datasets),
..
} => {
let datasets: Vec<_> = datasets
.iter()
.map(|subsets| match subsets {
Value {
value: UntaggedValue::Table(subsets),
..
} => {
let subsets: Vec<_> = subsets
.clone()
.into_iter()
.map(|data| match data {
Value {
value: UntaggedValue::Table(data),
..
} => {
let data: Vec<_> = data
.into_iter()
.map(|x| match evaluate_with(x, tag.clone()) {
Some(val) => val,
None => UntaggedValue::int(1).into_value(tag.clone()),
})
.collect();
UntaggedValue::Table(data).into_value(&tag)
}
_ => UntaggedValue::Table(vec![]).into_value(&tag),
})
.collect();
UntaggedValue::Table(subsets).into_value(&tag)
}
_ => UntaggedValue::Table(vec![]).into_value(&tag),
})
.collect();
UntaggedValue::Table(datasets).into_value(&tag)
}
_ => UntaggedValue::Table(vec![]).into_value(&tag),
};
Ok(results)
}
pub fn sum(data: Vec<Value>) -> Result<Value, ShellError> {
if data.is_empty() {
return Err(ShellError::unexpected(ERR_EMPTY_DATA));
}
let mut acc = Value::zero();
for value in data {
match value.value {
UntaggedValue::Primitive(_) => acc = acc + value,
_ => {
return Err(ShellError::labeled_error(
"Attempted to compute the sum of a value that cannot be summed.",
"value appears here",
value.tag.span,
))
}
}
}
Ok(acc)
}
pub fn max(data: Vec<Value>) -> Result<Value, ShellError> {
let mut biggest = data
.first()
.ok_or_else(|| ShellError::unexpected(ERR_EMPTY_DATA))?
.value
.clone();
for value in data.iter() {
if let Ok(greater_than) = compare_values(Operator::GreaterThan, &value.value, &biggest) {
if greater_than {
biggest = value.value.clone();
}
} else {
return Err(ShellError::unexpected(format!(
"Could not compare\nleft: {:?}\nright: {:?}",
biggest, value.value
)));
}
}
Ok(Value {
value: biggest,
tag: Tag::unknown(),
})
}
pub fn min(data: Vec<Value>) -> Result<Value, ShellError> {
let mut smallest = data
.first()
.ok_or_else(|| ShellError::unexpected(ERR_EMPTY_DATA))?
.value
.clone();
for value in data.iter() {
if let Ok(greater_than) = compare_values(Operator::LessThan, &value.value, &smallest) {
if greater_than {
smallest = value.value.clone();
}
} else {
return Err(ShellError::unexpected(format!(
"Could not compare\nleft: {:?}\nright: {:?}",
smallest, value.value
)));
}
}
Ok(Value {
value: smallest,
tag: Tag::unknown(),
})
}
fn formula(
acc_begin: Value,
calculator: Box<dyn Fn(Vec<Value>) -> Result<Value, ShellError> + Send + Sync + 'static>,
) -> Box<dyn Fn(Value, Vec<Value>) -> Result<Value, ShellError> + Send + Sync + 'static> {
Box::new(move |acc, datax| -> Result<Value, ShellError> {
let result = acc * acc_begin.clone();
match calculator(datax) {
Ok(total) => Ok(result + total),
Err(reason) => Err(reason),
}
})
}
pub fn reducer_for(
command: Reduce,
) -> Box<dyn Fn(Value, Vec<Value>) -> Result<Value, ShellError> + Send + Sync + 'static> {
match command {
Reduce::Summation | Reduce::Default => Box::new(formula(Value::zero(), Box::new(sum))),
Reduce::Minimum => Box::new(|_, values| min(values)),
Reduce::Maximum => Box::new(|_, values| max(values)),
}
}
pub enum Reduce {
Summation,
Minimum,
Maximum,
Default,
}
pub fn reduce(
values: &Value,
reducer: Option<String>,
tag: impl Into<Tag>,
) -> Result<Value, ShellError> {
let tag = tag.into();
let reduce_with = match reducer {
Some(cmd) if cmd == "sum" => reducer_for(Reduce::Summation),
Some(cmd) if cmd == "min" => reducer_for(Reduce::Minimum),
Some(cmd) if cmd == "max" => reducer_for(Reduce::Maximum),
Some(_) | None => reducer_for(Reduce::Default),
};
let results: Value = match values {
Value {
value: UntaggedValue::Table(datasets),
..
} => {
let datasets: Vec<_> = datasets
.iter()
.map(|subsets| {
let acc = Value::zero();
match subsets {
Value {
value: UntaggedValue::Table(data),
..
} => {
let data = data
.iter()
.map(|d| {
if let Value {
value: UntaggedValue::Table(x),
..
} = d
{
if let Ok(Value {
value:
UntaggedValue::Primitive(Primitive::Int(computed)),
..
}) = reduce_with(acc.clone(), x.clone())
{
UntaggedValue::int(computed).into_value(&tag)
} else {
UntaggedValue::int(0).into_value(&tag)
}
} else {
UntaggedValue::int(0).into_value(&tag)
}
})
.collect::<Vec<_>>();
UntaggedValue::Table(data).into_value(&tag)
}
_ => UntaggedValue::Table(vec![]).into_value(&tag),
}
})
.collect();
UntaggedValue::Table(datasets).into_value(&tag)
}
_ => UntaggedValue::Table(vec![]).into_value(&tag),
};
Ok(results)
}
pub fn map_max(
values: &Value,
_map_by_column_name: Option<String>,
tag: impl Into<Tag>,
) -> Result<Value, ShellError> {
let tag = tag.into();
let results: Value = match values {
Value {
value: UntaggedValue::Table(datasets),
..
} => {
let datasets: Vec<Value> = datasets
.iter()
.map(|subsets| match subsets {
Value {
value: UntaggedValue::Table(data),
..
} => data.iter().fold(Value::zero(), |acc, value| {
let left = &value.value;
let right = &acc.value;
if let Ok(is_greater_than) =
compare_values(Operator::GreaterThan, left, right)
{
if is_greater_than {
value.clone()
} else {
acc
}
} else {
acc
}
}),
_ => UntaggedValue::int(0).into_value(&tag),
})
.collect();
datasets.into_iter().fold(Value::zero(), |max, value| {
let left = &value.value;
let right = &max.value;
if let Ok(is_greater_than) = compare_values(Operator::GreaterThan, left, right) {
if is_greater_than {
value
} else {
max
}
} else {
max
}
})
}
_ => UntaggedValue::int(-1).into_value(&tag),
};
Ok(results)
}
#[cfg(test)]
mod tests {
use super::{columns_sorted, evaluate, fetch, map_max, reduce, reducer_for, t_sort, Reduce};
use crate::commands::group_by::group;
use indexmap::IndexMap;
use nu_errors::ShellError;
use nu_protocol::{UntaggedValue, Value};
use nu_source::*;
use num_bigint::BigInt;
use num_traits::Zero;
fn int(s: impl Into<BigInt>) -> Value {
UntaggedValue::int(s).into_untagged_value()
}
fn string(input: impl Into<String>) -> Value {
UntaggedValue::string(input.into()).into_untagged_value()
}
fn row(entries: IndexMap<String, Value>) -> Value {
UntaggedValue::row(entries).into_untagged_value()
}
fn table(list: &[Value]) -> Value {
UntaggedValue::table(list).into_untagged_value()
}
fn nu_releases_grouped_by_date() -> Result<Value, ShellError> {
let key = Some(String::from("date").tagged_unknown());
let sample = table(&nu_releases_committers());
group(&key, &sample, Tag::unknown())
}
fn nu_releases_sorted_by_date() -> Result<Value, ShellError> {
let key = String::from("date").tagged(Tag::unknown());
t_sort(
Some(key),
None,
&nu_releases_grouped_by_date()?,
Tag::unknown(),
)
}
fn nu_releases_evaluated_by_default_one() -> Result<Value, ShellError> {
evaluate(&nu_releases_sorted_by_date()?, None, Tag::unknown())
}
fn nu_releases_reduced_by_sum() -> Result<Value, ShellError> {
reduce(
&nu_releases_evaluated_by_default_one()?,
Some(String::from("sum")),
Tag::unknown(),
)
}
fn nu_releases_committers() -> Vec<Value> {
vec![
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")},
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")},
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")},
),
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("September 24-2019")},
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")},
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("September 24-2019")},
),
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")},
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("September 24-2019")},
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")},
),
]
}
#[test]
fn show_columns_sorted_given_a_column_to_sort_by() -> Result<(), ShellError> {
let by_column = String::from("date").tagged(Tag::unknown());
assert_eq!(
columns_sorted(
Some(by_column),
&nu_releases_grouped_by_date()?,
Tag::unknown()
),
vec![
"August 23-2019".to_string().tagged_unknown(),
"September 24-2019".to_string().tagged_unknown(),
"October 10-2019".to_string().tagged_unknown()
]
);
Ok(())
}
#[test]
fn sorts_the_tables() -> Result<(), ShellError> {
let group_by = String::from("date").tagged(Tag::unknown());
assert_eq!(
t_sort(
Some(group_by),
None,
&nu_releases_grouped_by_date()?,
Tag::unknown()
)?,
table(&[table(&[
table(&[
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("August 23-2019")}
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("August 23-2019")}
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("August 23-2019")}
)
]),
table(&[
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("September 24-2019")}
),
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("September 24-2019")}
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("September 24-2019")}
)
]),
table(&[
row(
indexmap! {"name".into() => string("YK"), "country".into() => string("US"), "date".into() => string("October 10-2019")}
),
row(
indexmap! {"name".into() => string("JT"), "country".into() => string("NZ"), "date".into() => string("October 10-2019")}
),
row(
indexmap! {"name".into() => string("AR"), "country".into() => string("EC"), "date".into() => string("October 10-2019")}
)
]),
]),])
);
Ok(())
}
#[test]
fn evaluator_fetches_by_column_if_supplied_a_column_name() -> Result<(), ShellError> {
let subject = row(indexmap! { "name".into() => string("andres") });
let evaluator = fetch(Some(String::from("name")));
assert_eq!(evaluator(subject, Tag::unknown()), Some(string("andres")));
Ok(())
}
#[test]
fn evaluator_returns_1_if_no_column_name_given() -> Result<(), ShellError> {
let subject = row(indexmap! { "name".into() => string("andres") });
let evaluator = fetch(None);
assert_eq!(
evaluator(subject, Tag::unknown()),
Some(UntaggedValue::int(1).into_untagged_value())
);
Ok(())
}
#[test]
fn evaluates_the_tables() -> Result<(), ShellError> {
assert_eq!(
evaluate(&nu_releases_sorted_by_date()?, None, Tag::unknown())?,
table(&[table(&[
table(&[int(1), int(1), int(1)]),
table(&[int(1), int(1), int(1)]),
table(&[int(1), int(1), int(1)]),
]),])
);
Ok(())
}
#[test]
fn evaluates_the_tables_with_custom_evaluator() -> Result<(), ShellError> {
let eval = String::from("name");
assert_eq!(
evaluate(&nu_releases_sorted_by_date()?, Some(eval), Tag::unknown())?,
table(&[table(&[
table(&[string("AR"), string("JT"), string("YK")]),
table(&[string("AR"), string("YK"), string("JT")]),
table(&[string("YK"), string("JT"), string("AR")]),
]),])
);
Ok(())
}
#[test]
fn reducer_computes_given_a_sum_command() -> Result<(), ShellError> {
let subject = vec![int(1), int(1), int(1)];
let action = reducer_for(Reduce::Summation);
assert_eq!(action(Value::zero(), subject)?, int(3));
Ok(())
}
#[test]
fn reducer_computes() -> Result<(), ShellError> {
assert_eq!(
reduce(
&nu_releases_evaluated_by_default_one()?,
Some(String::from("sum")),
Tag::unknown()
)?,
table(&[table(&[int(3), int(3), int(3)])])
);
Ok(())
}
#[test]
fn maps_and_gets_max_value() -> Result<(), ShellError> {
assert_eq!(
map_max(&nu_releases_reduced_by_sum()?, None, Tag::unknown())?,
int(3)
);
Ok(())
}
}

25
crates/nu-cli/tests/commands/math/sum.rs
View File

@ -36,30 +36,9 @@ fn all() {
#[test] #[test]
fn outputs_zero_with_no_input() { fn outputs_zero_with_no_input() {
Playground::setup("sum_test_2", |dirs, sandbox| { let actual = nu!(cwd: ".", "math sum | echo $it");
sandbox.with_files(vec![FileWithContentToBeTrimmed(
"meals.json",
r#"
{
meals: [
{description: "1 large egg", calories: 90},
{description: "1 cup white rice", calories: 250},
{description: "1 tablespoon fish oil", calories: 108}
]
}
"#,
)]);
let actual = nu!(
cwd: dirs.test(), pipeline(
r#"
math sum
| echo $it
"#
));
assert_eq!(actual.out, "0"); assert_eq!(actual.out, "0");
})
} }
#[test] #[test]
@ -112,5 +91,5 @@ fn sum_of_a_row_containing_a_table_is_an_error() {
); );
assert!(actual assert!(actual
.err .err
.contains("Attempted to compute the sum of a value that cannot be summed.")); .contains("Attempted to compute values that can't be operated on"));
} }

62
crates/nu-protocol/src/value.rs
View File

@ -78,6 +78,11 @@ impl UntaggedValue {
matches!(self, UntaggedValue::Primitive(Primitive::Boolean(true))) matches!(self, UntaggedValue::Primitive(Primitive::Boolean(true)))
} }
/// Returns true if this value represents a filesize
pub fn is_filesize(&self) -> bool {
matches!(self, UntaggedValue::Primitive(Primitive::Filesize(_)))
}
/// Returns true if this value represents a table /// Returns true if this value represents a table
pub fn is_table(&self) -> bool { pub fn is_table(&self) -> bool {
matches!(self, UntaggedValue::Table(_)) matches!(self, UntaggedValue::Table(_))
@ -281,7 +286,7 @@ impl Value {
pub fn convert_to_string(&self) -> String { pub fn convert_to_string(&self) -> String {
match &self.value { match &self.value {
UntaggedValue::Primitive(Primitive::String(s)) => s.clone(), UntaggedValue::Primitive(Primitive::String(s)) => s.clone(),
UntaggedValue::Primitive(Primitive::Date(dt)) => dt.format("%Y-%b-%d").to_string(), UntaggedValue::Primitive(Primitive::Date(dt)) => dt.format("%Y-%m-%d").to_string(),
UntaggedValue::Primitive(Primitive::Boolean(x)) => format!("{}", x), UntaggedValue::Primitive(Primitive::Boolean(x)) => format!("{}", x),
UntaggedValue::Primitive(Primitive::Decimal(x)) => format!("{}", x), UntaggedValue::Primitive(Primitive::Decimal(x)) => format!("{}", x),
UntaggedValue::Primitive(Primitive::Int(x)) => format!("{}", x), UntaggedValue::Primitive(Primitive::Int(x)) => format!("{}", x),
@ -494,61 +499,6 @@ impl ShellTypeName for UntaggedValue {
} }
} }
impl num_traits::Zero for Value {
fn zero() -> Self {
Value {
value: UntaggedValue::Primitive(Primitive::zero()),
tag: Tag::unknown(),
}
}
fn is_zero(&self) -> bool {
match &self.value {
UntaggedValue::Primitive(primitive) => primitive.is_zero(),
UntaggedValue::Row(row) => row.entries.is_empty(),
UntaggedValue::Table(rows) => rows.is_empty(),
_ => false,
}
}
}
impl std::ops::Mul for Value {
type Output = Self;
fn mul(self, rhs: Self) -> Self {
let tag = self.tag.clone();
match (&*self, &*rhs) {
(UntaggedValue::Primitive(left), UntaggedValue::Primitive(right)) => {
let left = left.clone();
let right = right.clone();
UntaggedValue::from(left.mul(right)).into_value(tag)
}
(_, _) => unimplemented!("Internal error: can't multiply non-primitives."),
}
}
}
impl std::ops::Add for Value {
type Output = Self;
fn add(self, rhs: Self) -> Self {
let tag = self.tag.clone();
match (&*self, &*rhs) {
(UntaggedValue::Primitive(left), UntaggedValue::Primitive(right)) => {
let left = left.clone();
let right = right.clone();
UntaggedValue::from(left.add(right)).into_value(tag)
}
(_, _) => UntaggedValue::Error(ShellError::unimplemented("Can't add non-primitives."))
.into_value(tag),
}
}
}
pub fn merge_descriptors(values: &[Value]) -> Vec<String> { pub fn merge_descriptors(values: &[Value]) -> Vec<String> {
let mut ret: Vec<String> = vec![]; let mut ret: Vec<String> = vec![];
let value_column = "".to_string(); let value_column = "".to_string();

84
crates/nu-protocol/src/value/primitive.rs
View File

@ -72,8 +72,15 @@ impl Primitive {
"converting an integer into a 64-bit integer", "converting an integer into a 64-bit integer",
) )
}), }),
Primitive::Decimal(decimal) => decimal.to_u64().ok_or_else(|| {
ShellError::range_error(
ExpectedRange::U64,
&format!("{}", decimal).spanned(span),
"converting a decimal into a 64-bit integer",
)
}),
other => Err(ShellError::type_error( other => Err(ShellError::type_error(
"integer", "number",
other.type_name().spanned(span), other.type_name().spanned(span),
)), )),
} }
@ -132,81 +139,6 @@ impl Primitive {
} }
} }
impl num_traits::Zero for Primitive {
fn zero() -> Self {
Primitive::Int(BigInt::zero())
}
fn is_zero(&self) -> bool {
match self {
Primitive::Int(int) => int.is_zero(),
Primitive::Decimal(decimal) => decimal.is_zero(),
Primitive::Filesize(num_bytes) => num_bytes.is_zero(),
_ => false,
}
}
}
impl std::ops::Add for Primitive {
type Output = Primitive;
fn add(self, rhs: Self) -> Self {
match (self, rhs) {
(Primitive::Int(left), Primitive::Int(right)) => Primitive::Int(left + right),
(Primitive::Int(left), Primitive::Decimal(right)) => {
Primitive::Decimal(BigDecimal::from(left) + right)
}
(Primitive::Decimal(left), Primitive::Decimal(right)) => {
Primitive::Decimal(left + right)
}
(Primitive::Decimal(left), Primitive::Int(right)) => {
Primitive::Decimal(left + BigDecimal::from(right))
}
(Primitive::Filesize(left), right) => match right {
Primitive::Filesize(right) => Primitive::Filesize(left + right),
Primitive::Int(right) => {
Primitive::Filesize(left + right.to_u64().unwrap_or_else(|| 0 as u64))
}
Primitive::Decimal(right) => {
Primitive::Filesize(left + right.to_u64().unwrap_or_else(|| 0 as u64))
}
_ => Primitive::Filesize(left),
},
(left, Primitive::Filesize(right)) => match left {
Primitive::Filesize(left) => Primitive::Filesize(left + right),
Primitive::Int(left) => {
Primitive::Filesize(left.to_u64().unwrap_or_else(|| 0 as u64) + right)
}
Primitive::Decimal(left) => {
Primitive::Filesize(left.to_u64().unwrap_or_else(|| 0 as u64) + right)
}
_ => Primitive::Filesize(right),
},
_ => Primitive::zero(),
}
}
}
impl std::ops::Mul for Primitive {
type Output = Self;
fn mul(self, rhs: Self) -> Self {
match (self, rhs) {
(Primitive::Int(left), Primitive::Int(right)) => Primitive::Int(left * right),
(Primitive::Int(left), Primitive::Decimal(right)) => {
Primitive::Decimal(BigDecimal::from(left) * right)
}
(Primitive::Decimal(left), Primitive::Decimal(right)) => {
Primitive::Decimal(left * right)
}
(Primitive::Decimal(left), Primitive::Int(right)) => {
Primitive::Decimal(left * BigDecimal::from(right))
}
_ => unimplemented!("Internal error: can't multiply incompatible primitives."),
}
}
}
impl From<&str> for Primitive { impl From<&str> for Primitive {
/// Helper to convert from string slices to a primitive /// Helper to convert from string slices to a primitive
fn from(s: &str) -> Primitive { fn from(s: &str) -> Primitive {

2
crates/nu-value-ext/src/lib.rs
View File

@ -556,7 +556,7 @@ pub fn as_path_member(value: &Value) -> Result<PathMember, ShellError> {
pub fn as_string(value: &Value) -> Result<String, ShellError> { pub fn as_string(value: &Value) -> Result<String, ShellError> {
match &value.value { match &value.value {
UntaggedValue::Primitive(Primitive::String(s)) => Ok(s.clone()), UntaggedValue::Primitive(Primitive::String(s)) => Ok(s.clone()),
UntaggedValue::Primitive(Primitive::Date(dt)) => Ok(dt.format("%Y-%b-%d").to_string()), UntaggedValue::Primitive(Primitive::Date(dt)) => Ok(dt.format("%Y-%m-%d").to_string()),
UntaggedValue::Primitive(Primitive::Boolean(x)) => Ok(format!("{}", x)), UntaggedValue::Primitive(Primitive::Boolean(x)) => Ok(format!("{}", x)),
UntaggedValue::Primitive(Primitive::Decimal(x)) => Ok(format!("{}", x)), UntaggedValue::Primitive(Primitive::Decimal(x)) => Ok(format!("{}", x)),
UntaggedValue::Primitive(Primitive::Int(x)) => Ok(format!("{}", x)), UntaggedValue::Primitive(Primitive::Int(x)) => Ok(format!("{}", x)),