From c7d159a0f39275913f79a22c10460a9c7c6f3f3b Mon Sep 17 00:00:00 2001 From: Luccas Mateus Date: Fri, 5 Nov 2021 14:58:40 -0300 Subject: [PATCH] Last three math commands, `eval`, `variance` and `stddev` (#292) * MathEval Variance and Stddev * Fix tests and linting * Typo --- Cargo.lock | 23 +++++ crates/nu-command/Cargo.toml | 1 + crates/nu-command/src/default_context.rs | 3 + crates/nu-command/src/math/eval.rs | 117 +++++++++++++++++++++ crates/nu-command/src/math/mod.rs | 6 ++ crates/nu-command/src/math/stddev.rs | 85 +++++++++++++++ crates/nu-command/src/math/utils.rs | 12 ++- crates/nu-command/src/math/variance.rs | 126 +++++++++++++++++++++++ 8 files changed, 368 insertions(+), 5 deletions(-) create mode 100644 crates/nu-command/src/math/eval.rs create mode 100644 crates/nu-command/src/math/stddev.rs create mode 100644 crates/nu-command/src/math/variance.rs diff --git a/Cargo.lock b/Cargo.lock index d38d6ea45..edc3be6bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -404,6 +404,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "getrandom" version = "0.2.3" @@ -545,6 +551,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "meval" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f79496a5651c8d57cd033c5add8ca7ee4e3d5f7587a4777484640d9cb60392d9" +dependencies = [ + "fnv", + "nom", +] + [[package]] name = "miette" version = "3.2.0" @@ -620,6 +636,12 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nom" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5b8c256fd9471521bcb84c3cdba98921497f1a331cbc15b8030fc63b82050ce" + [[package]] name = "ntapi" version = "0.3.6" @@ -665,6 +687,7 @@ dependencies = [ "dialoguer", "glob", "lscolors", + "meval", "nu-engine", "nu-json", "nu-parser", diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 6e0353206..f510b22f9 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -30,6 +30,7 @@ bytesize = "1.1.0" dialoguer = "0.9.0" rayon = "1.5.1" titlecase = "1.1.0" +meval = "0.2.0" [features] trash-support = ["trash"] diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index 02d292209..be85486ff 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -66,6 +66,7 @@ pub fn create_default_context() -> EngineState { MathAvg, MathCeil, MathFloor, + MathEval, MathMax, MathMedian, MathMin, @@ -73,7 +74,9 @@ pub fn create_default_context() -> EngineState { MathProduct, MathRound, MathSqrt, + MathStddev, MathSum, + MathVariance, Mkdir, Module, Mv, diff --git a/crates/nu-command/src/math/eval.rs b/crates/nu-command/src/math/eval.rs new file mode 100644 index 000000000..b8c79c116 --- /dev/null +++ b/crates/nu-command/src/math/eval.rs @@ -0,0 +1,117 @@ +use nu_engine::CallExt; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{ + Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Value, +}; + +#[derive(Clone)] +pub struct SubCommand; + +impl Command for SubCommand { + fn name(&self) -> &str { + "math eval" + } + + fn usage(&self) -> &str { + "Evaluate a math expression into a number" + } + + fn signature(&self) -> Signature { + Signature::build("math eval").optional( + "math expression", + SyntaxShape::String, + "the math expression to evaluate", + ) + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let spanned_expr: Option> = call.opt(engine_state, stack, 0)?; + eval(spanned_expr, input, engine_state) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "Evalulate math in the pipeline", + example: "'10 / 4' | math eval", + result: Some(Value::Float { + val: 2.5, + span: Span::unknown(), + }), + }] + } +} + +pub fn eval( + spanned_expr: Option>, + input: PipelineData, + engine_state: &EngineState, +) -> Result { + if let Some(expr) = spanned_expr { + match parse(&expr.item, &expr.span) { + Ok(value) => Ok(PipelineData::Value(value)), + Err(err) => Err(ShellError::UnsupportedInput( + format!("Math evaluation error: {}", err), + expr.span, + )), + } + } else { + if let PipelineData::Value(Value::Nothing { .. }) = input { + return Ok(input); + } + input.map( + move |val| { + if let Ok(string) = val.as_string() { + match parse(&string, &val.span().unwrap_or_else(|_| Span::unknown())) { + Ok(value) => value, + Err(err) => Value::Error { + error: ShellError::UnsupportedInput( + format!("Math evaluation error: {}", err), + val.span().unwrap_or_else(|_| Span::unknown()), + ), + }, + } + } else { + Value::Error { + error: ShellError::UnsupportedInput( + "Expected a string from pipeline".to_string(), + val.span().unwrap_or_else(|_| Span::unknown()), + ), + } + } + }, + engine_state.ctrlc.clone(), + ) + } +} + +pub fn parse(math_expression: &str, span: &Span) -> Result { + let mut ctx = meval::Context::new(); + ctx.var("tau", std::f64::consts::TAU); + match meval::eval_str_with_context(math_expression, &ctx) { + Ok(num) if num.is_infinite() || num.is_nan() => Err("cannot represent result".to_string()), + Ok(num) => Ok(Value::Float { + val: num, + span: *span, + }), + Err(error) => Err(error.to_string().to_lowercase()), + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(SubCommand {}) + } +} diff --git a/crates/nu-command/src/math/mod.rs b/crates/nu-command/src/math/mod.rs index bf392f68c..0309e8df9 100644 --- a/crates/nu-command/src/math/mod.rs +++ b/crates/nu-command/src/math/mod.rs @@ -2,6 +2,7 @@ mod abs; mod avg; mod ceil; pub mod command; +mod eval; mod floor; mod max; mod median; @@ -11,13 +12,16 @@ mod product; mod reducers; mod round; mod sqrt; +mod stddev; mod sum; mod utils; +mod variance; pub use abs::SubCommand as MathAbs; pub use avg::SubCommand as MathAvg; pub use ceil::SubCommand as MathCeil; pub use command::MathCommand as Math; +pub use eval::SubCommand as MathEval; pub use floor::SubCommand as MathFloor; pub use max::SubCommand as MathMax; pub use median::SubCommand as MathMedian; @@ -26,4 +30,6 @@ pub use mode::SubCommand as MathMode; pub use product::SubCommand as MathProduct; pub use round::SubCommand as MathRound; pub use sqrt::SubCommand as MathSqrt; +pub use stddev::SubCommand as MathStddev; pub use sum::SubCommand as MathSum; +pub use variance::SubCommand as MathVariance; diff --git a/crates/nu-command/src/math/stddev.rs b/crates/nu-command/src/math/stddev.rs new file mode 100644 index 000000000..2d7b2393e --- /dev/null +++ b/crates/nu-command/src/math/stddev.rs @@ -0,0 +1,85 @@ +use super::variance::compute_variance as variance; +use crate::math::utils::run_with_function; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{Example, PipelineData, ShellError, Signature, Span, Value}; + +#[derive(Clone)] +pub struct SubCommand; + +impl Command for SubCommand { + fn name(&self) -> &str { + "math stddev" + } + + fn signature(&self) -> Signature { + Signature::build("math stddev").switch( + "sample", + "calculate sample standard deviation", + Some('s'), + ) + } + + fn usage(&self) -> &str { + "Finds the stddev of a list of numbers or tables" + } + + fn run( + &self, + _engine_state: &EngineState, + _stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let sample = call.has_flag("sample"); + run_with_function(call, input, compute_stddev(sample)) + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Get the stddev of a list of numbers", + example: "[1 2 3 4 5] | math stddev", + result: Some(Value::Float { + val: std::f64::consts::SQRT_2, + span: Span::unknown(), + }), + }, + Example { + description: "Get the sample stddev of a list of numbers", + example: "[1 2 3 4 5] | math stddev -s", + result: Some(Value::Float { + val: 1.5811388300841898, + span: Span::unknown(), + }), + }, + ] + } +} + +pub fn compute_stddev(sample: bool) -> impl Fn(&[Value], &Span) -> Result { + move |values: &[Value], span: &Span| { + let variance = variance(sample)(values, span); + match variance { + Ok(Value::Float { val, span }) => Ok(Value::Float { val: val.sqrt(), span }), + Ok(Value::Int { val, span }) => Ok(Value::Float { val: (val as f64).sqrt(), span }), + Err(ShellError::UnsupportedInput(_, err_span)) => Err(ShellError::UnsupportedInput( + "Attempted to compute the standard deviation with an item that cannot be used for that.".to_string(), + err_span, + )), + other => other + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(SubCommand {}) + } +} diff --git a/crates/nu-command/src/math/utils.rs b/crates/nu-command/src/math/utils.rs index 770940c4c..9ffffbebb 100644 --- a/crates/nu-command/src/math/utils.rs +++ b/crates/nu-command/src/math/utils.rs @@ -2,12 +2,10 @@ use nu_protocol::ast::Call; use nu_protocol::{IntoPipelineData, PipelineData, ShellError, Span, Value}; use std::collections::HashMap; -pub type MathFunction = fn(values: &[Value], span: &Span) -> Result; - pub fn run_with_function( call: &Call, input: PipelineData, - mf: MathFunction, + mf: impl Fn(&[Value], &Span) -> Result, ) -> Result { let name = call.head; let res = calculate(input, name, mf); @@ -20,7 +18,7 @@ pub fn run_with_function( fn helper_for_tables( values: PipelineData, name: Span, - mf: MathFunction, + mf: impl Fn(&[Value], &Span) -> Result, ) -> Result { // If we are not dealing with Primitives, then perhaps we are dealing with a table // Create a key for each column name @@ -63,7 +61,11 @@ fn helper_for_tables( }) } -pub fn calculate(values: PipelineData, name: Span, mf: MathFunction) -> Result { +pub fn calculate( + values: PipelineData, + name: Span, + mf: impl Fn(&[Value], &Span) -> Result, +) -> Result { match values { PipelineData::Stream(_) => helper_for_tables(values, name, mf), PipelineData::Value(Value::List { ref vals, .. }) => match &vals[..] { diff --git a/crates/nu-command/src/math/variance.rs b/crates/nu-command/src/math/variance.rs new file mode 100644 index 000000000..a82968201 --- /dev/null +++ b/crates/nu-command/src/math/variance.rs @@ -0,0 +1,126 @@ +use crate::math::utils::run_with_function; +use nu_protocol::ast::Call; +use nu_protocol::engine::{Command, EngineState, Stack}; +use nu_protocol::{Example, PipelineData, ShellError, Signature, Span, Value}; + +#[derive(Clone)] +pub struct SubCommand; + +impl Command for SubCommand { + fn name(&self) -> &str { + "math variance" + } + + fn signature(&self) -> Signature { + Signature::build("math variance").switch("sample", "calculate sample variance", Some('s')) + } + + fn usage(&self) -> &str { + "Finds the variance of a list of numbers or tables" + } + + fn run( + &self, + _engine_state: &EngineState, + _stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let sample = call.has_flag("sample"); + run_with_function(call, input, compute_variance(sample)) + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Get the variance of a list of numbers", + example: "echo [1 2 3 4 5] | math variance", + result: Some(Value::Float { + val: 2.0, + span: Span::unknown(), + }), + }, + Example { + description: "Get the sample variance of a list of numbers", + example: "[1 2 3 4 5] | math variance -s", + result: Some(Value::Float { + val: 2.5, + span: Span::unknown(), + }), + }, + ] + } +} + +fn sum_of_squares(values: &[Value], span: &Span) -> Result { + let n = Value::Int { + val: values.len() as i64, + span: *span, + }; + let mut sum_x = Value::Int { + val: 0, + span: *span, + }; + let mut sum_x2 = Value::Int { + val: 0, + span: *span, + }; + for value in values { + let v = match &value { + Value::Int { .. } + | Value::Float { .. } => { + Ok(value) + }, + _ => Err(ShellError::UnsupportedInput( + "Attempted to compute the sum of squared values of a value that cannot be summed or squared.".to_string(), + value.span().unwrap_or_else(|_| Span::unknown()), + )) + }?; + let v_squared = &v.mul(*span, v)?; + sum_x2 = sum_x2.add(*span, v_squared)?; + sum_x = sum_x.add(*span, v)?; + } + + let sum_x_squared = sum_x.mul(*span, &sum_x)?; + let sum_x_squared_div_n = sum_x_squared.div(*span, &n)?; + + let ss = sum_x2.sub(*span, &sum_x_squared_div_n)?; + + Ok(ss) +} + +pub fn compute_variance(sample: bool) -> impl Fn(&[Value], &Span) -> Result { + move |values: &[Value], span: &Span| { + let n = if sample { + values.len() - 1 + } else { + values.len() + }; + let sum_of_squares = sum_of_squares(values, span); + let ss = match sum_of_squares { + Err(ShellError::UnsupportedInput(_, err_span)) => Err(ShellError::UnsupportedInput( + "Attempted to compute the variance with an item that cannot be used for that." + .to_string(), + err_span, + )), + other => other, + }?; + let n = Value::Int { + val: n as i64, + span: *span, + }; + ss.div(*span, &n) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(SubCommand {}) + } +}