From bdef5d7d72eab107346c94c156210d32b86b1437 Mon Sep 17 00:00:00 2001 From: bailey-layzer Date: Sat, 11 Jul 2020 20:57:39 -0700 Subject: [PATCH] Add 'str from' subcommand (#2125) * add human, precision commands * add 'str from' subcommand (converted from human/precision commands) move human tests to str from * add default locale, platform-specific SystemLocale use * fix platform specific num-format dependency, remove invalid test * change 'str from' localization to static num_format::Locale::en * minor cleanup, nudge ci * re-attempt ci --- Cargo.lock | 31 ++- crates/nu-cli/Cargo.toml | 7 + crates/nu-cli/src/cli.rs | 1 + crates/nu-cli/src/commands.rs | 4 +- crates/nu-cli/src/commands/str_/from.rs | 263 ++++++++++++++++++++++++ crates/nu-cli/src/commands/str_/mod.rs | 2 + crates/nu-cli/src/commands/str_/trim.rs | 28 +++ crates/nu-cli/tests/commands/str_.rs | 51 +++++ 8 files changed, 383 insertions(+), 4 deletions(-) create mode 100644 crates/nu-cli/src/commands/str_/from.rs diff --git a/Cargo.lock b/Cargo.lock index 61b85d246..2182a1daf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,6 +81,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +[[package]] +name = "arrayvec" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +dependencies = [ + "nodrop", +] + [[package]] name = "arrayvec" version = "0.5.1" @@ -303,7 +312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" dependencies = [ "arrayref", - "arrayvec", + "arrayvec 0.5.1", "constant_time_eq", ] @@ -1988,7 +1997,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616" dependencies = [ - "arrayvec", + "arrayvec 0.5.1", "bitflags", "cfg-if", "ryu", @@ -2392,6 +2401,12 @@ dependencies = [ "void", ] +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + [[package]] name = "nom" version = "1.2.4" @@ -2529,6 +2544,7 @@ dependencies = [ "nu-test-support", "nu-value-ext", "num-bigint", + "num-format", "num-traits 0.2.12", "parking_lot 0.11.0", "pin-utils", @@ -2867,6 +2883,17 @@ dependencies = [ "serde 1.0.114", ] +[[package]] +name = "num-format" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" +dependencies = [ + "arrayvec 0.4.12", + "itoa", + "num-bigint", +] + [[package]] name = "num-integer" version = "0.1.43" diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index f2b06ca9b..dab3e5c83 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -57,6 +57,7 @@ log = "0.4.8" meval = "0.2" natural = "0.5.0" num-bigint = {version = "0.2.6", features = ["serde"]} +num-format = {version = "0.4", features = ["with-num-bigint"]} num-traits = "0.2.11" parking_lot = "0.11.0" pin-utils = "0.1.0" @@ -98,6 +99,12 @@ trash = {version = "1.0.1", optional = true} [target.'cfg(unix)'.dependencies] users = "0.10.0" +# TODO this will be possible with new dependency resolver +# (currently on nightly behind -Zfeatures=itarget): +# https://github.com/rust-lang/cargo/issues/7914 +#[target.'cfg(not(windows))'.dependencies] +#num-format = {version = "0.4", features = ["with-system-locale"]} + [dependencies.rusqlite] features = ["bundled", "blob"] version = "0.23.1" diff --git a/crates/nu-cli/src/cli.rs b/crates/nu-cli/src/cli.rs index f67542107..1543e69ed 100644 --- a/crates/nu-cli/src/cli.rs +++ b/crates/nu-cli/src/cli.rs @@ -307,6 +307,7 @@ pub fn create_default_context( whole_stream_command(StrUpcase), whole_stream_command(StrCapitalize), whole_stream_command(StrFindReplace), + whole_stream_command(StrFrom), whole_stream_command(StrSubstring), whole_stream_command(StrSet), whole_stream_command(StrToDatetime), diff --git a/crates/nu-cli/src/commands.rs b/crates/nu-cli/src/commands.rs index b05796290..08cde2b61 100644 --- a/crates/nu-cli/src/commands.rs +++ b/crates/nu-cli/src/commands.rs @@ -251,8 +251,8 @@ pub(crate) use sort_by::SortBy; pub(crate) use split::{Split, SplitChars, SplitColumn, SplitRow}; pub(crate) use split_by::SplitBy; pub(crate) use str_::{ - Str, StrCapitalize, StrCollect, StrDowncase, StrFindReplace, StrLength, StrSet, StrSubstring, - StrToDatetime, StrToDecimal, StrToInteger, StrTrim, StrUpcase, + Str, StrCapitalize, StrCollect, StrDowncase, StrFindReplace, StrFrom, StrLength, StrSet, + StrSubstring, StrToDatetime, StrToDecimal, StrToInteger, StrTrim, StrUpcase, }; #[allow(unused_imports)] pub(crate) use t_sort_by::TSortBy; diff --git a/crates/nu-cli/src/commands/str_/from.rs b/crates/nu-cli/src/commands/str_/from.rs new file mode 100644 index 000000000..4344723b4 --- /dev/null +++ b/crates/nu-cli/src/commands/str_/from.rs @@ -0,0 +1,263 @@ +use crate::commands::str_::trim::trim_char; +use crate::commands::WholeStreamCommand; +use crate::prelude::*; +use nu_errors::ShellError; +use nu_protocol::{ + ColumnPath, Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value, +}; +use nu_source::Tagged; +use num_bigint::{BigInt, BigUint, ToBigInt}; + +// TODO num_format::SystemLocale once platform-specific dependencies are stable (see Cargo.toml) +use num_format::{Locale, ToFormattedString}; +use num_traits::{Pow, Signed}; +use std::iter; + +pub struct SubCommand; + +#[derive(Deserialize)] +struct Arguments { + rest: Vec, + decimals: Option>, + #[serde(rename(deserialize = "group-digits"))] + group_digits: bool, +} + +#[async_trait] +impl WholeStreamCommand for SubCommand { + fn name(&self) -> &str { + "str from" + } + + fn signature(&self) -> Signature { + Signature::build("str from") + .rest( + SyntaxShape::ColumnPath, + "optionally convert to string by column paths", + ) + .named( + "decimals", + SyntaxShape::Int, + "decimal digits to which to round", + Some('d'), + ) + .switch( + "group-digits", + // TODO according to system localization + "group digits, currently by thousand with commas", + Some('g'), + ) + } + + fn usage(&self) -> &str { + "Converts numeric types to strings. Trims trailing zeros unless decimals parameter is specified." + } + + async fn run( + &self, + args: CommandArgs, + registry: &CommandRegistry, + ) -> Result { + operate(args, registry).await + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "round to nearest integer", + example: "= 1.7 | str from -d 0", + result: Some(vec![UntaggedValue::string("2").into_untagged_value()]), + }, + Example { + description: "format large number with localized digit grouping", + example: "= 1000000.2 | str from -g", + result: Some(vec![ + UntaggedValue::string("1,000,000.2").into_untagged_value() + ]), + }, + ] + } +} + +async fn operate( + args: CommandArgs, + registry: &CommandRegistry, +) -> Result { + let ( + Arguments { + decimals, + group_digits, + rest: column_paths, + }, + input, + ) = args.process(®istry.clone()).await?; + let digits = decimals.map(|tagged| tagged.item); + + Ok(input + .map(move |v| { + if column_paths.is_empty() { + match action(&v, v.tag(), digits, group_digits) { + Ok(out) => ReturnSuccess::value(out), + Err(err) => Err(err), + } + } else { + let mut ret = v; + for path in &column_paths { + let swapping = ret.swap_data_by_column_path( + path, + Box::new(move |old| action(old, old.tag(), digits, group_digits)), + ); + match swapping { + Ok(new_value) => { + ret = new_value; + } + Err(err) => { + return Err(err); + } + } + } + + ReturnSuccess::value(ret) + } + }) + .to_output_stream()) +} + +// TODO If you're using the with-system-locale feature and you're on Windows, Clang 3.9 or higher is also required. +fn action( + input: &Value, + tag: impl Into, + digits: Option, + group_digits: bool, +) -> Result { + match &input.value { + UntaggedValue::Primitive(prim) => Ok(UntaggedValue::string(match prim { + Primitive::Int(int) => { + if group_digits { + format_bigint(int) // int.to_formatted_string(*locale) + } else { + int.to_string() + } + } + Primitive::Decimal(dec) => format_decimal(dec.clone(), digits, group_digits), + _ => { + return Err(ShellError::unimplemented( + "str from for non-numeric primitives", + )) + } + }) + .into_value(tag)), + UntaggedValue::Row(_) => Err(ShellError::labeled_error( + "specify column to use 'str from'", + "found table", + input.tag.clone(), + )), + _ => Err(ShellError::unimplemented( + "str from for non-primitive, non-table types", + )), + } +} + +fn format_bigint(int: &BigInt) -> String { + int.to_formatted_string(&Locale::en) + + // TODO once platform-specific dependencies are stable (see Cargo.toml) + // #[cfg(windows)] + // { + // int.to_formatted_string(&Locale::en) + // } + // #[cfg(not(windows))] + // { + // match SystemLocale::default() { + // Ok(locale) => int.to_formatted_string(&locale), + // Err(_) => int.to_formatted_string(&Locale::en), + // } + // } +} + +fn format_decimal(mut decimal: BigDecimal, digits: Option, group_digits: bool) -> String { + if let Some(n) = digits { + decimal = round_decimal(&decimal, n) + } + + if decimal.is_integer() && (digits.is_none() || digits == Some(0)) { + let int = decimal + .to_bigint() + .expect("integer BigDecimal should convert to BigInt"); + return if group_digits { + int.to_string() + } else { + format_bigint(&int) + }; + } + + let (int, exp) = decimal.as_bigint_and_exponent(); + let factor = BigInt::from(10).pow(BigUint::from(exp as u64)); // exp > 0 for non-int decimal + let int_part = &int / &factor; + let dec_part = (&int % &factor) + .abs() + .to_biguint() + .expect("BigInt::abs should always produce positive signed BigInt and thus BigUInt") + .to_str_radix(10); + + let dec_str = if let Some(n) = digits { + dec_part + .chars() + .chain(iter::repeat('0')) + .take(n as usize) + .collect() + } else { + trim_char(dec_part, '0', false, true) + }; + + let format_default_loc = |int_part: BigInt| { + let loc = Locale::en; + let (int_str, sep) = ( + int_part.to_formatted_string(&loc), + String::from(loc.decimal()), + ); + + format!("{}{}{}", int_str, sep, dec_str) + }; + + format_default_loc(int_part) + + // TODO once platform-specific dependencies are stable (see Cargo.toml) + // #[cfg(windows)] + // { + // format_default_loc(int_part) + // } + // #[cfg(not(windows))] + // { + // match SystemLocale::default() { + // Ok(sys_loc) => { + // let int_str = int_part.to_formatted_string(&sys_loc); + // let sep = String::from(sys_loc.decimal()); + // format!("{}{}{}", int_str, sep, dec_str) + // } + // Err(_) => format_default_loc(int_part), + // } + // } +} + +fn round_decimal(decimal: &BigDecimal, mut digits: u64) -> BigDecimal { + let mut mag = decimal.clone(); + while mag >= BigDecimal::from(1) { + mag = mag / 10; + digits += 1; + } + + decimal.with_prec(digits) +} + +#[cfg(test)] +mod tests { + use super::SubCommand; + + #[test] + fn examples_work_as_expected() { + use crate::examples::test as test_examples; + + test_examples(SubCommand {}) + } +} diff --git a/crates/nu-cli/src/commands/str_/mod.rs b/crates/nu-cli/src/commands/str_/mod.rs index 20ac62184..794ff4977 100644 --- a/crates/nu-cli/src/commands/str_/mod.rs +++ b/crates/nu-cli/src/commands/str_/mod.rs @@ -3,6 +3,7 @@ mod collect; mod command; mod downcase; mod find_replace; +mod from; mod length; mod set; mod substring; @@ -17,6 +18,7 @@ pub use collect::SubCommand as StrCollect; pub use command::Command as Str; pub use downcase::SubCommand as StrDowncase; pub use find_replace::SubCommand as StrFindReplace; +pub use from::SubCommand as StrFrom; pub use length::SubCommand as StrLength; pub use set::SubCommand as StrSet; pub use substring::SubCommand as StrSubstring; diff --git a/crates/nu-cli/src/commands/str_/trim.rs b/crates/nu-cli/src/commands/str_/trim.rs index 3d5cf5b2d..d96f4daa8 100644 --- a/crates/nu-cli/src/commands/str_/trim.rs +++ b/crates/nu-cli/src/commands/str_/trim.rs @@ -108,6 +108,34 @@ fn action(input: &Value, tag: impl Into) -> Result { } } +// TODO make callable using flag +pub fn trim_char(s: String, to_trim: char, leading: bool, trailing: bool) -> String { + let mut trimmed = String::from(""); + let mut backlog = String::from(""); + let mut at_left = true; + s.chars().for_each(|ch| match ch { + c if c == to_trim => { + if !(leading && at_left) { + if trailing { + backlog.push(c) + } else { + trimmed.push(c) + } + } + } + other => { + at_left = false; + if trailing { + trimmed.push_str(backlog.as_str()); + backlog = String::from(""); + } + trimmed.push(other); + } + }); + + trimmed +} + #[cfg(test)] mod tests { use super::{action, SubCommand}; diff --git a/crates/nu-cli/tests/commands/str_.rs b/crates/nu-cli/tests/commands/str_.rs index 3f2f574ac..61d18ffbf 100644 --- a/crates/nu-cli/tests/commands/str_.rs +++ b/crates/nu-cli/tests/commands/str_.rs @@ -340,3 +340,54 @@ fn substrings_the_input_and_treats_end_index_as_length_if_blank_end_index_given( assert_eq!(actual.out, "arepas"); }) } + +#[test] +fn from_decimal_correct_trailing_zeros() { + let actual = nu!( + cwd: ".", pipeline( + r#" + = 1.23000 | str from -d 3 + "# + )); + + assert!(actual.out.contains("1.230")); +} + +#[test] +fn from_int_decimal_correct_trailing_zeros() { + let actual = nu!( + cwd: ".", pipeline( + r#" + = 1.00000 | str from -d 3 + "# + )); + + assert!(actual.out.contains("1.000")); +} + +#[test] +fn from_int_decimal_trim_trailing_zeros() { + let actual = nu!( + cwd: ".", pipeline( + r#" + = 1.00000 | str from | format "{$it} flat" + "# + )); + + assert!(actual.out.contains("1 flat")); // "1" would match "1.0" +} + +#[test] +fn from_table() { + let actual = nu!( + cwd: ".", pipeline( + r#" + echo '[{"name": "foo", "weight": 32.377}, {"name": "bar", "weight": 15.2}]' + | from json + | str from weight -d 2 + "# + )); + + assert!(actual.out.contains("32.38")); + assert!(actual.out.contains("15.20")); +}