diff --git a/Cargo.lock b/Cargo.lock index 54d80b1449..3e818b221d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1153,6 +1153,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "data-encoding" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + [[package]] name = "deranged" version = "0.3.11" @@ -3066,6 +3072,7 @@ dependencies = [ "chrono-tz 0.8.6", "crossterm", "csv", + "data-encoding", "deunicode", "dialoguer", "digest", @@ -3120,6 +3127,7 @@ dependencies = [ "quickcheck", "quickcheck_macros", "rand", + "rand_chacha", "rayon", "regex", "rmp", diff --git a/Cargo.toml b/Cargo.toml index aa3eb60e45..ebb26171c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,6 +136,7 @@ quickcheck = "1.0" quickcheck_macros = "1.0" quote = "1.0" rand = "0.8" +rand_chacha = "0.3.1" ratatui = "0.26" rayon = "1.10" reedline = "0.34.0" diff --git a/crates/nu-cmd-extra/src/extra/mod.rs b/crates/nu-cmd-extra/src/extra/mod.rs index 1877039a6f..3936b94a0f 100644 --- a/crates/nu-cmd-extra/src/extra/mod.rs +++ b/crates/nu-cmd-extra/src/extra/mod.rs @@ -45,8 +45,6 @@ pub fn add_extra_command_context(mut engine_state: EngineState) -> EngineState { bind_command!( strings::format::FormatPattern, - strings::encode_decode::EncodeHex, - strings::encode_decode::DecodeHex, strings::str_::case::Str, strings::str_::case::StrCamelCase, strings::str_::case::StrKebabCase, diff --git a/crates/nu-cmd-extra/src/extra/strings/encode_decode/hex.rs b/crates/nu-cmd-extra/src/extra/strings/encode_decode/hex.rs deleted file mode 100644 index be681b382a..0000000000 --- a/crates/nu-cmd-extra/src/extra/strings/encode_decode/hex.rs +++ /dev/null @@ -1,192 +0,0 @@ -use nu_cmd_base::input_handler::{operate as general_operate, CmdArgument}; -use nu_engine::command_prelude::*; - -enum HexDecodingError { - InvalidLength(usize), - InvalidDigit(usize, char), -} - -fn hex_decode(value: &str) -> Result, HexDecodingError> { - let mut digits = value - .chars() - .enumerate() - .filter(|(_, c)| !c.is_whitespace()); - - let mut res = Vec::with_capacity(value.len() / 2); - loop { - let c1 = match digits.next() { - Some((ind, c)) => match c.to_digit(16) { - Some(d) => d, - None => return Err(HexDecodingError::InvalidDigit(ind, c)), - }, - None => return Ok(res), - }; - let c2 = match digits.next() { - Some((ind, c)) => match c.to_digit(16) { - Some(d) => d, - None => return Err(HexDecodingError::InvalidDigit(ind, c)), - }, - None => { - return Err(HexDecodingError::InvalidLength(value.len())); - } - }; - res.push((c1 << 4 | c2) as u8); - } -} - -fn hex_digit(num: u8) -> char { - match num { - 0..=9 => (num + b'0') as char, - 10..=15 => (num - 10 + b'A') as char, - _ => unreachable!(), - } -} - -fn hex_encode(bytes: &[u8]) -> String { - let mut res = String::with_capacity(bytes.len() * 2); - for byte in bytes { - res.push(hex_digit(byte >> 4)); - res.push(hex_digit(byte & 0b1111)); - } - res -} - -#[derive(Clone)] -pub struct HexConfig { - pub action_type: ActionType, -} - -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ActionType { - Encode, - Decode, -} - -struct Arguments { - cell_paths: Option>, - encoding_config: HexConfig, -} - -impl CmdArgument for Arguments { - fn take_cell_paths(&mut self) -> Option> { - self.cell_paths.take() - } -} - -pub fn operate( - action_type: ActionType, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let cell_paths: Vec = call.rest(engine_state, stack, 0)?; - let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - - let args = Arguments { - encoding_config: HexConfig { action_type }, - cell_paths, - }; - - general_operate(action, args, input, call.head, engine_state.signals()) -} - -fn action( - input: &Value, - // only used for `decode` action - args: &Arguments, - command_span: Span, -) -> Value { - let hex_config = &args.encoding_config; - - match input { - // Propagate existing errors. - Value::Error { .. } => input.clone(), - Value::Binary { val, .. } => match hex_config.action_type { - ActionType::Encode => Value::string(hex_encode(val.as_ref()), command_span), - ActionType::Decode => Value::error( - ShellError::UnsupportedInput { msg: "Binary data can only be encoded".to_string(), input: "value originates from here".into(), msg_span: command_span, input_span: input.span() }, - command_span, - ), - }, - Value::String { val, .. } => { - match hex_config.action_type { - ActionType::Encode => Value::error( - ShellError::UnsupportedInput { msg: "String value can only be decoded".to_string(), input: "value originates from here".into(), msg_span: command_span, input_span: input.span() }, - command_span, - ), - - ActionType::Decode => match hex_decode(val.as_ref()) { - Ok(decoded_value) => Value::binary(decoded_value, command_span), - Err(HexDecodingError::InvalidLength(len)) => Value::error(ShellError::GenericError { - error: "value could not be hex decoded".into(), - msg: format!("invalid hex input length: {len}. The length should be even"), - span: Some(command_span), - help: None, - inner: vec![], - }, - command_span, - ), - Err(HexDecodingError::InvalidDigit(index, digit)) => Value::error(ShellError::GenericError { - error: "value could not be hex decoded".into(), - msg: format!("invalid hex digit: '{digit}' at index {index}. Only 0-9, A-F, a-f are allowed in hex encoding"), - span: Some(command_span), - help: None, - inner: vec![], - }, - command_span, - ), - }, - } - } - other => Value::error( - ShellError::TypeMismatch { - err_message: format!("string or binary, not {}", other.get_type()), - span: other.span(), - }, - other.span(), - ), - } -} - -#[cfg(test)] -mod tests { - use super::{action, ActionType, Arguments, HexConfig}; - use nu_protocol::{Span, Value}; - - #[test] - fn hex_encode() { - let word = Value::binary([77, 97, 110], Span::test_data()); - let expected = Value::test_string("4D616E"); - - let actual = action( - &word, - &Arguments { - encoding_config: HexConfig { - action_type: ActionType::Encode, - }, - cell_paths: None, - }, - Span::test_data(), - ); - assert_eq!(actual, expected); - } - - #[test] - fn hex_decode() { - let word = Value::test_string("4D 61\r\n\n6E"); - let expected = Value::binary([77, 97, 110], Span::test_data()); - - let actual = action( - &word, - &Arguments { - encoding_config: HexConfig { - action_type: ActionType::Decode, - }, - cell_paths: None, - }, - Span::test_data(), - ); - assert_eq!(actual, expected); - } -} diff --git a/crates/nu-cmd-extra/src/extra/strings/encode_decode/mod.rs b/crates/nu-cmd-extra/src/extra/strings/encode_decode/mod.rs deleted file mode 100644 index 581b87679f..0000000000 --- a/crates/nu-cmd-extra/src/extra/strings/encode_decode/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -mod decode_hex; -mod encode_hex; -mod hex; - -pub(crate) use decode_hex::DecodeHex; -pub(crate) use encode_hex::EncodeHex; diff --git a/crates/nu-cmd-extra/src/extra/strings/mod.rs b/crates/nu-cmd-extra/src/extra/strings/mod.rs index 391cf108e0..8d43cd8a65 100644 --- a/crates/nu-cmd-extra/src/extra/strings/mod.rs +++ b/crates/nu-cmd-extra/src/extra/strings/mod.rs @@ -1,3 +1,2 @@ -pub(crate) mod encode_decode; pub(crate) mod format; pub(crate) mod str_; diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index f905f804fd..3c455eabcf 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -102,6 +102,7 @@ v_htmlescape = { workspace = true } wax = { workspace = true } which = { workspace = true } unicode-width = { workspace = true } +data-encoding = { version = "2.6.0", features = ["alloc"] } [target.'cfg(windows)'.dependencies] winreg = { workspace = true } @@ -146,4 +147,5 @@ quickcheck = { workspace = true } quickcheck_macros = { workspace = true } rstest = { workspace = true, default-features = false } pretty_assertions = { workspace = true } -tempfile = { workspace = true } \ No newline at end of file +tempfile = { workspace = true } +rand_chacha = { workspace = true } diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index 6ef188d571..88b229fc48 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -179,8 +179,16 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState { Char, Decode, Encode, + DecodeHex, + EncodeHex, + DecodeBase32, + EncodeBase32, + DecodeBase32Hex, + EncodeBase32Hex, DecodeBase64, EncodeBase64, + DecodeBase64Old, + EncodeBase64Old, DetectColumns, Parse, Split, diff --git a/crates/nu-command/src/strings/base/base32.rs b/crates/nu-command/src/strings/base/base32.rs new file mode 100644 index 0000000000..995cbb6294 --- /dev/null +++ b/crates/nu-command/src/strings/base/base32.rs @@ -0,0 +1,180 @@ +use data_encoding::Encoding; + +use nu_engine::command_prelude::*; + +const EXTRA_USAGE: &str = r"The default alphabet is taken from RFC 4648, section 6. + +Note this command will collect stream input."; + +#[derive(Clone)] +pub struct DecodeBase32; + +impl Command for DecodeBase32 { + fn name(&self) -> &str { + "decode base32" + } + + fn signature(&self) -> Signature { + Signature::build("decode base32") + .input_output_types(vec![(Type::String, Type::Binary)]) + .allow_variants_without_examples(true) + .switch("nopad", "Do not pad the output.", None) + .category(Category::Formats) + } + + fn description(&self) -> &str { + "Decode a Base32 value." + } + + fn extra_description(&self) -> &str { + EXTRA_USAGE + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Decode arbitrary binary data", + example: r#""AEBAGBAF" | decode base32"#, + result: Some(Value::test_binary(vec![1, 2, 3, 4, 5])), + }, + Example { + description: "Decode an encoded string", + example: r#""NBUQ====" | decode base32 | decode"#, + result: None, + }, + Example { + description: "Parse a string without padding", + example: r#""NBUQ" | decode base32 --nopad"#, + result: Some(Value::test_binary(vec![0x68, 0x69])), + }, + ] + } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = if call.has_flag(engine_state, stack, "nopad")? { + data_encoding::BASE32_NOPAD + } else { + data_encoding::BASE32 + }; + super::decode(encoding, call.span(), input) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = if call.has_flag_const(working_set, "nopad")? { + data_encoding::BASE32_NOPAD + } else { + data_encoding::BASE32 + }; + super::decode(encoding, call.span(), input) + } +} + +#[derive(Clone)] +pub struct EncodeBase32; + +impl Command for EncodeBase32 { + fn name(&self) -> &str { + "encode base32" + } + + fn signature(&self) -> Signature { + Signature::build("encode base32") + .input_output_types(vec![ + (Type::String, Type::String), + (Type::Binary, Type::String), + ]) + .switch("nopad", "Don't accept padding.", None) + .category(Category::Formats) + } + + fn description(&self) -> &str { + "Encode a string or binary value using Base32." + } + + fn extra_description(&self) -> &str { + EXTRA_USAGE + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Encode a binary value", + example: r#"0x[01 02 10] | encode base32"#, + result: Some(Value::test_string("AEBBA===")), + }, + Example { + description: "Encode a string", + example: r#""hello there" | encode base32"#, + result: Some(Value::test_string("NBSWY3DPEB2GQZLSMU======")), + }, + Example { + description: "Don't apply padding to the output", + example: r#""hi" | encode base32 --nopad"#, + result: Some(Value::test_string("NBUQ")), + }, + ] + } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = if call.has_flag(engine_state, stack, "nopad")? { + data_encoding::BASE32_NOPAD + } else { + data_encoding::BASE32 + }; + super::encode(encoding, call.span(), input) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = if call.has_flag_const(working_set, "nopad")? { + data_encoding::BASE32_NOPAD + } else { + data_encoding::BASE32 + }; + super::encode(encoding, call.span(), input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_examples_decode() { + crate::test_examples(DecodeBase32) + } + + #[test] + fn test_examples_encode() { + crate::test_examples(EncodeBase32) + } +} diff --git a/crates/nu-command/src/strings/base/base32hex.rs b/crates/nu-command/src/strings/base/base32hex.rs new file mode 100644 index 0000000000..044a23415f --- /dev/null +++ b/crates/nu-command/src/strings/base/base32hex.rs @@ -0,0 +1,181 @@ +use nu_engine::command_prelude::*; + +const EXTRA_USAGE: &str = r"This command uses an alternative Base32 alphabet, defined in RFC 4648, section 7. + +Note this command will collect stream input."; + +#[derive(Clone)] +pub struct DecodeBase32Hex; + +impl Command for DecodeBase32Hex { + fn name(&self) -> &str { + "decode base32hex" + } + + fn signature(&self) -> Signature { + Signature::build("decode base32hex") + .input_output_types(vec![(Type::String, Type::Binary)]) + .allow_variants_without_examples(true) + .switch("nopad", "Reject input with padding.", None) + .category(Category::Formats) + } + + fn description(&self) -> &str { + "Encode a base32hex value." + } + + fn extra_description(&self) -> &str { + EXTRA_USAGE + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Decode arbitrary binary data", + example: r#""ATNAQ===" | decode base32hex"#, + result: Some(Value::test_binary(vec![0x57, 0x6E, 0xAD])), + }, + Example { + description: "Decode an encoded string", + example: r#""D1KG====" | decode base32hex | decode"#, + result: None, + }, + Example { + description: "Parse a string without padding", + example: r#""ATNAQ" | decode base32hex --nopad"#, + result: Some(Value::test_binary(vec![0x57, 0x6E, 0xAD])), + }, + ] + } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = if call.has_flag(engine_state, stack, "nopad")? { + data_encoding::BASE32HEX_NOPAD + } else { + data_encoding::BASE32HEX + }; + + super::decode(encoding, call.head, input) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = if call.has_flag_const(working_set, "nopad")? { + data_encoding::BASE32HEX_NOPAD + } else { + data_encoding::BASE32HEX + }; + + super::decode(encoding, call.head, input) + } +} + +#[derive(Clone)] +pub struct EncodeBase32Hex; + +impl Command for EncodeBase32Hex { + fn name(&self) -> &str { + "encode base32hex" + } + + fn signature(&self) -> Signature { + Signature::build("encode base32hex") + .input_output_types(vec![ + (Type::String, Type::String), + (Type::Binary, Type::String), + ]) + .switch("nopad", "Don't pad the output.", None) + .category(Category::Formats) + } + + fn description(&self) -> &str { + "Encode a binary value or a string using base32hex." + } + + fn extra_description(&self) -> &str { + EXTRA_USAGE + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Encode a binary value", + example: r#"0x[57 6E AD] | encode base32hex"#, + result: Some(Value::test_string("ATNAQ===")), + }, + Example { + description: "Encode a string", + example: r#""hello there" | encode base32hex"#, + result: Some(Value::test_string("D1IMOR3F41Q6GPBICK======")), + }, + Example { + description: "Don't apply padding to the output", + example: r#""hello there" | encode base32hex --nopad"#, + result: Some(Value::test_string("D1IMOR3F41Q6GPBICK")), + }, + ] + } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = if call.has_flag(engine_state, stack, "nopad")? { + data_encoding::BASE32HEX_NOPAD + } else { + data_encoding::BASE32HEX + }; + + super::encode(encoding, call.head, input) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = if call.has_flag_const(working_set, "nopad")? { + data_encoding::BASE32HEX_NOPAD + } else { + data_encoding::BASE32HEX + }; + + super::encode(encoding, call.head, input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_examples_decode() { + crate::test_examples(DecodeBase32Hex) + } + #[test] + fn test_examples_encode() { + crate::test_examples(EncodeBase32Hex) + } +} diff --git a/crates/nu-command/src/strings/base/base64.rs b/crates/nu-command/src/strings/base/base64.rs new file mode 100644 index 0000000000..3a1ccad9ef --- /dev/null +++ b/crates/nu-command/src/strings/base/base64.rs @@ -0,0 +1,193 @@ +use data_encoding::Encoding; + +use nu_engine::command_prelude::*; + +const EXTRA_USAGE: &str = r"The default alphabet is taken from RFC 4648, section 4. A URL-safe version is available. + +Note this command will collect stream input."; + +fn get_encoding_from_flags(url: bool, nopad: bool) -> Encoding { + match (url, nopad) { + (false, false) => data_encoding::BASE64, + (false, true) => data_encoding::BASE64_NOPAD, + (true, false) => data_encoding::BASE64URL, + (true, true) => data_encoding::BASE64URL_NOPAD, + } +} + +fn get_encoding( + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, +) -> Result { + let url = call.has_flag(engine_state, stack, "url")?; + let nopad = call.has_flag(engine_state, stack, "nopad")?; + + Ok(get_encoding_from_flags(url, nopad)) +} + +fn get_encoding_const(working_set: &StateWorkingSet, call: &Call) -> Result { + let url = call.has_flag_const(working_set, "url")?; + let nopad = call.has_flag_const(working_set, "nopad")?; + + Ok(get_encoding_from_flags(url, nopad)) +} + +#[derive(Clone)] +pub struct DecodeBase64; + +impl Command for DecodeBase64 { + fn name(&self) -> &str { + "decode new-base64" + } + + fn signature(&self) -> Signature { + Signature::build("decode new-base64") + .input_output_types(vec![(Type::String, Type::Binary)]) + .allow_variants_without_examples(true) + .switch("url", "Decode the URL-safe Base64 version.", None) + .switch("nopad", "Reject padding.", None) + .category(Category::Formats) + } + + fn description(&self) -> &str { + "Decode a Base64 value." + } + + fn extra_description(&self) -> &str { + EXTRA_USAGE + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Decode a Base64 string", + example: r#""U29tZSBEYXRh" | decode new-base64 | decode"#, + result: None, + }, + Example { + description: "Decode arbitrary data", + example: r#""/w==" | decode new-base64"#, + result: Some(Value::test_binary(vec![0xFF])), + }, + Example { + description: "Decode a URL-safe Base64 string", + example: r#""_w==" | decode new-base64 --url"#, + result: Some(Value::test_binary(vec![0xFF])), + }, + ] + } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = get_encoding(engine_state, stack, call)?; + super::decode(encoding, call.head, input) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = get_encoding_const(working_set, call)?; + super::decode(encoding, call.head, input) + } +} + +#[derive(Clone)] +pub struct EncodeBase64; + +impl Command for EncodeBase64 { + fn name(&self) -> &str { + "encode new-base64" + } + + fn signature(&self) -> Signature { + Signature::build("encode new-base64") + .input_output_types(vec![ + (Type::String, Type::String), + (Type::Binary, Type::String), + ]) + .switch("url", "Use the URL-safe Base64 version.", None) + .switch("nopad", "Don't pad the output.", None) + .category(Category::Formats) + } + + fn description(&self) -> &str { + "Encode a string or binary value using Base64." + } + + fn extra_description(&self) -> &str { + EXTRA_USAGE + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Encode a string with Base64", + example: r#""Alphabet from A to Z" | encode new-base64"#, + result: Some(Value::test_string("QWxwaGFiZXQgZnJvbSBBIHRvIFo=")), + }, + Example { + description: "Encode arbitrary data", + example: r#"0x[BE EE FF] | encode new-base64"#, + result: Some(Value::test_string("vu7/")), + }, + Example { + description: "Use a URL-safe alphabet", + example: r#"0x[BE EE FF] | encode new-base64 --url"#, + result: Some(Value::test_string("vu7_")), + }, + ] + } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = get_encoding(engine_state, stack, call)?; + super::encode(encoding, call.head, input) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = get_encoding_const(working_set, call)?; + super::encode(encoding, call.head, input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_examples_decode() { + crate::test_examples(DecodeBase64) + } + + #[test] + fn test_examples_encode() { + crate::test_examples(EncodeBase64) + } +} diff --git a/crates/nu-command/src/strings/base/hex.rs b/crates/nu-command/src/strings/base/hex.rs new file mode 100644 index 0000000000..189b22742e --- /dev/null +++ b/crates/nu-command/src/strings/base/hex.rs @@ -0,0 +1,151 @@ +use nu_engine::command_prelude::*; + +#[derive(Clone)] +pub struct DecodeHex; + +impl Command for DecodeHex { + fn name(&self) -> &str { + "decode hex" + } + + fn signature(&self) -> Signature { + Signature::build("decode hex") + .input_output_types(vec![(Type::String, Type::Binary)]) + .category(Category::Formats) + } + + fn description(&self) -> &str { + "Hex decode a value." + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Decode arbitrary binary data", + example: r#""09FD" | decode hex"#, + result: Some(Value::test_binary(vec![0x09, 0xFD])), + }, + Example { + description: "Lowercase Hex is also accepted", + example: r#""09fd" | decode hex"#, + result: Some(Value::test_binary(vec![0x09, 0xFD])), + }, + ] + } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + super::decode(data_encoding::HEXLOWER_PERMISSIVE, call.head, input) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + todo!() + } +} + +#[derive(Clone)] +pub struct EncodeHex; + +impl Command for EncodeHex { + fn name(&self) -> &str { + "encode hex" + } + + fn signature(&self) -> Signature { + Signature::build("encode hex") + .input_output_types(vec![ + (Type::String, Type::String), + (Type::Binary, Type::String), + ]) + .switch("lower", "Encode to lowercase hex.", None) + .category(Category::Formats) + } + + fn description(&self) -> &str { + "Hex encode a binary value or a string." + } + + fn examples(&self) -> Vec { + vec![ + Example { + description: "Encode a binary value", + example: r#"0x[C3 06] | encode hex"#, + result: Some(Value::test_string("C306")), + }, + Example { + description: "Encode a string", + example: r#""hello" | encode hex"#, + result: Some(Value::test_string("68656C6C6F")), + }, + Example { + description: "Output a Lowercase version of the encoding", + example: r#"0x[AD EF] | encode hex --lower"#, + result: Some(Value::test_string("adef")), + }, + ] + } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = if call.has_flag(engine_state, stack, "lower")? { + data_encoding::HEXLOWER + } else { + data_encoding::HEXUPPER + }; + + super::encode(encoding, call.head, input) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding = if call.has_flag_const(working_set, "lower")? { + data_encoding::HEXLOWER + } else { + data_encoding::HEXUPPER + }; + + super::encode(encoding, call.head, input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_examples_decode() { + crate::test_examples(DecodeHex) + } + + #[test] + fn test_examples_encode() { + crate::test_examples(EncodeHex) + } +} diff --git a/crates/nu-command/src/strings/base/mod.rs b/crates/nu-command/src/strings/base/mod.rs new file mode 100644 index 0000000000..d7e248a032 --- /dev/null +++ b/crates/nu-command/src/strings/base/mod.rs @@ -0,0 +1,99 @@ +#![allow(unused)] + +use data_encoding::Encoding; + +use nu_engine::command_prelude::*; + +mod base32; +mod base32hex; +mod base64; +mod hex; + +pub use base32::{DecodeBase32, EncodeBase32}; +pub use base32hex::{DecodeBase32Hex, EncodeBase32Hex}; +pub use base64::{DecodeBase64, EncodeBase64}; +pub use hex::{DecodeHex, EncodeHex}; + +pub fn decode( + encoding: Encoding, + call_span: Span, + input: PipelineData, +) -> Result { + let metadata = input.metadata(); + let (input_str, input_span) = get_string(input, call_span)?; + let output = match encoding.decode(input_str.as_bytes()) { + Ok(output) => output, + Err(err) => { + return Err(ShellError::IncorrectValue { + msg: err.to_string(), + val_span: input_span, + call_span, + }); + } + }; + + Ok(Value::binary(output, call_span).into_pipeline_data_with_metadata(metadata)) +} + +pub fn encode( + encoding: Encoding, + call_span: Span, + input: PipelineData, +) -> Result { + let metadata = input.metadata(); + let (input_bytes, _) = get_binary(input, call_span)?; + let output = encoding.encode(&input_bytes); + + Ok(Value::string(output, call_span).into_pipeline_data_with_metadata(metadata)) +} + +fn get_string(input: PipelineData, call_span: Span) -> Result<(String, Span), ShellError> { + match input { + PipelineData::Value(val, ..) => { + let span = val.span(); + match val { + Value::String { val, .. } => Ok((val, span)), + + _ => { + todo!("Invalid type") + } + } + } + PipelineData::ListStream(..) => { + todo!() + } + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + Ok((stream.into_string()?, span)) + } + PipelineData::Empty => Err(ShellError::PipelineEmpty { + dst_span: call_span, + }), + } +} + +fn get_binary(input: PipelineData, call_span: Span) -> Result<(Vec, Span), ShellError> { + match input { + PipelineData::Value(val, ..) => { + let span = val.span(); + match val { + Value::Binary { val, .. } => Ok((val, span)), + Value::String { val, .. } => Ok((val.into_bytes(), span)), + + _ => { + todo!("Invalid type") + } + } + } + PipelineData::ListStream(..) => { + todo!() + } + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + Ok((stream.into_bytes()?, span)) + } + PipelineData::Empty => { + todo!("Can't have empty data"); + } + } +} diff --git a/crates/nu-command/src/strings/encode_decode/decode_base64.rs b/crates/nu-command/src/strings/encode_decode/decode_base64.rs index 083c1c8448..455aec915c 100644 --- a/crates/nu-command/src/strings/encode_decode/decode_base64.rs +++ b/crates/nu-command/src/strings/encode_decode/decode_base64.rs @@ -1,10 +1,11 @@ use super::base64::{operate, ActionType, Base64CommandArguments, CHARACTER_SET_DESC}; use nu_engine::command_prelude::*; +use nu_protocol::{report_warning_new, ParseWarning}; #[derive(Clone)] -pub struct DecodeBase64; +pub struct DecodeBase64Old; -impl Command for DecodeBase64 { +impl Command for DecodeBase64Old { fn name(&self) -> &str { "decode base64" } @@ -77,6 +78,16 @@ impl Command for DecodeBase64 { call: &Call, input: PipelineData, ) -> Result { + report_warning_new( + engine_state, + &ParseWarning::DeprecatedWarning { + old_command: "decode base64".into(), + new_suggestion: "the new `decode new-base64` version".into(), + span: call.head, + url: "`help decode new-base64`".into(), + }, + ); + let character_set: Option> = call.get_flag(engine_state, stack, "character-set")?; let binary = call.has_flag(engine_state, stack, "binary")?; @@ -114,6 +125,6 @@ mod tests { #[test] fn test_examples() { - crate::test_examples(DecodeBase64) + crate::test_examples(DecodeBase64Old) } } diff --git a/crates/nu-command/src/strings/encode_decode/encode_base64.rs b/crates/nu-command/src/strings/encode_decode/encode_base64.rs index 08a9cb5e51..a484eeb974 100644 --- a/crates/nu-command/src/strings/encode_decode/encode_base64.rs +++ b/crates/nu-command/src/strings/encode_decode/encode_base64.rs @@ -1,10 +1,11 @@ use super::base64::{operate, ActionType, Base64CommandArguments, CHARACTER_SET_DESC}; use nu_engine::command_prelude::*; +use nu_protocol::{report_warning_new, ParseWarning}; #[derive(Clone)] -pub struct EncodeBase64; +pub struct EncodeBase64Old; -impl Command for EncodeBase64 { +impl Command for EncodeBase64Old { fn name(&self) -> &str { "encode base64" } @@ -81,6 +82,16 @@ impl Command for EncodeBase64 { call: &Call, input: PipelineData, ) -> Result { + report_warning_new( + engine_state, + &ParseWarning::DeprecatedWarning { + old_command: "encode base64".into(), + new_suggestion: "the new `encode new-base64` version".into(), + span: call.head, + url: "`help encode new-base64`".into(), + }, + ); + let character_set: Option> = call.get_flag(engine_state, stack, "character-set")?; let binary = call.has_flag(engine_state, stack, "binary")?; @@ -118,6 +129,6 @@ mod tests { #[test] fn test_examples() { - crate::test_examples(EncodeBase64) + crate::test_examples(EncodeBase64Old) } } diff --git a/crates/nu-command/src/strings/encode_decode/mod.rs b/crates/nu-command/src/strings/encode_decode/mod.rs index e5d0ea06a6..5dde14e0f2 100644 --- a/crates/nu-command/src/strings/encode_decode/mod.rs +++ b/crates/nu-command/src/strings/encode_decode/mod.rs @@ -6,6 +6,6 @@ mod encode_base64; mod encoding; pub use self::decode::Decode; -pub use self::decode_base64::DecodeBase64; +pub use self::decode_base64::DecodeBase64Old; pub use self::encode::Encode; -pub use self::encode_base64::EncodeBase64; +pub use self::encode_base64::EncodeBase64Old; diff --git a/crates/nu-command/src/strings/mod.rs b/crates/nu-command/src/strings/mod.rs index 8b5af2dec4..08f50e537d 100644 --- a/crates/nu-command/src/strings/mod.rs +++ b/crates/nu-command/src/strings/mod.rs @@ -1,3 +1,4 @@ +mod base; mod char_; mod detect_columns; mod encode_decode; @@ -7,6 +8,10 @@ mod parse; mod split; mod str_; +pub use base::{ + DecodeBase32, DecodeBase32Hex, DecodeBase64, DecodeHex, EncodeBase32, EncodeBase32Hex, + EncodeBase64, EncodeHex, +}; pub use char_::Char; pub use detect_columns::*; pub use encode_decode::*; diff --git a/crates/nu-command/tests/commands/base/base32.rs b/crates/nu-command/tests/commands/base/base32.rs new file mode 100644 index 0000000000..e6fae31808 --- /dev/null +++ b/crates/nu-command/tests/commands/base/base32.rs @@ -0,0 +1,58 @@ +use nu_test_support::nu; + +#[test] +fn canonical() { + for value in super::random_bytes() { + let outcome = nu!("{} | encode base32 | decode base32 | to nuon", value); + assert_eq!(outcome.out, value); + + let outcome = nu!( + "{} | encode base32 --nopad | decode base32 --nopad | to nuon", + value + ); + assert_eq!(outcome.out, value); + } +} + +#[test] +fn encode() { + let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝"; + let encoded = "KPGIFTEPZSTMZF6NTFX43F6MRTGYVTFSZSZMZF3NZSFMZE6MQHGYFTE5MXGYJTEMZWCMZAGMU3GKDTE6ZSSCBTEOZS743BOMUXGJ3TFKM3GZPTMEZSG4ZBWMVLGLXTFHZWEXLTMMZWCMZLWMTXGYK3WNQTGIVTFZZSPGXTMYZSBMZLWNQ7GJ7TMOPHGL5TEVZSDM3BOMWLGKPTFAEDGIFTEQZSM43FWMVXGZI5GMQHGZRTEBZSPGLTF5ZSRM3FOMVB4M3C6MUV2MZEOMSTGZ3TMN"; + + let outcome = nu!("'{}' | encode base32 --nopad", text); + assert_eq!(outcome.out, encoded); +} + +#[test] +fn decode_string() { + let text = "Very important data"; + let encoded = "KZSXE6JANFWXA33SORQW45BAMRQXIYI="; + + let outcome = nu!("'{}' | decode base32 | decode", encoded); + assert_eq!(outcome.out, text); +} + +#[test] +fn decode_pad_nopad() { + let text = "®lnnE¾ˆë"; + let encoded_pad = "YKXGY3TOIXBL5S4GYOVQ===="; + let encoded_nopad = "YKXGY3TOIXBL5S4GYOVQ"; + + let outcome = nu!("'{}' | decode base32 | decode", encoded_pad); + assert_eq!(outcome.out, text); + + let outcome = nu!("'{}' | decode base32 --nopad | decode", encoded_nopad); + assert_eq!(outcome.out, text); +} + +#[test] +fn reject_pad_nopad() { + let encoded_nopad = "ME"; + let encoded_pad = "ME======"; + + let outcome = nu!("'{}' | decode base32", encoded_nopad); + assert!(!outcome.err.is_empty()); + + let outcome = nu!("'{}' | decode base32 --nopad", encoded_pad); + assert!(!outcome.err.is_empty()) +} diff --git a/crates/nu-command/tests/commands/base/base32hex.rs b/crates/nu-command/tests/commands/base/base32hex.rs new file mode 100644 index 0000000000..adb1494bd9 --- /dev/null +++ b/crates/nu-command/tests/commands/base/base32hex.rs @@ -0,0 +1,58 @@ +use nu_test_support::nu; + +#[test] +fn canonical() { + for value in super::random_bytes() { + let outcome = nu!("{} | encode base32hex | decode base32hex | to nuon", value); + assert_eq!(outcome.out, value); + + let outcome = nu!( + "{} | encode base32hex --nopad | decode base32hex --nopad | to nuon", + value + ); + assert_eq!(outcome.out, value); + } +} + +#[test] +fn encode() { + let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝"; + let encoded = "AF685J4FPIJCP5UDJ5NSR5UCHJ6OLJ5IPIPCP5RDPI5CP4UCG76O5J4TCN6O9J4CPM2CP06CKR6A3J4UPII21J4EPIVSR1ECKN69RJ5ACR6PFJC4PI6SP1MCLB6BNJ57PM4NBJCCPM2CPBMCJN6OARMDGJ68LJ5PPIF6NJCOPI1CPBMDGV69VJCEF76BTJ4LPI3CR1ECMB6AFJ5043685J4GPICSR5MCLN6P8T6CG76PHJ41PIF6BJ5TPIHCR5ECL1SCR2UCKLQCP4ECIJ6PRJCD"; + + let outcome = nu!("'{}' | encode base32hex --nopad", text); + assert_eq!(outcome.out, encoded); +} + +#[test] +fn decode_string() { + let text = "Very important data"; + let encoded = "APIN4U90D5MN0RRIEHGMST10CHGN8O8="; + + let outcome = nu!("'{}' | decode base32hex | decode", encoded); + assert_eq!(outcome.out, text); +} + +#[test] +fn decode_pad_nopad() { + let text = "®lnnE¾ˆë"; + let encoded_pad = "OAN6ORJE8N1BTIS6OELG===="; + let encoded_nopad = "OAN6ORJE8N1BTIS6OELG"; + + let outcome = nu!("'{}' | decode base32hex | decode", encoded_pad); + assert_eq!(outcome.out, text); + + let outcome = nu!("'{}' | decode base32hex --nopad | decode", encoded_nopad); + assert_eq!(outcome.out, text); +} + +#[test] +fn reject_pad_nopad() { + let encoded_nopad = "D1KG"; + let encoded_pad = "D1KG===="; + + let outcome = nu!("'{}' | decode base32hex", encoded_nopad); + assert!(!outcome.err.is_empty()); + + let outcome = nu!("'{}' | decode base32hex --nopad", encoded_pad); + assert!(!outcome.err.is_empty()) +} diff --git a/crates/nu-command/tests/commands/base/base64.rs b/crates/nu-command/tests/commands/base/base64.rs new file mode 100644 index 0000000000..12b5f49fff --- /dev/null +++ b/crates/nu-command/tests/commands/base/base64.rs @@ -0,0 +1,86 @@ +use nu_test_support::nu; + +#[test] +fn canonical() { + for value in super::random_bytes() { + let outcome = nu!( + "{} | encode new-base64 | decode new-base64 | to nuon", + value + ); + assert_eq!(outcome.out, value); + + let outcome = nu!( + "{} | encode new-base64 --url | decode new-base64 --url | to nuon", + value + ); + assert_eq!(outcome.out, value); + + let outcome = nu!( + "{} | encode new-base64 --nopad | decode new-base64 --nopad | to nuon", + value + ); + assert_eq!(outcome.out, value); + + let outcome = nu!( + "{} | encode new-base64 --url --nopad | decode new-base64 --url --nopad | to nuon", + value + ); + assert_eq!(outcome.out, value); + } +} + +#[test] +fn encode() { + let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝"; + let encoded = "U8yCzI/MpsyXzZlvzZfMjM2KzLLMssyXbcyKzJPMgc2CzJ1lzYTMjM2EzIDMpsyhzJ7MpCDMjsy/zYXMpcydzKpmzZfNhMyNzIbMqsy7zKfNiXXNjM2EzK7Mnc2Fbs2EzIrMucyea82YzILMrs2HzJ/NjnnMvsyVzIbNhcyyzKfMoCDMgsyQzJnNlsytzZR0zIHNmMyBzJ5lzL3Mos2VzKh4zYvMpXTMkcyUzZ3NjQ=="; + + let outcome = nu!("'{}' | encode new-base64", text); + assert_eq!(outcome.out, encoded); +} + +#[test] +fn decode_string() { + let text = "Very important data"; + let encoded = "VmVyeSBpbXBvcnRhbnQgZGF0YQ=="; + + let outcome = nu!("'{}' | decode new-base64 | decode", encoded); + assert_eq!(outcome.out, text); +} + +#[test] +fn decode_pad_nopad() { + let text = "”¥.ä@°bZö¢"; + let encoded_pad = "4oCdwqUuw6RAwrBiWsO2wqI="; + let encoded_nopad = "4oCdwqUuw6RAwrBiWsO2wqI"; + + let outcome = nu!("'{}' | decode new-base64 | decode", encoded_pad); + assert_eq!(outcome.out, text); + + let outcome = nu!("'{}' | decode new-base64 --nopad | decode", encoded_nopad); + assert_eq!(outcome.out, text); +} + +#[test] +fn decode_url() { + let text = "p:gטݾ߫t+?"; + let encoded = "cDpn15jdvt+rdCs/"; + let encoded_url = "cDpn15jdvt-rdCs_"; + + let outcome = nu!("'{}' | decode new-base64 | decode", encoded); + assert_eq!(outcome.out, text); + + let outcome = nu!("'{}' | decode new-base64 --url | decode", encoded_url); + assert_eq!(outcome.out, text); +} + +#[test] +fn reject_pad_nopad() { + let encoded_nopad = "YQ"; + let encoded_pad = "YQ=="; + + let outcome = nu!("'{}' | decode new-base64", encoded_nopad); + assert!(!outcome.err.is_empty()); + + let outcome = nu!("'{}' | decode new-base64 --nopad", encoded_pad); + assert!(!outcome.err.is_empty()) +} diff --git a/crates/nu-command/tests/commands/base/hex.rs b/crates/nu-command/tests/commands/base/hex.rs new file mode 100644 index 0000000000..5f497408b6 --- /dev/null +++ b/crates/nu-command/tests/commands/base/hex.rs @@ -0,0 +1,36 @@ +use nu_test_support::nu; + +#[test] +fn canonical() { + for value in super::random_bytes() { + let outcome = nu!("{} | encode hex | decode hex | to nuon", value); + assert_eq!(outcome.out, value); + } +} + +#[test] +fn encode() { + let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝"; + let encoded = "53CC82CC8FCCA6CC97CD996FCD97CC8CCD8ACCB2CCB2CC976DCC8ACC93CC81CD82CC9D65CD84CC8CCD84CC80CCA6CCA1CC9ECCA420CC8ECCBFCD85CCA5CC9DCCAA66CD97CD84CC8DCC86CCAACCBBCCA7CD8975CD8CCD84CCAECC9DCD856ECD84CC8ACCB9CC9E6BCD98CC82CCAECD87CC9FCD8E79CCBECC95CC86CD85CCB2CCA7CCA020CC82CC90CC99CD96CCADCD9474CC81CD98CC81CC9E65CCBDCCA2CD95CCA878CD8BCCA574CC91CC94CD9DCD8D"; + + let outcome = nu!("'{}' | encode hex", text); + assert_eq!(outcome.out, encoded); +} + +#[test] +fn decode_string() { + let text = "Very important data"; + let encoded = "5665727920696D706F7274616E742064617461"; + + let outcome = nu!("'{}' | decode hex | decode", encoded); + assert_eq!(outcome.out, text); +} + +#[test] +fn decode_case_mixing() { + let text = "®lnnE¾ˆë"; + let mixed_encoded = "c2aE6c6e6E45C2BeCB86c3ab"; + + let outcome = nu!("'{}' | decode hex | decode", mixed_encoded); + assert_eq!(outcome.out, text); +} diff --git a/crates/nu-command/tests/commands/base/mod.rs b/crates/nu-command/tests/commands/base/mod.rs new file mode 100644 index 0000000000..09828c9211 --- /dev/null +++ b/crates/nu-command/tests/commands/base/mod.rs @@ -0,0 +1,24 @@ +use data_encoding::HEXUPPER; +use rand::prelude::*; +use rand_chacha::ChaCha8Rng; + +mod base32; +mod base32hex; +mod base64; +mod hex; + +/// Generate a few random binaries. +pub fn random_bytes() -> Vec { + const NUM: usize = 32; + let mut rng = ChaCha8Rng::seed_from_u64(4); + + (0..NUM) + .map(|_| { + let length = rng.gen_range(0..512); + let mut bytes = vec![0u8; length]; + rng.fill_bytes(&mut bytes); + let hex_bytes = HEXUPPER.encode(&bytes); + format!("0x[{}]", hex_bytes) + }) + .collect() +} diff --git a/crates/nu-command/tests/commands/mod.rs b/crates/nu-command/tests/commands/mod.rs index 3cb468ad30..63911ebfbc 100644 --- a/crates/nu-command/tests/commands/mod.rs +++ b/crates/nu-command/tests/commands/mod.rs @@ -3,6 +3,7 @@ mod all; mod any; mod append; mod assignment; +mod base; mod break_; mod bytes; mod cal;