encode/decode for multiple alphabets (#13428)

Based on the discussion in #13419.


## Description

Reworks the `decode`/`encode` commands by adding/changing the following
bases:

- `base32`
- `base32hex`
- `hex`
- `new-base64`

The `hex` base is compatible with the previous version of `hex` out of
the box (it only adds more flags). `base64` isn't, so the PR adds a new
version and deprecates the old one.

All commands have `string -> binary` signature for decoding and `string
| binary -> string` signature for encoding. A few `base64` encodings,
which are not a part of the
[RFC4648](https://datatracker.ietf.org/doc/html/rfc4648#section-6), have
been dropped.


## Example usage

```Nushell
~/fork/nushell> "string" | encode base32 | decode base32 | decode
string
```

```Nushell
~/fork/nushell> "ORSXG5A=" | decode base32
# `decode` always returns a binary value
Length: 4 (0x4) bytes | printable whitespace ascii_other non_ascii
00000000:   74 65 73 74                                          test
```


## User-Facing Changes

- New commands: `encode/decode base32/base32hex`.
- `encode hex` gets a `--lower` flag.
- `encode/decode base64` deprecated in favor of `encode/decode
new-base64`.
This commit is contained in:
Andrej Kolčin 2024-08-23 19:18:51 +03:00 committed by GitHub
parent 39b0f3bdda
commit 0560826414
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 1122 additions and 210 deletions

8
Cargo.lock generated
View File

@ -1153,6 +1153,12 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "data-encoding"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
[[package]]
name = "deranged"
version = "0.3.11"
@ -3066,6 +3072,7 @@ dependencies = [
"chrono-tz 0.8.6",
"crossterm",
"csv",
"data-encoding",
"deunicode",
"dialoguer",
"digest",
@ -3120,6 +3127,7 @@ dependencies = [
"quickcheck",
"quickcheck_macros",
"rand",
"rand_chacha",
"rayon",
"regex",
"rmp",

View File

@ -136,6 +136,7 @@ quickcheck = "1.0"
quickcheck_macros = "1.0"
quote = "1.0"
rand = "0.8"
rand_chacha = "0.3.1"
ratatui = "0.26"
rayon = "1.10"
reedline = "0.34.0"

View File

@ -45,8 +45,6 @@ pub fn add_extra_command_context(mut engine_state: EngineState) -> EngineState {
bind_command!(
strings::format::FormatPattern,
strings::encode_decode::EncodeHex,
strings::encode_decode::DecodeHex,
strings::str_::case::Str,
strings::str_::case::StrCamelCase,
strings::str_::case::StrKebabCase,

View File

@ -1,192 +0,0 @@
use nu_cmd_base::input_handler::{operate as general_operate, CmdArgument};
use nu_engine::command_prelude::*;
enum HexDecodingError {
InvalidLength(usize),
InvalidDigit(usize, char),
}
fn hex_decode(value: &str) -> Result<Vec<u8>, HexDecodingError> {
let mut digits = value
.chars()
.enumerate()
.filter(|(_, c)| !c.is_whitespace());
let mut res = Vec::with_capacity(value.len() / 2);
loop {
let c1 = match digits.next() {
Some((ind, c)) => match c.to_digit(16) {
Some(d) => d,
None => return Err(HexDecodingError::InvalidDigit(ind, c)),
},
None => return Ok(res),
};
let c2 = match digits.next() {
Some((ind, c)) => match c.to_digit(16) {
Some(d) => d,
None => return Err(HexDecodingError::InvalidDigit(ind, c)),
},
None => {
return Err(HexDecodingError::InvalidLength(value.len()));
}
};
res.push((c1 << 4 | c2) as u8);
}
}
fn hex_digit(num: u8) -> char {
match num {
0..=9 => (num + b'0') as char,
10..=15 => (num - 10 + b'A') as char,
_ => unreachable!(),
}
}
fn hex_encode(bytes: &[u8]) -> String {
let mut res = String::with_capacity(bytes.len() * 2);
for byte in bytes {
res.push(hex_digit(byte >> 4));
res.push(hex_digit(byte & 0b1111));
}
res
}
#[derive(Clone)]
pub struct HexConfig {
pub action_type: ActionType,
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum ActionType {
Encode,
Decode,
}
struct Arguments {
cell_paths: Option<Vec<CellPath>>,
encoding_config: HexConfig,
}
impl CmdArgument for Arguments {
fn take_cell_paths(&mut self) -> Option<Vec<CellPath>> {
self.cell_paths.take()
}
}
pub fn operate(
action_type: ActionType,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let cell_paths: Vec<CellPath> = call.rest(engine_state, stack, 0)?;
let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths);
let args = Arguments {
encoding_config: HexConfig { action_type },
cell_paths,
};
general_operate(action, args, input, call.head, engine_state.signals())
}
fn action(
input: &Value,
// only used for `decode` action
args: &Arguments,
command_span: Span,
) -> Value {
let hex_config = &args.encoding_config;
match input {
// Propagate existing errors.
Value::Error { .. } => input.clone(),
Value::Binary { val, .. } => match hex_config.action_type {
ActionType::Encode => Value::string(hex_encode(val.as_ref()), command_span),
ActionType::Decode => Value::error(
ShellError::UnsupportedInput { msg: "Binary data can only be encoded".to_string(), input: "value originates from here".into(), msg_span: command_span, input_span: input.span() },
command_span,
),
},
Value::String { val, .. } => {
match hex_config.action_type {
ActionType::Encode => Value::error(
ShellError::UnsupportedInput { msg: "String value can only be decoded".to_string(), input: "value originates from here".into(), msg_span: command_span, input_span: input.span() },
command_span,
),
ActionType::Decode => match hex_decode(val.as_ref()) {
Ok(decoded_value) => Value::binary(decoded_value, command_span),
Err(HexDecodingError::InvalidLength(len)) => Value::error(ShellError::GenericError {
error: "value could not be hex decoded".into(),
msg: format!("invalid hex input length: {len}. The length should be even"),
span: Some(command_span),
help: None,
inner: vec![],
},
command_span,
),
Err(HexDecodingError::InvalidDigit(index, digit)) => Value::error(ShellError::GenericError {
error: "value could not be hex decoded".into(),
msg: format!("invalid hex digit: '{digit}' at index {index}. Only 0-9, A-F, a-f are allowed in hex encoding"),
span: Some(command_span),
help: None,
inner: vec![],
},
command_span,
),
},
}
}
other => Value::error(
ShellError::TypeMismatch {
err_message: format!("string or binary, not {}", other.get_type()),
span: other.span(),
},
other.span(),
),
}
}
#[cfg(test)]
mod tests {
use super::{action, ActionType, Arguments, HexConfig};
use nu_protocol::{Span, Value};
#[test]
fn hex_encode() {
let word = Value::binary([77, 97, 110], Span::test_data());
let expected = Value::test_string("4D616E");
let actual = action(
&word,
&Arguments {
encoding_config: HexConfig {
action_type: ActionType::Encode,
},
cell_paths: None,
},
Span::test_data(),
);
assert_eq!(actual, expected);
}
#[test]
fn hex_decode() {
let word = Value::test_string("4D 61\r\n\n6E");
let expected = Value::binary([77, 97, 110], Span::test_data());
let actual = action(
&word,
&Arguments {
encoding_config: HexConfig {
action_type: ActionType::Decode,
},
cell_paths: None,
},
Span::test_data(),
);
assert_eq!(actual, expected);
}
}

View File

@ -1,6 +0,0 @@
mod decode_hex;
mod encode_hex;
mod hex;
pub(crate) use decode_hex::DecodeHex;
pub(crate) use encode_hex::EncodeHex;

View File

@ -1,3 +1,2 @@
pub(crate) mod encode_decode;
pub(crate) mod format;
pub(crate) mod str_;

View File

@ -102,6 +102,7 @@ v_htmlescape = { workspace = true }
wax = { workspace = true }
which = { workspace = true }
unicode-width = { workspace = true }
data-encoding = { version = "2.6.0", features = ["alloc"] }
[target.'cfg(windows)'.dependencies]
winreg = { workspace = true }
@ -146,4 +147,5 @@ quickcheck = { workspace = true }
quickcheck_macros = { workspace = true }
rstest = { workspace = true, default-features = false }
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
tempfile = { workspace = true }
rand_chacha = { workspace = true }

View File

@ -179,8 +179,16 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState {
Char,
Decode,
Encode,
DecodeHex,
EncodeHex,
DecodeBase32,
EncodeBase32,
DecodeBase32Hex,
EncodeBase32Hex,
DecodeBase64,
EncodeBase64,
DecodeBase64Old,
EncodeBase64Old,
DetectColumns,
Parse,
Split,

View File

@ -0,0 +1,180 @@
use data_encoding::Encoding;
use nu_engine::command_prelude::*;
const EXTRA_USAGE: &str = r"The default alphabet is taken from RFC 4648, section 6.
Note this command will collect stream input.";
#[derive(Clone)]
pub struct DecodeBase32;
impl Command for DecodeBase32 {
fn name(&self) -> &str {
"decode base32"
}
fn signature(&self) -> Signature {
Signature::build("decode base32")
.input_output_types(vec![(Type::String, Type::Binary)])
.allow_variants_without_examples(true)
.switch("nopad", "Do not pad the output.", None)
.category(Category::Formats)
}
fn description(&self) -> &str {
"Decode a Base32 value."
}
fn extra_description(&self) -> &str {
EXTRA_USAGE
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Decode arbitrary binary data",
example: r#""AEBAGBAF" | decode base32"#,
result: Some(Value::test_binary(vec![1, 2, 3, 4, 5])),
},
Example {
description: "Decode an encoded string",
example: r#""NBUQ====" | decode base32 | decode"#,
result: None,
},
Example {
description: "Parse a string without padding",
example: r#""NBUQ" | decode base32 --nopad"#,
result: Some(Value::test_binary(vec![0x68, 0x69])),
},
]
}
fn is_const(&self) -> bool {
true
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = if call.has_flag(engine_state, stack, "nopad")? {
data_encoding::BASE32_NOPAD
} else {
data_encoding::BASE32
};
super::decode(encoding, call.span(), input)
}
fn run_const(
&self,
working_set: &StateWorkingSet,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = if call.has_flag_const(working_set, "nopad")? {
data_encoding::BASE32_NOPAD
} else {
data_encoding::BASE32
};
super::decode(encoding, call.span(), input)
}
}
#[derive(Clone)]
pub struct EncodeBase32;
impl Command for EncodeBase32 {
fn name(&self) -> &str {
"encode base32"
}
fn signature(&self) -> Signature {
Signature::build("encode base32")
.input_output_types(vec![
(Type::String, Type::String),
(Type::Binary, Type::String),
])
.switch("nopad", "Don't accept padding.", None)
.category(Category::Formats)
}
fn description(&self) -> &str {
"Encode a string or binary value using Base32."
}
fn extra_description(&self) -> &str {
EXTRA_USAGE
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Encode a binary value",
example: r#"0x[01 02 10] | encode base32"#,
result: Some(Value::test_string("AEBBA===")),
},
Example {
description: "Encode a string",
example: r#""hello there" | encode base32"#,
result: Some(Value::test_string("NBSWY3DPEB2GQZLSMU======")),
},
Example {
description: "Don't apply padding to the output",
example: r#""hi" | encode base32 --nopad"#,
result: Some(Value::test_string("NBUQ")),
},
]
}
fn is_const(&self) -> bool {
true
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = if call.has_flag(engine_state, stack, "nopad")? {
data_encoding::BASE32_NOPAD
} else {
data_encoding::BASE32
};
super::encode(encoding, call.span(), input)
}
fn run_const(
&self,
working_set: &StateWorkingSet,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = if call.has_flag_const(working_set, "nopad")? {
data_encoding::BASE32_NOPAD
} else {
data_encoding::BASE32
};
super::encode(encoding, call.span(), input)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples_decode() {
crate::test_examples(DecodeBase32)
}
#[test]
fn test_examples_encode() {
crate::test_examples(EncodeBase32)
}
}

View File

@ -0,0 +1,181 @@
use nu_engine::command_prelude::*;
const EXTRA_USAGE: &str = r"This command uses an alternative Base32 alphabet, defined in RFC 4648, section 7.
Note this command will collect stream input.";
#[derive(Clone)]
pub struct DecodeBase32Hex;
impl Command for DecodeBase32Hex {
fn name(&self) -> &str {
"decode base32hex"
}
fn signature(&self) -> Signature {
Signature::build("decode base32hex")
.input_output_types(vec![(Type::String, Type::Binary)])
.allow_variants_without_examples(true)
.switch("nopad", "Reject input with padding.", None)
.category(Category::Formats)
}
fn description(&self) -> &str {
"Encode a base32hex value."
}
fn extra_description(&self) -> &str {
EXTRA_USAGE
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Decode arbitrary binary data",
example: r#""ATNAQ===" | decode base32hex"#,
result: Some(Value::test_binary(vec![0x57, 0x6E, 0xAD])),
},
Example {
description: "Decode an encoded string",
example: r#""D1KG====" | decode base32hex | decode"#,
result: None,
},
Example {
description: "Parse a string without padding",
example: r#""ATNAQ" | decode base32hex --nopad"#,
result: Some(Value::test_binary(vec![0x57, 0x6E, 0xAD])),
},
]
}
fn is_const(&self) -> bool {
true
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = if call.has_flag(engine_state, stack, "nopad")? {
data_encoding::BASE32HEX_NOPAD
} else {
data_encoding::BASE32HEX
};
super::decode(encoding, call.head, input)
}
fn run_const(
&self,
working_set: &StateWorkingSet,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = if call.has_flag_const(working_set, "nopad")? {
data_encoding::BASE32HEX_NOPAD
} else {
data_encoding::BASE32HEX
};
super::decode(encoding, call.head, input)
}
}
#[derive(Clone)]
pub struct EncodeBase32Hex;
impl Command for EncodeBase32Hex {
fn name(&self) -> &str {
"encode base32hex"
}
fn signature(&self) -> Signature {
Signature::build("encode base32hex")
.input_output_types(vec![
(Type::String, Type::String),
(Type::Binary, Type::String),
])
.switch("nopad", "Don't pad the output.", None)
.category(Category::Formats)
}
fn description(&self) -> &str {
"Encode a binary value or a string using base32hex."
}
fn extra_description(&self) -> &str {
EXTRA_USAGE
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Encode a binary value",
example: r#"0x[57 6E AD] | encode base32hex"#,
result: Some(Value::test_string("ATNAQ===")),
},
Example {
description: "Encode a string",
example: r#""hello there" | encode base32hex"#,
result: Some(Value::test_string("D1IMOR3F41Q6GPBICK======")),
},
Example {
description: "Don't apply padding to the output",
example: r#""hello there" | encode base32hex --nopad"#,
result: Some(Value::test_string("D1IMOR3F41Q6GPBICK")),
},
]
}
fn is_const(&self) -> bool {
true
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = if call.has_flag(engine_state, stack, "nopad")? {
data_encoding::BASE32HEX_NOPAD
} else {
data_encoding::BASE32HEX
};
super::encode(encoding, call.head, input)
}
fn run_const(
&self,
working_set: &StateWorkingSet,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = if call.has_flag_const(working_set, "nopad")? {
data_encoding::BASE32HEX_NOPAD
} else {
data_encoding::BASE32HEX
};
super::encode(encoding, call.head, input)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples_decode() {
crate::test_examples(DecodeBase32Hex)
}
#[test]
fn test_examples_encode() {
crate::test_examples(EncodeBase32Hex)
}
}

View File

@ -0,0 +1,193 @@
use data_encoding::Encoding;
use nu_engine::command_prelude::*;
const EXTRA_USAGE: &str = r"The default alphabet is taken from RFC 4648, section 4. A URL-safe version is available.
Note this command will collect stream input.";
fn get_encoding_from_flags(url: bool, nopad: bool) -> Encoding {
match (url, nopad) {
(false, false) => data_encoding::BASE64,
(false, true) => data_encoding::BASE64_NOPAD,
(true, false) => data_encoding::BASE64URL,
(true, true) => data_encoding::BASE64URL_NOPAD,
}
}
fn get_encoding(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Encoding, ShellError> {
let url = call.has_flag(engine_state, stack, "url")?;
let nopad = call.has_flag(engine_state, stack, "nopad")?;
Ok(get_encoding_from_flags(url, nopad))
}
fn get_encoding_const(working_set: &StateWorkingSet, call: &Call) -> Result<Encoding, ShellError> {
let url = call.has_flag_const(working_set, "url")?;
let nopad = call.has_flag_const(working_set, "nopad")?;
Ok(get_encoding_from_flags(url, nopad))
}
#[derive(Clone)]
pub struct DecodeBase64;
impl Command for DecodeBase64 {
fn name(&self) -> &str {
"decode new-base64"
}
fn signature(&self) -> Signature {
Signature::build("decode new-base64")
.input_output_types(vec![(Type::String, Type::Binary)])
.allow_variants_without_examples(true)
.switch("url", "Decode the URL-safe Base64 version.", None)
.switch("nopad", "Reject padding.", None)
.category(Category::Formats)
}
fn description(&self) -> &str {
"Decode a Base64 value."
}
fn extra_description(&self) -> &str {
EXTRA_USAGE
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Decode a Base64 string",
example: r#""U29tZSBEYXRh" | decode new-base64 | decode"#,
result: None,
},
Example {
description: "Decode arbitrary data",
example: r#""/w==" | decode new-base64"#,
result: Some(Value::test_binary(vec![0xFF])),
},
Example {
description: "Decode a URL-safe Base64 string",
example: r#""_w==" | decode new-base64 --url"#,
result: Some(Value::test_binary(vec![0xFF])),
},
]
}
fn is_const(&self) -> bool {
true
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = get_encoding(engine_state, stack, call)?;
super::decode(encoding, call.head, input)
}
fn run_const(
&self,
working_set: &StateWorkingSet,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = get_encoding_const(working_set, call)?;
super::decode(encoding, call.head, input)
}
}
#[derive(Clone)]
pub struct EncodeBase64;
impl Command for EncodeBase64 {
fn name(&self) -> &str {
"encode new-base64"
}
fn signature(&self) -> Signature {
Signature::build("encode new-base64")
.input_output_types(vec![
(Type::String, Type::String),
(Type::Binary, Type::String),
])
.switch("url", "Use the URL-safe Base64 version.", None)
.switch("nopad", "Don't pad the output.", None)
.category(Category::Formats)
}
fn description(&self) -> &str {
"Encode a string or binary value using Base64."
}
fn extra_description(&self) -> &str {
EXTRA_USAGE
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Encode a string with Base64",
example: r#""Alphabet from A to Z" | encode new-base64"#,
result: Some(Value::test_string("QWxwaGFiZXQgZnJvbSBBIHRvIFo=")),
},
Example {
description: "Encode arbitrary data",
example: r#"0x[BE EE FF] | encode new-base64"#,
result: Some(Value::test_string("vu7/")),
},
Example {
description: "Use a URL-safe alphabet",
example: r#"0x[BE EE FF] | encode new-base64 --url"#,
result: Some(Value::test_string("vu7_")),
},
]
}
fn is_const(&self) -> bool {
true
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = get_encoding(engine_state, stack, call)?;
super::encode(encoding, call.head, input)
}
fn run_const(
&self,
working_set: &StateWorkingSet,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = get_encoding_const(working_set, call)?;
super::encode(encoding, call.head, input)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples_decode() {
crate::test_examples(DecodeBase64)
}
#[test]
fn test_examples_encode() {
crate::test_examples(EncodeBase64)
}
}

View File

@ -0,0 +1,151 @@
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct DecodeHex;
impl Command for DecodeHex {
fn name(&self) -> &str {
"decode hex"
}
fn signature(&self) -> Signature {
Signature::build("decode hex")
.input_output_types(vec![(Type::String, Type::Binary)])
.category(Category::Formats)
}
fn description(&self) -> &str {
"Hex decode a value."
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Decode arbitrary binary data",
example: r#""09FD" | decode hex"#,
result: Some(Value::test_binary(vec![0x09, 0xFD])),
},
Example {
description: "Lowercase Hex is also accepted",
example: r#""09fd" | decode hex"#,
result: Some(Value::test_binary(vec![0x09, 0xFD])),
},
]
}
fn is_const(&self) -> bool {
true
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
super::decode(data_encoding::HEXLOWER_PERMISSIVE, call.head, input)
}
fn run_const(
&self,
working_set: &StateWorkingSet,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
todo!()
}
}
#[derive(Clone)]
pub struct EncodeHex;
impl Command for EncodeHex {
fn name(&self) -> &str {
"encode hex"
}
fn signature(&self) -> Signature {
Signature::build("encode hex")
.input_output_types(vec![
(Type::String, Type::String),
(Type::Binary, Type::String),
])
.switch("lower", "Encode to lowercase hex.", None)
.category(Category::Formats)
}
fn description(&self) -> &str {
"Hex encode a binary value or a string."
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Encode a binary value",
example: r#"0x[C3 06] | encode hex"#,
result: Some(Value::test_string("C306")),
},
Example {
description: "Encode a string",
example: r#""hello" | encode hex"#,
result: Some(Value::test_string("68656C6C6F")),
},
Example {
description: "Output a Lowercase version of the encoding",
example: r#"0x[AD EF] | encode hex --lower"#,
result: Some(Value::test_string("adef")),
},
]
}
fn is_const(&self) -> bool {
true
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = if call.has_flag(engine_state, stack, "lower")? {
data_encoding::HEXLOWER
} else {
data_encoding::HEXUPPER
};
super::encode(encoding, call.head, input)
}
fn run_const(
&self,
working_set: &StateWorkingSet,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let encoding = if call.has_flag_const(working_set, "lower")? {
data_encoding::HEXLOWER
} else {
data_encoding::HEXUPPER
};
super::encode(encoding, call.head, input)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_examples_decode() {
crate::test_examples(DecodeHex)
}
#[test]
fn test_examples_encode() {
crate::test_examples(EncodeHex)
}
}

View File

@ -0,0 +1,99 @@
#![allow(unused)]
use data_encoding::Encoding;
use nu_engine::command_prelude::*;
mod base32;
mod base32hex;
mod base64;
mod hex;
pub use base32::{DecodeBase32, EncodeBase32};
pub use base32hex::{DecodeBase32Hex, EncodeBase32Hex};
pub use base64::{DecodeBase64, EncodeBase64};
pub use hex::{DecodeHex, EncodeHex};
pub fn decode(
encoding: Encoding,
call_span: Span,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let metadata = input.metadata();
let (input_str, input_span) = get_string(input, call_span)?;
let output = match encoding.decode(input_str.as_bytes()) {
Ok(output) => output,
Err(err) => {
return Err(ShellError::IncorrectValue {
msg: err.to_string(),
val_span: input_span,
call_span,
});
}
};
Ok(Value::binary(output, call_span).into_pipeline_data_with_metadata(metadata))
}
pub fn encode(
encoding: Encoding,
call_span: Span,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let metadata = input.metadata();
let (input_bytes, _) = get_binary(input, call_span)?;
let output = encoding.encode(&input_bytes);
Ok(Value::string(output, call_span).into_pipeline_data_with_metadata(metadata))
}
fn get_string(input: PipelineData, call_span: Span) -> Result<(String, Span), ShellError> {
match input {
PipelineData::Value(val, ..) => {
let span = val.span();
match val {
Value::String { val, .. } => Ok((val, span)),
_ => {
todo!("Invalid type")
}
}
}
PipelineData::ListStream(..) => {
todo!()
}
PipelineData::ByteStream(stream, ..) => {
let span = stream.span();
Ok((stream.into_string()?, span))
}
PipelineData::Empty => Err(ShellError::PipelineEmpty {
dst_span: call_span,
}),
}
}
fn get_binary(input: PipelineData, call_span: Span) -> Result<(Vec<u8>, Span), ShellError> {
match input {
PipelineData::Value(val, ..) => {
let span = val.span();
match val {
Value::Binary { val, .. } => Ok((val, span)),
Value::String { val, .. } => Ok((val.into_bytes(), span)),
_ => {
todo!("Invalid type")
}
}
}
PipelineData::ListStream(..) => {
todo!()
}
PipelineData::ByteStream(stream, ..) => {
let span = stream.span();
Ok((stream.into_bytes()?, span))
}
PipelineData::Empty => {
todo!("Can't have empty data");
}
}
}

View File

@ -1,10 +1,11 @@
use super::base64::{operate, ActionType, Base64CommandArguments, CHARACTER_SET_DESC};
use nu_engine::command_prelude::*;
use nu_protocol::{report_warning_new, ParseWarning};
#[derive(Clone)]
pub struct DecodeBase64;
pub struct DecodeBase64Old;
impl Command for DecodeBase64 {
impl Command for DecodeBase64Old {
fn name(&self) -> &str {
"decode base64"
}
@ -77,6 +78,16 @@ impl Command for DecodeBase64 {
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
report_warning_new(
engine_state,
&ParseWarning::DeprecatedWarning {
old_command: "decode base64".into(),
new_suggestion: "the new `decode new-base64` version".into(),
span: call.head,
url: "`help decode new-base64`".into(),
},
);
let character_set: Option<Spanned<String>> =
call.get_flag(engine_state, stack, "character-set")?;
let binary = call.has_flag(engine_state, stack, "binary")?;
@ -114,6 +125,6 @@ mod tests {
#[test]
fn test_examples() {
crate::test_examples(DecodeBase64)
crate::test_examples(DecodeBase64Old)
}
}

View File

@ -1,10 +1,11 @@
use super::base64::{operate, ActionType, Base64CommandArguments, CHARACTER_SET_DESC};
use nu_engine::command_prelude::*;
use nu_protocol::{report_warning_new, ParseWarning};
#[derive(Clone)]
pub struct EncodeBase64;
pub struct EncodeBase64Old;
impl Command for EncodeBase64 {
impl Command for EncodeBase64Old {
fn name(&self) -> &str {
"encode base64"
}
@ -81,6 +82,16 @@ impl Command for EncodeBase64 {
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
report_warning_new(
engine_state,
&ParseWarning::DeprecatedWarning {
old_command: "encode base64".into(),
new_suggestion: "the new `encode new-base64` version".into(),
span: call.head,
url: "`help encode new-base64`".into(),
},
);
let character_set: Option<Spanned<String>> =
call.get_flag(engine_state, stack, "character-set")?;
let binary = call.has_flag(engine_state, stack, "binary")?;
@ -118,6 +129,6 @@ mod tests {
#[test]
fn test_examples() {
crate::test_examples(EncodeBase64)
crate::test_examples(EncodeBase64Old)
}
}

View File

@ -6,6 +6,6 @@ mod encode_base64;
mod encoding;
pub use self::decode::Decode;
pub use self::decode_base64::DecodeBase64;
pub use self::decode_base64::DecodeBase64Old;
pub use self::encode::Encode;
pub use self::encode_base64::EncodeBase64;
pub use self::encode_base64::EncodeBase64Old;

View File

@ -1,3 +1,4 @@
mod base;
mod char_;
mod detect_columns;
mod encode_decode;
@ -7,6 +8,10 @@ mod parse;
mod split;
mod str_;
pub use base::{
DecodeBase32, DecodeBase32Hex, DecodeBase64, DecodeHex, EncodeBase32, EncodeBase32Hex,
EncodeBase64, EncodeHex,
};
pub use char_::Char;
pub use detect_columns::*;
pub use encode_decode::*;

View File

@ -0,0 +1,58 @@
use nu_test_support::nu;
#[test]
fn canonical() {
for value in super::random_bytes() {
let outcome = nu!("{} | encode base32 | decode base32 | to nuon", value);
assert_eq!(outcome.out, value);
let outcome = nu!(
"{} | encode base32 --nopad | decode base32 --nopad | to nuon",
value
);
assert_eq!(outcome.out, value);
}
}
#[test]
fn encode() {
let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝";
let encoded = "KPGIFTEPZSTMZF6NTFX43F6MRTGYVTFSZSZMZF3NZSFMZE6MQHGYFTE5MXGYJTEMZWCMZAGMU3GKDTE6ZSSCBTEOZS743BOMUXGJ3TFKM3GZPTMEZSG4ZBWMVLGLXTFHZWEXLTMMZWCMZLWMTXGYK3WNQTGIVTFZZSPGXTMYZSBMZLWNQ7GJ7TMOPHGL5TEVZSDM3BOMWLGKPTFAEDGIFTEQZSM43FWMVXGZI5GMQHGZRTEBZSPGLTF5ZSRM3FOMVB4M3C6MUV2MZEOMSTGZ3TMN";
let outcome = nu!("'{}' | encode base32 --nopad", text);
assert_eq!(outcome.out, encoded);
}
#[test]
fn decode_string() {
let text = "Very important data";
let encoded = "KZSXE6JANFWXA33SORQW45BAMRQXIYI=";
let outcome = nu!("'{}' | decode base32 | decode", encoded);
assert_eq!(outcome.out, text);
}
#[test]
fn decode_pad_nopad() {
let text = "®lnnE¾ˆë";
let encoded_pad = "YKXGY3TOIXBL5S4GYOVQ====";
let encoded_nopad = "YKXGY3TOIXBL5S4GYOVQ";
let outcome = nu!("'{}' | decode base32 | decode", encoded_pad);
assert_eq!(outcome.out, text);
let outcome = nu!("'{}' | decode base32 --nopad | decode", encoded_nopad);
assert_eq!(outcome.out, text);
}
#[test]
fn reject_pad_nopad() {
let encoded_nopad = "ME";
let encoded_pad = "ME======";
let outcome = nu!("'{}' | decode base32", encoded_nopad);
assert!(!outcome.err.is_empty());
let outcome = nu!("'{}' | decode base32 --nopad", encoded_pad);
assert!(!outcome.err.is_empty())
}

View File

@ -0,0 +1,58 @@
use nu_test_support::nu;
#[test]
fn canonical() {
for value in super::random_bytes() {
let outcome = nu!("{} | encode base32hex | decode base32hex | to nuon", value);
assert_eq!(outcome.out, value);
let outcome = nu!(
"{} | encode base32hex --nopad | decode base32hex --nopad | to nuon",
value
);
assert_eq!(outcome.out, value);
}
}
#[test]
fn encode() {
let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝";
let encoded = "AF685J4FPIJCP5UDJ5NSR5UCHJ6OLJ5IPIPCP5RDPI5CP4UCG76O5J4TCN6O9J4CPM2CP06CKR6A3J4UPII21J4EPIVSR1ECKN69RJ5ACR6PFJC4PI6SP1MCLB6BNJ57PM4NBJCCPM2CPBMCJN6OARMDGJ68LJ5PPIF6NJCOPI1CPBMDGV69VJCEF76BTJ4LPI3CR1ECMB6AFJ5043685J4GPICSR5MCLN6P8T6CG76PHJ41PIF6BJ5TPIHCR5ECL1SCR2UCKLQCP4ECIJ6PRJCD";
let outcome = nu!("'{}' | encode base32hex --nopad", text);
assert_eq!(outcome.out, encoded);
}
#[test]
fn decode_string() {
let text = "Very important data";
let encoded = "APIN4U90D5MN0RRIEHGMST10CHGN8O8=";
let outcome = nu!("'{}' | decode base32hex | decode", encoded);
assert_eq!(outcome.out, text);
}
#[test]
fn decode_pad_nopad() {
let text = "®lnnE¾ˆë";
let encoded_pad = "OAN6ORJE8N1BTIS6OELG====";
let encoded_nopad = "OAN6ORJE8N1BTIS6OELG";
let outcome = nu!("'{}' | decode base32hex | decode", encoded_pad);
assert_eq!(outcome.out, text);
let outcome = nu!("'{}' | decode base32hex --nopad | decode", encoded_nopad);
assert_eq!(outcome.out, text);
}
#[test]
fn reject_pad_nopad() {
let encoded_nopad = "D1KG";
let encoded_pad = "D1KG====";
let outcome = nu!("'{}' | decode base32hex", encoded_nopad);
assert!(!outcome.err.is_empty());
let outcome = nu!("'{}' | decode base32hex --nopad", encoded_pad);
assert!(!outcome.err.is_empty())
}

View File

@ -0,0 +1,86 @@
use nu_test_support::nu;
#[test]
fn canonical() {
for value in super::random_bytes() {
let outcome = nu!(
"{} | encode new-base64 | decode new-base64 | to nuon",
value
);
assert_eq!(outcome.out, value);
let outcome = nu!(
"{} | encode new-base64 --url | decode new-base64 --url | to nuon",
value
);
assert_eq!(outcome.out, value);
let outcome = nu!(
"{} | encode new-base64 --nopad | decode new-base64 --nopad | to nuon",
value
);
assert_eq!(outcome.out, value);
let outcome = nu!(
"{} | encode new-base64 --url --nopad | decode new-base64 --url --nopad | to nuon",
value
);
assert_eq!(outcome.out, value);
}
}
#[test]
fn encode() {
let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝";
let encoded = "U8yCzI/MpsyXzZlvzZfMjM2KzLLMssyXbcyKzJPMgc2CzJ1lzYTMjM2EzIDMpsyhzJ7MpCDMjsy/zYXMpcydzKpmzZfNhMyNzIbMqsy7zKfNiXXNjM2EzK7Mnc2Fbs2EzIrMucyea82YzILMrs2HzJ/NjnnMvsyVzIbNhcyyzKfMoCDMgsyQzJnNlsytzZR0zIHNmMyBzJ5lzL3Mos2VzKh4zYvMpXTMkcyUzZ3NjQ==";
let outcome = nu!("'{}' | encode new-base64", text);
assert_eq!(outcome.out, encoded);
}
#[test]
fn decode_string() {
let text = "Very important data";
let encoded = "VmVyeSBpbXBvcnRhbnQgZGF0YQ==";
let outcome = nu!("'{}' | decode new-base64 | decode", encoded);
assert_eq!(outcome.out, text);
}
#[test]
fn decode_pad_nopad() {
let text = "”¥.ä@°bZö¢";
let encoded_pad = "4oCdwqUuw6RAwrBiWsO2wqI=";
let encoded_nopad = "4oCdwqUuw6RAwrBiWsO2wqI";
let outcome = nu!("'{}' | decode new-base64 | decode", encoded_pad);
assert_eq!(outcome.out, text);
let outcome = nu!("'{}' | decode new-base64 --nopad | decode", encoded_nopad);
assert_eq!(outcome.out, text);
}
#[test]
fn decode_url() {
let text = "p:gטݾ߫t+?";
let encoded = "cDpn15jdvt+rdCs/";
let encoded_url = "cDpn15jdvt-rdCs_";
let outcome = nu!("'{}' | decode new-base64 | decode", encoded);
assert_eq!(outcome.out, text);
let outcome = nu!("'{}' | decode new-base64 --url | decode", encoded_url);
assert_eq!(outcome.out, text);
}
#[test]
fn reject_pad_nopad() {
let encoded_nopad = "YQ";
let encoded_pad = "YQ==";
let outcome = nu!("'{}' | decode new-base64", encoded_nopad);
assert!(!outcome.err.is_empty());
let outcome = nu!("'{}' | decode new-base64 --nopad", encoded_pad);
assert!(!outcome.err.is_empty())
}

View File

@ -0,0 +1,36 @@
use nu_test_support::nu;
#[test]
fn canonical() {
for value in super::random_bytes() {
let outcome = nu!("{} | encode hex | decode hex | to nuon", value);
assert_eq!(outcome.out, value);
}
}
#[test]
fn encode() {
let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝";
let encoded = "53CC82CC8FCCA6CC97CD996FCD97CC8CCD8ACCB2CCB2CC976DCC8ACC93CC81CD82CC9D65CD84CC8CCD84CC80CCA6CCA1CC9ECCA420CC8ECCBFCD85CCA5CC9DCCAA66CD97CD84CC8DCC86CCAACCBBCCA7CD8975CD8CCD84CCAECC9DCD856ECD84CC8ACCB9CC9E6BCD98CC82CCAECD87CC9FCD8E79CCBECC95CC86CD85CCB2CCA7CCA020CC82CC90CC99CD96CCADCD9474CC81CD98CC81CC9E65CCBDCCA2CD95CCA878CD8BCCA574CC91CC94CD9DCD8D";
let outcome = nu!("'{}' | encode hex", text);
assert_eq!(outcome.out, encoded);
}
#[test]
fn decode_string() {
let text = "Very important data";
let encoded = "5665727920696D706F7274616E742064617461";
let outcome = nu!("'{}' | decode hex | decode", encoded);
assert_eq!(outcome.out, text);
}
#[test]
fn decode_case_mixing() {
let text = "®lnnE¾ˆë";
let mixed_encoded = "c2aE6c6e6E45C2BeCB86c3ab";
let outcome = nu!("'{}' | decode hex | decode", mixed_encoded);
assert_eq!(outcome.out, text);
}

View File

@ -0,0 +1,24 @@
use data_encoding::HEXUPPER;
use rand::prelude::*;
use rand_chacha::ChaCha8Rng;
mod base32;
mod base32hex;
mod base64;
mod hex;
/// Generate a few random binaries.
pub fn random_bytes() -> Vec<String> {
const NUM: usize = 32;
let mut rng = ChaCha8Rng::seed_from_u64(4);
(0..NUM)
.map(|_| {
let length = rng.gen_range(0..512);
let mut bytes = vec![0u8; length];
rng.fill_bytes(&mut bytes);
let hex_bytes = HEXUPPER.encode(&bytes);
format!("0x[{}]", hex_bytes)
})
.collect()
}

View File

@ -3,6 +3,7 @@ mod all;
mod any;
mod append;
mod assignment;
mod base;
mod break_;
mod bytes;
mod cal;