encode/decode for multiple alphabets (#13428)

Based on the discussion in #13419.


## Description

Reworks the `decode`/`encode` commands by adding/changing the following
bases:

- `base32`
- `base32hex`
- `hex`
- `new-base64`

The `hex` base is compatible with the previous version of `hex` out of
the box (it only adds more flags). `base64` isn't, so the PR adds a new
version and deprecates the old one.

All commands have `string -> binary` signature for decoding and `string
| binary -> string` signature for encoding. A few `base64` encodings,
which are not a part of the
[RFC4648](https://datatracker.ietf.org/doc/html/rfc4648#section-6), have
been dropped.


## Example usage

```Nushell
~/fork/nushell> "string" | encode base32 | decode base32 | decode
string
```

```Nushell
~/fork/nushell> "ORSXG5A=" | decode base32
# `decode` always returns a binary value
Length: 4 (0x4) bytes | printable whitespace ascii_other non_ascii
00000000:   74 65 73 74                                          test
```


## User-Facing Changes

- New commands: `encode/decode base32/base32hex`.
- `encode hex` gets a `--lower` flag.
- `encode/decode base64` deprecated in favor of `encode/decode
new-base64`.

crates/nu-command/tests/commands/base/base32.rs

@@ -0,0 +1,58 @@
+use nu_test_support::nu;
+
+#[test]
+fn canonical() {
+    for value in super::random_bytes() {
+        let outcome = nu!("{} | encode base32 | decode base32 | to nuon", value);
+        assert_eq!(outcome.out, value);
+
+        let outcome = nu!(
+            "{} | encode base32 --nopad | decode base32 --nopad | to nuon",
+            value
+        );
+        assert_eq!(outcome.out, value);
+    }
+}
+
+#[test]
+fn encode() {
+    let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝";
+    let encoded = "KPGIFTEPZSTMZF6NTFX43F6MRTGYVTFSZSZMZF3NZSFMZE6MQHGYFTE5MXGYJTEMZWCMZAGMU3GKDTE6ZSSCBTEOZS743BOMUXGJ3TFKM3GZPTMEZSG4ZBWMVLGLXTFHZWEXLTMMZWCMZLWMTXGYK3WNQTGIVTFZZSPGXTMYZSBMZLWNQ7GJ7TMOPHGL5TEVZSDM3BOMWLGKPTFAEDGIFTEQZSM43FWMVXGZI5GMQHGZRTEBZSPGLTF5ZSRM3FOMVB4M3C6MUV2MZEOMSTGZ3TMN";
+
+    let outcome = nu!("'{}' | encode base32 --nopad", text);
+    assert_eq!(outcome.out, encoded);
+}
+
+#[test]
+fn decode_string() {
+    let text = "Very important data";
+    let encoded = "KZSXE6JANFWXA33SORQW45BAMRQXIYI=";
+
+    let outcome = nu!("'{}' | decode base32 | decode", encoded);
+    assert_eq!(outcome.out, text);
+}
+
+#[test]
+fn decode_pad_nopad() {
+    let text = "®lnnE¾ˆë";
+    let encoded_pad = "YKXGY3TOIXBL5S4GYOVQ====";
+    let encoded_nopad = "YKXGY3TOIXBL5S4GYOVQ";
+
+    let outcome = nu!("'{}' | decode base32 | decode", encoded_pad);
+    assert_eq!(outcome.out, text);
+
+    let outcome = nu!("'{}' | decode base32 --nopad | decode", encoded_nopad);
+    assert_eq!(outcome.out, text);
+}
+
+#[test]
+fn reject_pad_nopad() {
+    let encoded_nopad = "ME";
+    let encoded_pad = "ME======";
+
+    let outcome = nu!("'{}' | decode base32", encoded_nopad);
+    assert!(!outcome.err.is_empty());
+
+    let outcome = nu!("'{}' | decode base32 --nopad", encoded_pad);
+    assert!(!outcome.err.is_empty())
+}

crates/nu-command/tests/commands/base/base32hex.rs

@@ -0,0 +1,58 @@
+use nu_test_support::nu;
+
+#[test]
+fn canonical() {
+    for value in super::random_bytes() {
+        let outcome = nu!("{} | encode base32hex | decode base32hex | to nuon", value);
+        assert_eq!(outcome.out, value);
+
+        let outcome = nu!(
+            "{} | encode base32hex --nopad | decode base32hex --nopad | to nuon",
+            value
+        );
+        assert_eq!(outcome.out, value);
+    }
+}
+
+#[test]
+fn encode() {
+    let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝";
+    let encoded = "AF685J4FPIJCP5UDJ5NSR5UCHJ6OLJ5IPIPCP5RDPI5CP4UCG76O5J4TCN6O9J4CPM2CP06CKR6A3J4UPII21J4EPIVSR1ECKN69RJ5ACR6PFJC4PI6SP1MCLB6BNJ57PM4NBJCCPM2CPBMCJN6OARMDGJ68LJ5PPIF6NJCOPI1CPBMDGV69VJCEF76BTJ4LPI3CR1ECMB6AFJ5043685J4GPICSR5MCLN6P8T6CG76PHJ41PIF6BJ5TPIHCR5ECL1SCR2UCKLQCP4ECIJ6PRJCD";
+
+    let outcome = nu!("'{}' | encode base32hex --nopad", text);
+    assert_eq!(outcome.out, encoded);
+}
+
+#[test]
+fn decode_string() {
+    let text = "Very important data";
+    let encoded = "APIN4U90D5MN0RRIEHGMST10CHGN8O8=";
+
+    let outcome = nu!("'{}' | decode base32hex | decode", encoded);
+    assert_eq!(outcome.out, text);
+}
+
+#[test]
+fn decode_pad_nopad() {
+    let text = "®lnnE¾ˆë";
+    let encoded_pad = "OAN6ORJE8N1BTIS6OELG====";
+    let encoded_nopad = "OAN6ORJE8N1BTIS6OELG";
+
+    let outcome = nu!("'{}' | decode base32hex | decode", encoded_pad);
+    assert_eq!(outcome.out, text);
+
+    let outcome = nu!("'{}' | decode base32hex --nopad | decode", encoded_nopad);
+    assert_eq!(outcome.out, text);
+}
+
+#[test]
+fn reject_pad_nopad() {
+    let encoded_nopad = "D1KG";
+    let encoded_pad = "D1KG====";
+
+    let outcome = nu!("'{}' | decode base32hex", encoded_nopad);
+    assert!(!outcome.err.is_empty());
+
+    let outcome = nu!("'{}' | decode base32hex --nopad", encoded_pad);
+    assert!(!outcome.err.is_empty())
+}

crates/nu-command/tests/commands/base/base64.rs

@@ -0,0 +1,86 @@
+use nu_test_support::nu;
+
+#[test]
+fn canonical() {
+    for value in super::random_bytes() {
+        let outcome = nu!(
+            "{} | encode new-base64 | decode new-base64 | to nuon",
+            value
+        );
+        assert_eq!(outcome.out, value);
+
+        let outcome = nu!(
+            "{} | encode new-base64 --url | decode new-base64 --url | to nuon",
+            value
+        );
+        assert_eq!(outcome.out, value);
+
+        let outcome = nu!(
+            "{} | encode new-base64 --nopad | decode new-base64 --nopad | to nuon",
+            value
+        );
+        assert_eq!(outcome.out, value);
+
+        let outcome = nu!(
+            "{} | encode new-base64 --url --nopad | decode new-base64 --url --nopad | to nuon",
+            value
+        );
+        assert_eq!(outcome.out, value);
+    }
+}
+
+#[test]
+fn encode() {
+    let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝";
+    let encoded = "U8yCzI/MpsyXzZlvzZfMjM2KzLLMssyXbcyKzJPMgc2CzJ1lzYTMjM2EzIDMpsyhzJ7MpCDMjsy/zYXMpcydzKpmzZfNhMyNzIbMqsy7zKfNiXXNjM2EzK7Mnc2Fbs2EzIrMucyea82YzILMrs2HzJ/NjnnMvsyVzIbNhcyyzKfMoCDMgsyQzJnNlsytzZR0zIHNmMyBzJ5lzL3Mos2VzKh4zYvMpXTMkcyUzZ3NjQ==";
+
+    let outcome = nu!("'{}' | encode new-base64", text);
+    assert_eq!(outcome.out, encoded);
+}
+
+#[test]
+fn decode_string() {
+    let text = "Very important data";
+    let encoded = "VmVyeSBpbXBvcnRhbnQgZGF0YQ==";
+
+    let outcome = nu!("'{}' | decode new-base64 | decode", encoded);
+    assert_eq!(outcome.out, text);
+}
+
+#[test]
+fn decode_pad_nopad() {
+    let text = "”¥.ä@°bZö¢";
+    let encoded_pad = "4oCdwqUuw6RAwrBiWsO2wqI=";
+    let encoded_nopad = "4oCdwqUuw6RAwrBiWsO2wqI";
+
+    let outcome = nu!("'{}' | decode new-base64 | decode", encoded_pad);
+    assert_eq!(outcome.out, text);
+
+    let outcome = nu!("'{}' | decode new-base64 --nopad | decode", encoded_nopad);
+    assert_eq!(outcome.out, text);
+}
+
+#[test]
+fn decode_url() {
+    let text = "p:gטݾ߫t+?";
+    let encoded = "cDpn15jdvt+rdCs/";
+    let encoded_url = "cDpn15jdvt-rdCs_";
+
+    let outcome = nu!("'{}' | decode new-base64 | decode", encoded);
+    assert_eq!(outcome.out, text);
+
+    let outcome = nu!("'{}' | decode new-base64 --url | decode", encoded_url);
+    assert_eq!(outcome.out, text);
+}
+
+#[test]
+fn reject_pad_nopad() {
+    let encoded_nopad = "YQ";
+    let encoded_pad = "YQ==";
+
+    let outcome = nu!("'{}' | decode new-base64", encoded_nopad);
+    assert!(!outcome.err.is_empty());
+
+    let outcome = nu!("'{}' | decode new-base64 --nopad", encoded_pad);
+    assert!(!outcome.err.is_empty())
+}

crates/nu-command/tests/commands/base/hex.rs

@@ -0,0 +1,36 @@
+use nu_test_support::nu;
+
+#[test]
+fn canonical() {
+    for value in super::random_bytes() {
+        let outcome = nu!("{} | encode hex | decode hex | to nuon", value);
+        assert_eq!(outcome.out, value);
+    }
+}
+
+#[test]
+fn encode() {
+    let text = "Ș̗͙̂̏o̲̲̗͗̌͊m̝̊̓́͂ë̡̦̞̤́̌̈́̀ ̥̝̪̎̿ͅf̧̪̻͉͗̈́̍̆u̮̝͌̈́ͅn̹̞̈́̊k̮͇̟͎̂͘y̧̲̠̾̆̕ͅ ̙͖̭͔̂̐t̞́́͘e̢̨͕̽x̥͋t͍̑̔͝";
+    let encoded = "53CC82CC8FCCA6CC97CD996FCD97CC8CCD8ACCB2CCB2CC976DCC8ACC93CC81CD82CC9D65CD84CC8CCD84CC80CCA6CCA1CC9ECCA420CC8ECCBFCD85CCA5CC9DCCAA66CD97CD84CC8DCC86CCAACCBBCCA7CD8975CD8CCD84CCAECC9DCD856ECD84CC8ACCB9CC9E6BCD98CC82CCAECD87CC9FCD8E79CCBECC95CC86CD85CCB2CCA7CCA020CC82CC90CC99CD96CCADCD9474CC81CD98CC81CC9E65CCBDCCA2CD95CCA878CD8BCCA574CC91CC94CD9DCD8D";
+
+    let outcome = nu!("'{}' | encode hex", text);
+    assert_eq!(outcome.out, encoded);
+}
+
+#[test]
+fn decode_string() {
+    let text = "Very important data";
+    let encoded = "5665727920696D706F7274616E742064617461";
+
+    let outcome = nu!("'{}' | decode hex | decode", encoded);
+    assert_eq!(outcome.out, text);
+}
+
+#[test]
+fn decode_case_mixing() {
+    let text = "®lnnE¾ˆë";
+    let mixed_encoded = "c2aE6c6e6E45C2BeCB86c3ab";
+
+    let outcome = nu!("'{}' | decode hex | decode", mixed_encoded);
+    assert_eq!(outcome.out, text);
+}

crates/nu-command/tests/commands/base/mod.rs

@@ -0,0 +1,24 @@
+use data_encoding::HEXUPPER;
+use rand::prelude::*;
+use rand_chacha::ChaCha8Rng;
+
+mod base32;
+mod base32hex;
+mod base64;
+mod hex;
+
+/// Generate a few random binaries.
+pub fn random_bytes() -> Vec<String> {
+    const NUM: usize = 32;
+    let mut rng = ChaCha8Rng::seed_from_u64(4);
+
+    (0..NUM)
+        .map(|_| {
+            let length = rng.gen_range(0..512);
+            let mut bytes = vec![0u8; length];
+            rng.fill_bytes(&mut bytes);
+            let hex_bytes = HEXUPPER.encode(&bytes);
+            format!("0x[{}]", hex_bytes)
+        })
+        .collect()
+}

crates/nu-command/tests/commands/mod.rs

@@ -3,6 +3,7 @@ mod all;
 mod any;
 mod append;
 mod assignment;
+mod base;
 mod break_;
 mod bytes;
 mod cal;