From 835bbb2e44c41bc2db1ccdc097d766702b9e6338 Mon Sep 17 00:00:00 2001 From: Darren Schroeder <343840+fdncred@users.noreply.github.com> Date: Fri, 13 Jan 2023 07:16:14 -0600 Subject: [PATCH] update base64 implementation to newer crate (#7739) # Description This PR updates the base64 crate, which has changed significantly, so all the base64 implementations had to be changed too. Tests pass. I hope that's enough. # User-Facing Changes None, except added a new character encoding imap-mutf7 as mutf7. # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date. --- Cargo.lock | 12 +++-- crates/nu-command/Cargo.toml | 53 +++++++++++-------- crates/nu-command/src/network/fetch.rs | 27 +++++++--- crates/nu-command/src/network/post.rs | 23 +++++--- .../src/strings/encode_decode/base64.rs | 50 ++++++++++++----- 5 files changed, 111 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b77e1857..f51d59a29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -156,7 +156,7 @@ checksum = "ee6f62e41078c967a4c063fcbdfd3801a2a9632276402c045311c4d73d0845f3" dependencies = [ "ahash 0.7.6", "arrow-format", - "base64", + "base64 0.13.0", "bytemuck", "chrono", "dyn-clone", @@ -272,6 +272,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bincode" version = "1.3.3" @@ -2639,7 +2645,7 @@ dependencies = [ "Inflector", "alphanumeric-sort", "atty", - "base64", + "base64 0.21.0", "byteorder", "bytesize", "calamine", @@ -4137,7 +4143,7 @@ version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" dependencies = [ - "base64", + "base64 0.13.0", "bytes", "encoding_rs", "futures-core", diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index bded2fea4..cd13546ee 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -11,17 +11,17 @@ build = "build.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nu-color-config = { path = "../nu-color-config", version = "0.74.1" } -nu-engine = { path = "../nu-engine", version = "0.74.1" } +nu-color-config = { path = "../nu-color-config", version = "0.74.1" } +nu-engine = { path = "../nu-engine", version = "0.74.1" } nu-glob = { path = "../nu-glob", version = "0.74.1" } -nu-json = { path = "../nu-json", version = "0.74.1" } -nu-parser = { path = "../nu-parser", version = "0.74.1" } -nu-path = { path = "../nu-path", version = "0.74.1" } -nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.74.1" } -nu-protocol = { path = "../nu-protocol", version = "0.74.1" } -nu-system = { path = "../nu-system", version = "0.74.1" } -nu-table = { path = "../nu-table", version = "0.74.1" } -nu-term-grid = { path = "../nu-term-grid", version = "0.74.1" } +nu-json = { path = "../nu-json", version = "0.74.1" } +nu-parser = { path = "../nu-parser", version = "0.74.1" } +nu-path = { path = "../nu-path", version = "0.74.1" } +nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.74.1" } +nu-protocol = { path = "../nu-protocol", version = "0.74.1" } +nu-system = { path = "../nu-system", version = "0.74.1" } +nu-table = { path = "../nu-table", version = "0.74.1" } +nu-term-grid = { path = "../nu-term-grid", version = "0.74.1" } nu-utils = { path = "../nu-utils", version = "0.74.1" } nu-explore = { path = "../nu-explore", version = "0.74.1" } nu-ansi-term = "0.46.0" @@ -30,11 +30,14 @@ num-format = { version = "0.4.3" } # Potential dependencies for extras alphanumeric-sort = "1.4.4" atty = "0.2.14" -base64 = "0.13.0" +base64 = "0.21.0" byteorder = "1.4.3" bytesize = "1.1.0" calamine = "0.19.1" -chrono = { version = "0.4.23", features = ["unstable-locales", "std"], default-features = false } +chrono = { version = "0.4.23", features = [ + "unstable-locales", + "std", +], default-features = false } chrono-humanize = "0.2.1" chrono-tz = "0.6.3" crossterm = "0.24.0" @@ -50,13 +53,15 @@ filetime = "0.2.15" fs_extra = "1.2.0" htmlescape = "0.3.1" ical = "0.7.0" -indexmap = { version="1.7", features=["serde-1"] } +indexmap = { version = "1.7", features = ["serde-1"] } indicatif = "0.17.2" Inflector = "0.11" is-root = "0.1.2" itertools = "0.10.0" log = "0.4.14" -lscolors = { version = "0.12.0", features = ["crossterm"], default-features = false } +lscolors = { version = "0.12.0", features = [ + "crossterm", +], default-features = false } md5 = { package = "md-5", version = "0.10.0" } mime = "0.3.16" mime_guess = "2.0.4" @@ -71,11 +76,11 @@ quick-xml = "0.25" rand = "0.8" rayon = "1.5.1" regex = "1.6.0" -reqwest = {version = "0.11", features = ["blocking", "json"] } +reqwest = { version = "0.11", features = ["blocking", "json"] } roxmltree = "0.16.0" rust-embed = "6.3.0" same-file = "1.0.6" -serde = { version="1.0.123", features=["derive"] } +serde = { version = "1.0.123", features = ["derive"] } serde_ini = "0.2.0" serde_urlencoded = "0.7.0" serde_yaml = "0.9.4" @@ -92,8 +97,8 @@ url = "2.2.1" percent-encoding = "2.2.0" uuid = { version = "1.2.2", features = ["v4"] } which = { version = "4.3.0", optional = true } -reedline = { version = "0.14.0", features = ["bashisms", "sqlite"]} -wax = { version = "0.5.0" } +reedline = { version = "0.14.0", features = ["bashisms", "sqlite"] } +wax = { version = "0.5.0" } rusqlite = { version = "0.28.0", features = ["bundled"], optional = true } sqlparser = { version = "0.23.0", features = ["serde"], optional = true } @@ -143,8 +148,8 @@ features = [ [target.'cfg(windows)'.dependencies.windows] version = "0.43.0" features = [ - "Win32_Foundation", - "Win32_Storage_FileSystem", + "Win32_Foundation", + "Win32_Storage_FileSystem", "Win32_System_SystemServices", ] @@ -153,17 +158,19 @@ trash-support = ["trash"] which-support = ["which"] plugin = ["nu-parser/plugin"] dataframe = ["polars", "num", "sqlparser"] -sqlite = ["rusqlite"] # TODO: given that rusqlite is included in reedline, should we just always include it? +sqlite = [ + "rusqlite", +] # TODO: given that rusqlite is included in reedline, should we just always include it? [build-dependencies] shadow-rs = { version = "0.16.1", default-features = false } [dev-dependencies] -nu-test-support = { path = "../nu-test-support", version = "0.74.1" } +nu-test-support = { path = "../nu-test-support", version = "0.74.1" } hamcrest2 = "0.3.0" dirs-next = "2.0.0" proptest = "1.0.0" quickcheck = "1.0.3" quickcheck_macros = "1.0.0" -rstest = {version = "0.15.0", default-features = false} +rstest = { version = "0.15.0", default-features = false } diff --git a/crates/nu-command/src/network/fetch.rs b/crates/nu-command/src/network/fetch.rs index adf5f3194..31a2c013b 100644 --- a/crates/nu-command/src/network/fetch.rs +++ b/crates/nu-command/src/network/fetch.rs @@ -1,19 +1,16 @@ -use base64::encode; +use base64::{alphabet, engine::general_purpose::PAD, engine::GeneralPurpose, Engine}; use nu_engine::CallExt; use nu_protocol::ast::Call; use nu_protocol::engine::{Command, EngineState, Stack}; use nu_protocol::util::BufferedReader; use nu_protocol::RawStream; - use nu_protocol::{ Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value, }; use reqwest::blocking::Response; - +use reqwest::StatusCode; use std::collections::HashMap; use std::io::BufReader; - -use reqwest::StatusCode; use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; @@ -165,10 +162,24 @@ fn helper( let timeout = args.timeout; let headers = args.headers; let raw = args.raw; + let base64_engine = GeneralPurpose::new(&alphabet::STANDARD, PAD); + let login = match (user, password) { - (Some(user), Some(password)) => Some(encode(format!("{}:{}", user, password))), - (Some(user), _) => Some(encode(format!("{}:", user))), - (_, Some(password)) => Some(encode(format!(":{}", password))), + (Some(user), Some(password)) => { + let mut enc_str = String::new(); + base64_engine.encode_string(&format!("{}:{}", user, password), &mut enc_str); + Some(enc_str) + } + (Some(user), _) => { + let mut enc_str = String::new(); + base64_engine.encode_string(&format!("{}:", user), &mut enc_str); + Some(enc_str) + } + (_, Some(password)) => { + let mut enc_str = String::new(); + base64_engine.encode_string(&format!(":{}", password), &mut enc_str); + Some(enc_str) + } _ => None, }; diff --git a/crates/nu-command/src/network/post.rs b/crates/nu-command/src/network/post.rs index de94fb78a..49674aa01 100644 --- a/crates/nu-command/src/network/post.rs +++ b/crates/nu-command/src/network/post.rs @@ -1,19 +1,18 @@ use crate::formats::value_to_json_value; -use base64::encode; +use base64::{alphabet, engine::general_purpose::PAD, engine::GeneralPurpose, Engine}; use nu_engine::CallExt; use nu_protocol::ast::Call; use nu_protocol::engine::{Command, EngineState, Stack}; use nu_protocol::util::BufferedReader; use nu_protocol::RawStream; -use reqwest::{blocking::Response, StatusCode}; -use std::path::PathBuf; -use std::str::FromStr; - use nu_protocol::{ Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value, }; +use reqwest::{blocking::Response, StatusCode}; use std::collections::HashMap; use std::io::BufReader; +use std::path::PathBuf; +use std::str::FromStr; #[derive(Clone)] pub struct SubCommand; @@ -186,9 +185,19 @@ fn helper( let headers = args.headers; let location = url; let raw = args.raw; + let base64_engine = GeneralPurpose::new(&alphabet::STANDARD, PAD); + let login = match (user, password) { - (Some(user), Some(password)) => Some(encode(&format!("{}:{}", user, password))), - (Some(user), _) => Some(encode(&format!("{}:", user))), + (Some(user), Some(password)) => { + let mut enc_str = String::new(); + base64_engine.encode_string(&format!("{}:{}", user, password), &mut enc_str); + Some(enc_str) + } + (Some(user), _) => { + let mut enc_str = String::new(); + base64_engine.encode_string(&format!("{}:", user), &mut enc_str); + Some(enc_str) + } _ => None, }; diff --git a/crates/nu-command/src/strings/encode_decode/base64.rs b/crates/nu-command/src/strings/encode_decode/base64.rs index 33908e70d..a29335e57 100644 --- a/crates/nu-command/src/strings/encode_decode/base64.rs +++ b/crates/nu-command/src/strings/encode_decode/base64.rs @@ -1,5 +1,8 @@ use crate::input_handler::{operate as general_operate, CmdArgument}; -use base64::{decode_config, encode_config}; +use base64::{ + alphabet, engine::general_purpose::NO_PAD, engine::general_purpose::PAD, + engine::GeneralPurpose, Engine, +}; use nu_engine::CallExt; use nu_protocol::ast::{Call, CellPath}; use nu_protocol::engine::{EngineState, Stack}; @@ -7,7 +10,7 @@ use nu_protocol::{PipelineData, ShellError, Span, Spanned, Value}; pub const CHARACTER_SET_DESC: &str = "specify the character rules for encoding the input.\n\ \tValid values are 'standard', 'standard-no-padding', 'url-safe', 'url-safe-no-padding',\ - 'binhex', 'bcrypt', 'crypt'"; + 'binhex', 'bcrypt', 'crypt', 'mutf7'"; #[derive(Clone)] pub struct Base64Config { @@ -78,14 +81,15 @@ fn action( let output_binary = args.binary; let config_character_set = &base64_config.character_set; - let base64_config_enum: base64::Config = match config_character_set.item.as_str() { - "standard" => base64::STANDARD, - "standard-no-padding" => base64::STANDARD_NO_PAD, - "url-safe" => base64::URL_SAFE, - "url-safe-no-padding" => base64::URL_SAFE_NO_PAD, - "binhex" => base64::BINHEX, - "bcrypt" => base64::BCRYPT, - "crypt" => base64::CRYPT, + let base64_engine: GeneralPurpose = match config_character_set.item.as_str() { + "standard" => GeneralPurpose::new(&alphabet::STANDARD, PAD), + "standard-no-padding" => GeneralPurpose::new(&alphabet::STANDARD, NO_PAD), + "url-safe" => GeneralPurpose::new(&alphabet::URL_SAFE, PAD), + "url-safe-no-padding" => GeneralPurpose::new(&alphabet::URL_SAFE, NO_PAD), + "bcrypt" => GeneralPurpose::new(&alphabet::BCRYPT, NO_PAD), + "binhex" => GeneralPurpose::new(&alphabet::BIN_HEX, NO_PAD), + "crypt" => GeneralPurpose::new(&alphabet::CRYPT, NO_PAD), + "mutf7" => GeneralPurpose::new(&alphabet::IMAP_MUTF7, NO_PAD), not_valid => return Value::Error { error:ShellError::GenericError( "value is not an accepted character set".to_string(), format!( @@ -102,7 +106,24 @@ fn action( Value::Error { .. } => input.clone(), Value::Binary { val, .. } => match base64_config.action_type { ActionType::Encode => { - Value::string(encode_config(val, base64_config_enum), command_span) + let mut enc_vec = Vec::new(); + enc_vec.resize(val.len() * 4 / 3 + 4, 0); + let bytes_written = match base64_engine.encode_slice(val, &mut enc_vec) { + Ok(bytes_written) => bytes_written, + Err(err) => { + return Value::Error { + error: ShellError::GenericError( + "Error encoding data".into(), + err.to_string(), + Some(Span::unknown()), + None, + Vec::new(), + ), + } + } + }; + enc_vec.truncate(bytes_written); + Value::string(std::str::from_utf8(&enc_vec).unwrap_or(""), command_span) } ActionType::Decode => Value::Error { error: ShellError::UnsupportedInput( @@ -120,7 +141,9 @@ fn action( } => { match base64_config.action_type { ActionType::Encode => { - Value::string(encode_config(val, base64_config_enum), command_span) + let mut enc_str = String::new(); + base64_engine.encode_string(val, &mut enc_str); + Value::string(enc_str, command_span) } ActionType::Decode => { @@ -128,7 +151,8 @@ fn action( let val = val.clone(); let val = val.replace("\r\n", "").replace('\n', ""); - match decode_config(&val, base64_config_enum) { + //match Engine::decode_string(&val, base64_engine) { + match base64_engine.decode(&val) { Ok(decoded_value) => { if output_binary { Value::binary(decoded_value, command_span)