Add hash command with base64 subcommand (#2769)

* WIP try testing hash command

Ensure test worked

fmt

WIP get it working for other types of base64

Use optional named arg

WIP

* rebased and refactored a little with encoding and decoding

Fix some typos

Add some more charactersets

refactor several args into the encoding config struct and fix character_set arg. It needs to match the field

Add main hash command so it can be found via help

Added tests for running the whole pipeline

* add test case to cover invalid character sets

* clippy and fmt
This commit is contained in:
Ryan Blecher 2020-11-30 12:47:35 -05:00 committed by GitHub
parent e299e76fcf
commit b193303aa3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 556 additions and 90 deletions

90
Cargo.lock generated
View File

@ -206,7 +206,7 @@ dependencies = [
"async-io",
"async-mutex",
"blocking 1.0.2",
"crossbeam-utils 0.8.0",
"crossbeam-utils 0.8.1",
"futures-channel",
"futures-core",
"futures-io",
@ -885,7 +885,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils 0.8.0",
"crossbeam-utils 0.8.1",
]
[[package]]
@ -906,8 +906,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch 0.9.0",
"crossbeam-utils 0.8.0",
"crossbeam-epoch 0.9.1",
"crossbeam-utils 0.8.1",
]
[[package]]
@ -921,21 +921,21 @@ dependencies = [
"crossbeam-utils 0.7.2",
"lazy_static 1.4.0",
"maybe-uninit",
"memoffset",
"memoffset 0.5.6",
"scopeguard",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.0"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0f606a85340376eef0d6d8fec399e6d4a544d648386c6645eb6d0653b27d9f"
checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
dependencies = [
"cfg-if 1.0.0",
"const_fn",
"crossbeam-utils 0.8.0",
"crossbeam-utils 0.8.1",
"lazy_static 1.4.0",
"memoffset",
"memoffset 0.6.1",
"scopeguard",
]
@ -973,13 +973,12 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec91540d98355f690a86367e566ecad2e9e579f230230eb7c21398372be73ea5"
checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d"
dependencies = [
"autocfg",
"cfg-if 1.0.0",
"const_fn",
"lazy_static 1.4.0",
]
@ -1063,9 +1062,9 @@ dependencies = [
[[package]]
name = "csv"
version = "1.1.4"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc4666154fd004af3fd6f1da2e81a96fd5a81927fe8ddb6ecc79e2aa6e138b54"
checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97"
dependencies = [
"bstr",
"csv-core",
@ -2575,9 +2574,9 @@ dependencies = [
[[package]]
name = "libnghttp2-sys"
version = "0.1.4+1.41.0"
version = "0.1.5+1.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03624ec6df166e79e139a2310ca213283d6b3c30810c54844f307086d4488df1"
checksum = "9657455ff47889b70ffd37c3e118e8cdd23fd1f9f3293a285f141070621c4c79"
dependencies = [
"cc",
"libc",
@ -2596,9 +2595,9 @@ dependencies = [
[[package]]
name = "libssh2-sys"
version = "0.2.19"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca46220853ba1c512fc82826d0834d87b06bcd3c2a42241b7de72f3d2fe17056"
checksum = "df40b13fe7ea1be9b9dffa365a51273816c345fc1811478b57ed7d964fbfc4ce"
dependencies = [
"cc",
"libc",
@ -2831,6 +2830,15 @@ dependencies = [
"autocfg",
]
[[package]]
name = "memoffset"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
dependencies = [
"autocfg",
]
[[package]]
name = "meval"
version = "0.2.0"
@ -2889,7 +2897,7 @@ dependencies = [
"kernel32-sys",
"libc",
"log 0.4.11",
"miow 0.2.1",
"miow 0.2.2",
"net2",
"slab 0.4.2",
"winapi 0.2.8",
@ -2921,9 +2929,9 @@ dependencies = [
[[package]]
name = "miow"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d"
dependencies = [
"kernel32-sys",
"net2",
@ -2985,9 +2993,9 @@ dependencies = [
[[package]]
name = "net2"
version = "0.2.35"
version = "0.2.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853"
checksum = "d7cf75f38f16cb05ea017784dc6dbfd354f76c223dba37701734c4f5a9337d02"
dependencies = [
"cfg-if 0.1.10",
"libc",
@ -3130,7 +3138,7 @@ dependencies = [
"ansi_term 0.12.1",
"async-recursion",
"async-trait",
"base64 0.12.3",
"base64 0.13.0",
"bigdecimal",
"byte-unit",
"bytes 0.5.6",
@ -3797,9 +3805,9 @@ checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
[[package]]
name = "onig"
version = "6.1.0"
version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a155d13862da85473665694f4c05d77fb96598bdceeaf696433c84ea9567e20"
checksum = "30b46fd9edbc018f0be4e366c24c46db44fac49cd01c039ae85308088b089dd5"
dependencies = [
"bitflags",
"lazy_static 1.4.0",
@ -3809,9 +3817,9 @@ dependencies = [
[[package]]
name = "onig_sys"
version = "69.5.1"
version = "69.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bff06597a6b17855040955cae613af000fc0bfc8ad49ea68b9479a74e59292d"
checksum = "ed063c96cf4c0f2e5d09324409d158b38a0a85a7b90fbd68c8cad75c495d5775"
dependencies = [
"cc",
"pkg-config",
@ -4549,7 +4557,7 @@ checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
dependencies = [
"crossbeam-channel 0.5.0",
"crossbeam-deque 0.8.0",
"crossbeam-utils 0.8.0",
"crossbeam-utils 0.8.1",
"lazy_static 1.4.0",
"num_cpus",
]
@ -4721,14 +4729,14 @@ dependencies = [
[[package]]
name = "rust-argon2"
version = "0.8.2"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19"
checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb"
dependencies = [
"base64 0.12.3",
"base64 0.13.0",
"blake2b_simd",
"constant_time_eq",
"crossbeam-utils 0.7.2",
"crossbeam-utils 0.8.1",
]
[[package]]
@ -5393,9 +5401,9 @@ dependencies = [
[[package]]
name = "syn"
version = "1.0.48"
version = "1.0.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac"
checksum = "6c1e438504729046a5cfae47f97c30d6d083c7d91d94603efdae3477fc070d4c"
dependencies = [
"proc-macro2",
"quote",
@ -5892,13 +5900,13 @@ checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860"
[[package]]
name = "tracing"
version = "0.1.21"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27"
checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3"
dependencies = [
"cfg-if 0.1.10",
"cfg-if 1.0.0",
"log 0.4.11",
"pin-project-lite 0.1.11",
"pin-project-lite 0.2.0",
"tracing-core",
]
@ -6014,9 +6022,9 @@ dependencies = [
[[package]]
name = "unicode-segmentation"
version = "1.7.0"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db8716a166f290ff49dabc18b44aa407cb7c6dbe1aa0971b44b8a24b0ca35aae"
checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
[[package]]
name = "unicode-width"

View File

@ -24,7 +24,7 @@ nu-value-ext = {version = "0.23.0", path = "../nu-value-ext"}
ansi_term = "0.12.1"
async-recursion = "0.3.1"
async-trait = "0.1.40"
base64 = "0.12.3"
base64 = "0.13.0"
bigdecimal = {version = "0.2.0", features = ["serde"]}
byte-unit = "4.0.9"
bytes = "0.5.6"

View File

@ -120,6 +120,8 @@ pub fn create_default_context(interactive: bool) -> Result<EvaluationContext, Bo
whole_stream_command(Autoview),
whole_stream_command(Table),
// Text manipulation
whole_stream_command(Hash),
whole_stream_command(HashBase64),
whole_stream_command(Split),
whole_stream_command(SplitColumn),
whole_stream_command(SplitRow),

View File

@ -62,6 +62,7 @@ pub(crate) mod from_yaml;
pub(crate) mod get;
pub(crate) mod group_by;
pub(crate) mod group_by_date;
pub(crate) mod hash_;
pub(crate) mod headers;
pub(crate) mod help;
pub(crate) mod histogram;
@ -199,6 +200,7 @@ pub(crate) use from_yaml::FromYML;
pub(crate) use get::Get;
pub(crate) use group_by::Command as GroupBy;
pub(crate) use group_by_date::GroupByDate;
pub(crate) use hash_::{Hash, HashBase64};
pub(crate) use headers::Headers;
pub(crate) use help::Help;
pub(crate) use histogram::Histogram;

View File

@ -0,0 +1,314 @@
use crate::commands::WholeStreamCommand;
use crate::prelude::*;
use nu_errors::ShellError;
use nu_protocol::ShellTypeName;
use nu_protocol::{
ColumnPath, Primitive, ReturnSuccess, Signature, SyntaxShape, UntaggedValue, Value,
};
use nu_source::{Tag, Tagged};
use base64::{decode_config, encode_config};
#[derive(Deserialize)]
pub struct Arguments {
pub character_set: Option<Tagged<String>>,
pub encode: Tagged<bool>,
pub decode: Tagged<bool>,
pub rest: Vec<ColumnPath>,
}
#[derive(Clone)]
pub struct Base64Config {
pub character_set: String,
pub action_type: ActionType,
}
#[derive(Clone, Copy, PartialEq)]
pub enum ActionType {
Encode,
Decode,
}
pub struct SubCommand;
#[async_trait]
impl WholeStreamCommand for SubCommand {
fn name(&self) -> &str {
"hash base64"
}
fn signature(&self) -> Signature {
Signature::build("hash base64")
.named(
"character_set",
SyntaxShape::String,
"specify the character rules for encoding the input.\n\
\tValid values are 'standard', 'standard-no-padding', 'url-safe', 'url-safe-no-padding',\
'binhex', 'bcrypt', 'crypt'",
Some('c'),
)
.switch(
"encode",
"encode the input as base64. This is the default behavior if not specified.",
Some('e')
)
.switch(
"decode",
"decode the input from base64",
Some('d'))
.rest(
SyntaxShape::ColumnPath,
"optionally base64 encode / decode data by column paths",
)
}
fn usage(&self) -> &str {
"base64 encode or decode a value"
}
async fn run(
&self,
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
operate(args, registry).await
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Base64 encode a string with default settings",
example: "echo 'username:password' | hash base64",
result: Some(vec![
UntaggedValue::string("dXNlcm5hbWU6cGFzc3dvcmQ=").into_untagged_value()
]),
},
Example {
description: "Base64 encode a string with the binhex character set",
example: "echo 'username:password' | hash base64 --character_set binhex --encode",
result: Some(vec![
UntaggedValue::string("F@0NEPjJD97kE'&bEhFZEP3").into_untagged_value()
]),
},
Example {
description: "Base64 decode a value",
example: "echo 'dXNlcm5hbWU6cGFzc3dvcmQ=' | hash base64 --decode",
result: Some(vec![
UntaggedValue::string("username:password").into_untagged_value()
]),
},
]
}
}
async fn operate(
args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
let registry = registry.clone();
let name_tag = &args.call_info.name_tag.clone();
let (
Arguments {
encode,
decode,
character_set,
rest,
},
input,
) = args.process(&registry).await?;
if encode.item && decode.item {
return Ok(OutputStream::one(Err(ShellError::labeled_error(
"only one of --decode and --encode flags can be used",
"conflicting flags",
name_tag,
))));
}
// Default the action to be encoding if no flags are specified.
let action_type = if *decode.item() {
ActionType::Decode
} else {
ActionType::Encode
};
// Default the character set to standard if the argument is not specified.
let character_set = match character_set {
Some(inner_tag) => inner_tag.item().to_string(),
None => "standard".to_string(),
};
let encoding_config = Base64Config {
character_set,
action_type,
};
let column_paths: Vec<_> = rest;
Ok(input
.map(move |v| {
if column_paths.is_empty() {
ReturnSuccess::value(action(&v, &encoding_config, v.tag())?)
} else {
let mut ret = v;
for path in &column_paths {
let config = encoding_config.clone();
ret = ret.swap_data_by_column_path(
path,
Box::new(move |old| action(old, &config, old.tag())),
)?;
}
ReturnSuccess::value(ret)
}
})
.to_output_stream())
}
fn action(
input: &Value,
base64_config: &Base64Config,
tag: impl Into<Tag>,
) -> Result<Value, ShellError> {
match &input.value {
UntaggedValue::Primitive(Primitive::Line(s))
| UntaggedValue::Primitive(Primitive::String(s)) => {
let base64_config_enum: base64::Config = if &base64_config.character_set == "standard" {
base64::STANDARD
} else if &base64_config.character_set == "standard-no-padding" {
base64::STANDARD_NO_PAD
} else if &base64_config.character_set == "url-safe" {
base64::URL_SAFE
} else if &base64_config.character_set == "url-safe-no-padding" {
base64::URL_SAFE_NO_PAD
} else if &base64_config.character_set == "binhex" {
base64::BINHEX
} else if &base64_config.character_set == "bcrypt" {
base64::BCRYPT
} else if &base64_config.character_set == "crypt" {
base64::CRYPT
} else {
return Err(ShellError::labeled_error(
"value is not an accepted character set",
format!(
"{} is not a valid character-set.\nPlease use `help hash base64` to see a list of valid character sets.",
&base64_config.character_set
),
tag.into().span,
));
};
match base64_config.action_type {
ActionType::Encode => Ok(UntaggedValue::string(encode_config(
&s,
base64_config_enum,
))
.into_value(tag)),
ActionType::Decode => {
let decode_result = decode_config(&s, base64_config_enum);
match decode_result {
Ok(decoded_value) => Ok(UntaggedValue::string(
std::string::String::from_utf8_lossy(&decoded_value),
)
.into_value(tag)),
Err(_) => Err(ShellError::labeled_error(
"value could not be base64 decoded",
format!(
"invalid base64 input for character set {}",
&base64_config.character_set
),
tag.into().span,
)),
}
}
}
}
other => {
let got = format!("got {}", other.type_name());
Err(ShellError::labeled_error(
"value is not string",
got,
tag.into().span,
))
}
}
}
#[cfg(test)]
mod tests {
use super::{action, ActionType, Base64Config};
use nu_protocol::UntaggedValue;
use nu_source::Tag;
use nu_test_support::value::string;
#[test]
fn base64_encode_standard() {
let word = string("username:password");
let expected = UntaggedValue::string("dXNlcm5hbWU6cGFzc3dvcmQ=").into_untagged_value();
let actual = action(
&word,
&Base64Config {
character_set: "standard".to_string(),
action_type: ActionType::Encode,
},
Tag::unknown(),
)
.unwrap();
assert_eq!(actual, expected);
}
#[test]
fn base64_encode_standard_no_padding() {
let word = string("username:password");
let expected = UntaggedValue::string("dXNlcm5hbWU6cGFzc3dvcmQ").into_untagged_value();
let actual = action(
&word,
&Base64Config {
character_set: "standard-no-padding".to_string(),
action_type: ActionType::Encode,
},
Tag::unknown(),
)
.unwrap();
assert_eq!(actual, expected);
}
#[test]
fn base64_encode_url_safe() {
let word = string("this is for url");
let expected = UntaggedValue::string("dGhpcyBpcyBmb3IgdXJs").into_untagged_value();
let actual = action(
&word,
&Base64Config {
character_set: "url-safe".to_string(),
action_type: ActionType::Encode,
},
Tag::unknown(),
)
.unwrap();
assert_eq!(actual, expected);
}
#[test]
fn base64_decode_binhex() {
let word = string("A5\"KC9jRB@IIF'8bF!");
let expected = UntaggedValue::string("a binhex test").into_untagged_value();
let actual = action(
&word,
&Base64Config {
character_set: "binhex".to_string(),
action_type: ActionType::Decode,
},
Tag::unknown(),
)
.unwrap();
assert_eq!(actual, expected);
}
}

View File

@ -0,0 +1,50 @@
use crate::commands::WholeStreamCommand;
use crate::prelude::*;
use nu_errors::ShellError;
use nu_protocol::{ReturnSuccess, Signature, SyntaxShape, UntaggedValue};
pub struct Command;
#[async_trait]
impl WholeStreamCommand for Command {
fn name(&self) -> &str {
"hash"
}
fn signature(&self) -> Signature {
Signature::build("hash").rest(
SyntaxShape::ColumnPath,
"optionally convert by column paths",
)
}
fn usage(&self) -> &str {
"Apply hash function."
}
async fn run(
&self,
_args: CommandArgs,
registry: &CommandRegistry,
) -> Result<OutputStream, ShellError> {
let registry = registry.clone();
Ok(OutputStream::one(ReturnSuccess::value(
UntaggedValue::string(crate::commands::help::get_help(&Command, &registry))
.into_value(Tag::unknown()),
)))
}
}
#[cfg(test)]
mod tests {
use super::Command;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test as test_examples;
Ok(test_examples(Command {})?)
}
}

View File

@ -0,0 +1,5 @@
mod base64_;
mod command;
pub use base64_::SubCommand as HashBase64;
pub use command::Command as Hash;

View File

@ -765,13 +765,11 @@ impl VarSyntaxShapeDeductor {
)],
)?;
}
Operator::In | Operator::NotIn => {
match var_side {
Operator::In | Operator::NotIn => match var_side {
BinarySide::Left => match &expr.expr {
Expression::List(list) => {
if !list.is_empty() {
let shapes_in_list = self
.get_shapes_in_list_or_insert_dependency(
let shapes_in_list = self.get_shapes_in_list_or_insert_dependency(
var,
bin_spanned,
&list,
@ -806,7 +804,9 @@ impl VarSyntaxShapeDeductor {
| Expression::Command
| Expression::Invocation(_)
| Expression::Boolean(_)
| Expression::Garbage => {unreachable!("Parser should have rejected code. In only applicable with rhs of type List")}
| Expression::Garbage => {
unreachable!("Parser should have rejected code. In only applicable with rhs of type List")
}
},
BinarySide::Right => {
self.checked_insert(
@ -817,8 +817,7 @@ impl VarSyntaxShapeDeductor {
),
)?;
}
}
}
},
Operator::Modulo => {
self.checked_insert(
var,

View File

@ -0,0 +1,85 @@
use nu_test_support::{nu, pipeline};
#[test]
fn base64_defaults_to_encoding_with_standard_character_type() {
let actual = nu!(
cwd: ".", pipeline(
r#"
echo 'username:password' | hash base64
"#
)
);
assert_eq!(actual.out, "dXNlcm5hbWU6cGFzc3dvcmQ=");
}
#[test]
fn base64_encode_characterset_binhex() {
let actual = nu!(
cwd: ".", pipeline(
r#"
echo 'username:password' | hash base64 --character_set binhex --encode
"#
)
);
assert_eq!(actual.out, "F@0NEPjJD97kE\'&bEhFZEP3");
}
#[test]
fn error_when_invalid_character_set_given() {
let actual = nu!(
cwd: ".", pipeline(
r#"
echo 'username:password' | hash base64 --character_set 'this is invalid' --encode
"#
)
);
assert!(actual
.err
.contains("this is invalid is not a valid character-set"));
}
#[test]
fn base64_decode_characterset_binhex() {
let actual = nu!(
cwd: ".", pipeline(
r#"
echo "F@0NEPjJD97kE'&bEhFZEP3" | hash base64 --character_set binhex --decode
"#
)
);
assert_eq!(actual.out, "username:password");
}
#[test]
fn error_invalid_decode_value() {
let actual = nu!(
cwd: ".", pipeline(
r#"
echo "this should not be a valid encoded value" | hash base64 --character_set url-safe --decode
"#
)
);
assert!(actual
.err
.contains("invalid base64 input for character set url-safe"));
}
#[test]
fn error_use_both_flags() {
let actual = nu!(
cwd: ".", pipeline(
r#"
echo 'username:password' | hash base64 --encode --decode
"#
)
);
assert!(actual
.err
.contains("only one of --decode and --encode flags can be used"));
}

View File

@ -20,6 +20,7 @@ mod flatten;
mod format;
mod get;
mod group_by;
mod hash_;
mod headers;
mod histogram;
mod insert;