Add binary literals (#4680)

crates/nu-cli/src/syntax_highlight.rs

@@ -55,6 +55,13 @@ impl Highlighter for NuHighlighter {
                     get_shape_color(shape.1.to_string(), &self.config),
                     next_token,
                 )),
+                FlatShape::Binary => {
+                    // nushell ?
+                    output.push((
+                        get_shape_color(shape.1.to_string(), &self.config),
+                        next_token,
+                    ))
+                }
                 FlatShape::Bool => {
                     // nushell ?
                     output.push((

crates/nu-color-config/src/shape_color.rs

@@ -10,6 +10,7 @@ pub fn get_shape_color(shape: String, conf: &Config) -> Style {
         },
         None => match shape.as_ref() {
             "shape_garbage" => Style::new().fg(Color::White).on(Color::Red).bold(),
+            "shape_binary" => Style::new().fg(Color::Purple).bold(),
             "shape_bool" => Style::new().fg(Color::LightCyan),
             "shape_int" => Style::new().fg(Color::Purple).bold(),
             "shape_float" => Style::new().fg(Color::Purple).bold(),

crates/nu-command/src/formats/from/nuon.rs

@@ -201,6 +201,7 @@ fn convert_to_value(
             "blocks not supported in nuon".into(),
             expr.span,
         )),
+        Expr::Binary(val) => Ok(Value::Binary { val, span }),
         Expr::Bool(val) => Ok(Value::Bool { val, span }),
         Expr::Call(..) => Err(ShellError::OutsideSpannedLabeledError(
             original_text.to_string(),

crates/nu-command/src/formats/to/nuon.rs

@@ -1,3 +1,4 @@
+use core::fmt::Write;
 use nu_engine::get_columns;
 use nu_protocol::ast::{Call, RangeInclusion};
 use nu_protocol::engine::{Command, EngineState, Stack};
@@ -46,10 +47,18 @@ impl Command for ToNuon {
 
 fn value_to_string(v: &Value, span: Span) -> Result<String, ShellError> {
     match v {
-        Value::Binary { .. } => Err(ShellError::UnsupportedInput(
+        Value::Binary { val, .. } => {
-            "binary not supported".into(),
+            let mut s = String::with_capacity(2 * val.len());
+            for byte in val {
+                if write!(s, "{:02X}", byte).is_err() {
+                    return Err(ShellError::UnsupportedInput(
+                        "binary could not translate to string".into(),
                         span,
-        )),
+                    ));
+                }
+            }
+            Ok(format!("0x[{}]", s))
+        }
         Value::Block { .. } => Err(ShellError::UnsupportedInput(
             "block not supported".into(),
             span,

crates/nu-command/tests/format_conversions/nuon.rs

@@ -102,3 +102,27 @@ fn to_nuon_records() {
 
     assert_eq!(actual.out, "true");
 }
+
+#[test]
+fn binary_to() {
+    let actual = nu!(
+        cwd: "tests/fixtures/formats", pipeline(
+        r#"
+            0x[ab cd ef] | to nuon
+        "#
+    ));
+
+    assert_eq!(actual.out, "0x[ABCDEF]");
+}
+
+#[test]
+fn binary_roundtrip() {
+    let actual = nu!(
+        cwd: "tests/fixtures/formats", pipeline(
+        r#"
+            "0x[1f ff]" | from nuon | to nuon
+        "#
+    ));
+
+    assert_eq!(actual.out, "0x[1FFF]");
+}

crates/nu-engine/src/eval.rs

@@ -227,6 +227,10 @@ pub fn eval_expression(
             val: *f,
             span: expr.span,
         }),
+        Expr::Binary(b) => Ok(Value::Binary {
+            val: b.clone(),
+            span: expr.span,
+        }),
         Expr::ValueWithUnit(e, unit) => match eval_expression(engine_state, stack, e)? {
             Value::Int { val, .. } => Ok(compute(val, unit.item, unit.span)),
             x => Err(ShellError::CantConvert(

crates/nu-parser/src/flatten.rs

@@ -7,6 +7,7 @@ pub enum FlatShape {
     Garbage,
     Nothing,
     Bool,
+    Binary,
     Int,
     Float,
     Range,
@@ -35,6 +36,7 @@ impl Display for FlatShape {
         match self {
             FlatShape::Garbage => write!(f, "shape_garbage"),
             FlatShape::Nothing => write!(f, "shape_nothing"),
+            FlatShape::Binary => write!(f, "shape_binary"),
             FlatShape::Bool => write!(f, "shape_bool"),
             FlatShape::Int => write!(f, "shape_int"),
             FlatShape::Float => write!(f, "shape_float"),
@@ -189,6 +191,9 @@ pub fn flatten_expression(
         Expr::DateTime(_) => {
             vec![(expr.span, FlatShape::DateTime)]
         }
+        Expr::Binary(_) => {
+            vec![(expr.span, FlatShape::Binary)]
+        }
         Expr::Int(_) => {
             vec![(expr.span, FlatShape::Int)]
         }

crates/nu-parser/src/parser.rs

@@ -21,7 +21,10 @@ use crate::parse_keywords::{
 };
 
 use log::trace;
-use std::collections::{HashMap, HashSet};
+use std::{
+    collections::{HashMap, HashSet},
+    num::ParseIntError,
+};
 
 #[cfg(feature = "plugin")]
 use crate::parse_keywords::parse_register;
@@ -964,6 +967,85 @@ pub fn parse_call(
     }
 }
 
+pub fn parse_binary(
+    working_set: &mut StateWorkingSet,
+    span: Span,
+) -> (Expression, Option<ParseError>) {
+    pub fn decode_hex(s: &str) -> Result<Vec<u8>, ParseIntError> {
+        (0..s.len())
+            .step_by(2)
+            .map(|i| u8::from_str_radix(&s[i..i + 2], 16))
+            .collect()
+    }
+
+    let token = working_set.get_span_contents(span);
+
+    if let Some(token) = token.strip_prefix(b"0x[") {
+        if let Some(token) = token.strip_suffix(b"]") {
+            let (lexed, err) = lex(token, span.start + 3, &[b',', b'\r', b'\n'], &[], true);
+
+            let mut hex_value = vec![];
+            for token in lexed {
+                match token.contents {
+                    TokenContents::Item => {
+                        let contents = working_set.get_span_contents(token.span);
+
+                        hex_value.extend_from_slice(contents);
+                    }
+                    TokenContents::Pipe => {
+                        return (
+                            garbage(span),
+                            Some(ParseError::Expected("binary".into(), span)),
+                        );
+                    }
+                    TokenContents::Comment | TokenContents::Semicolon | TokenContents::Eol => {}
+                }
+            }
+
+            if hex_value.len() % 2 != 0 {
+                return (
+                    garbage(span),
+                    Some(ParseError::IncorrectValue(
+                        "incomplete binary".into(),
+                        span,
+                        "number of binary digits needs to be a multiple of 2".into(),
+                    )),
+                );
+            }
+
+            let str = String::from_utf8_lossy(&hex_value).to_string();
+
+            match decode_hex(&str) {
+                Ok(v) => {
+                    return (
+                        Expression {
+                            expr: Expr::Binary(v),
+                            span,
+                            ty: Type::Binary,
+                            custom_completion: None,
+                        },
+                        err,
+                    )
+                }
+                Err(x) => {
+                    return (
+                        garbage(span),
+                        Some(ParseError::IncorrectValue(
+                            "not a binary value".into(),
+                            span,
+                            x.to_string(),
+                        )),
+                    )
+                }
+            }
+        }
+    }
+    (
+        garbage(span),
+        Some(ParseError::Expected("binary".into(), span)),
+    )
+}
+
 pub fn parse_int(token: &[u8], span: Span) -> (Expression, Option<ParseError>) {
     if let Some(token) = token.strip_prefix(b"0x") {
         if let Ok(v) = i64::from_str_radix(&String::from_utf8_lossy(token), 16) {
@@ -2132,6 +2214,7 @@ pub fn parse_shape_name(
 ) -> (SyntaxShape, Option<ParseError>) {
     let result = match bytes {
         b"any" => SyntaxShape::Any,
+        b"binary" => SyntaxShape::Binary,
         b"block" => SyntaxShape::Block(None), //FIXME: Blocks should have known output types
         b"cell-path" => SyntaxShape::CellPath,
         b"duration" => SyntaxShape::Duration,
@@ -3231,6 +3314,7 @@ pub fn parse_value(
         SyntaxShape::Filepath => parse_filepath(working_set, span),
         SyntaxShape::GlobPattern => parse_glob_pattern(working_set, span),
         SyntaxShape::String => parse_string(working_set, span),
+        SyntaxShape::Binary => parse_binary(working_set, span),
         SyntaxShape::Block(_) => {
             if bytes.starts_with(b"{") {
                 trace!("parsing value as a block expression");
@@ -3320,6 +3404,7 @@ pub fn parse_value(
                 parse_full_cell_path(working_set, None, span)
             } else {
                 let shapes = [
+                    SyntaxShape::Binary,
                     SyntaxShape::Int,
                     SyntaxShape::Number,
                     SyntaxShape::Range,
@@ -4020,6 +4105,7 @@ pub fn discover_captures_in_expr(
                 }
             }
         }
+        Expr::Binary(_) => {}
         Expr::Bool(_) => {}
         Expr::Call(call) => {
             let decl = working_set.get_decl(call.decl_id);

crates/nu-protocol/src/ast/expr.rs

@@ -8,6 +8,7 @@ pub enum Expr {
     Bool(bool),
     Int(i64),
     Float(f64),
+    Binary(Vec<u8>),
     Range(
         Option<Box<Expression>>, // from
         Option<Box<Expression>>, // next value after "from"

crates/nu-protocol/src/ast/expression.rs

@@ -125,6 +125,7 @@ impl Expression {
                     false
                 }
             }
+            Expr::Binary(_) => false,
             Expr::Bool(_) => false,
             Expr::Call(call) => {
                 for positional in &call.positional {
@@ -290,6 +291,7 @@ impl Expression {
                     .map(|x| if *x != IN_VARIABLE_ID { *x } else { new_var_id })
                     .collect();
             }
+            Expr::Binary(_) => {}
             Expr::Bool(_) => {}
             Expr::Call(call) => {
                 for positional in &mut call.positional {
@@ -430,6 +432,7 @@ impl Expression {
 
                 *block_id = working_set.add_block(block);
             }
+            Expr::Binary(_) => {}
             Expr::Bool(_) => {}
             Expr::Call(call) => {
                 if replaced.contains_span(call.head) {

crates/nu-protocol/src/syntax_shape.rs

@@ -40,6 +40,9 @@ pub enum SyntaxShape {
     /// A module path pattern used for imports
     ImportPattern,
 
+    /// A binary literal
+    Binary,
+
     /// A block is allowed, eg `{start this thing}`
     Block(Option<Vec<SyntaxShape>>),
 
@@ -95,6 +98,7 @@ impl SyntaxShape {
         match self {
             SyntaxShape::Any => Type::Unknown,
             SyntaxShape::Block(_) => Type::Block,
+            SyntaxShape::Binary => Type::Binary,
             SyntaxShape::CellPath => Type::Unknown,
             SyntaxShape::Custom(custom, _) => custom.to_type(),
             SyntaxShape::DateTime => Type::Date,
@@ -144,6 +148,7 @@ impl Display for SyntaxShape {
             SyntaxShape::GlobPattern => write!(f, "glob"),
             SyntaxShape::ImportPattern => write!(f, "import"),
             SyntaxShape::Block(_) => write!(f, "block"),
+            SyntaxShape::Binary => write!(f, "binary"),
             SyntaxShape::Table => write!(f, "table"),
             SyntaxShape::List(x) => write!(f, "list<{}>", x),
             SyntaxShape::Record => write!(f, "record"),

docs/commands/complete.md

@@ -0,0 +1,18 @@
+---
+title: complete
+layout: command
+version: 0.59.0
+---
+
+Complete the external piped in, collecting outputs and exit code
+
+## Signature
+
+```> complete ```
+
+## Examples
+
+Run the external completion
+```shell
+> ^external arg1 | complete
+```

docs/commands/dfr_describe.md

@@ -8,7 +8,11 @@ Describes dataframes numeric columns
 
 ## Signature
 
-```> dfr describe ```
+```> dfr describe --quantiles```
+
+## Parameters
+
+ -  `--quantiles {table}`: optional quantiles for describe
 
 ## Examples
 

docs/commands/find.md

@@ -8,12 +8,17 @@ Searches terms in the input or for elements of the input that satisfies the pred
 
 ## Signature
 
-```> find ...rest --predicate```
+```> find ...rest --predicate --regex --insensitive --multiline --dotall --invert```
 
 ## Parameters
 
  -  `...rest`: terms to search
  -  `--predicate {block}`: the predicate to satisfy
+ -  `--regex {string}`: regex to match with
+ -  `--insensitive`: case-insensitive search for regex (?i)
+ -  `--multiline`: multi-line mode: ^ and $ match begin/end of line for regex (?m)
+ -  `--dotall`: dotall mode: allow a dot . to match newline character \n for regex (?s)
+ -  `--invert`: invert the match
 
 ## Examples
 
@@ -37,12 +42,27 @@ Search a char in a list of string
 > [moe larry curly] | find l
 ```
 
-Find the first odd value
+Find odd values
 ```shell
-> echo [2 4 3 6 5 8] | find --predicate { |it| ($it mod 2) == 1 }
+> [2 4 3 6 5 8] | find --predicate { |it| ($it mod 2) == 1 }
 ```
 
 Find if a service is not running
 ```shell
-> echo [[version patch]; [0.1.0 $false] [0.1.1 $true] [0.2.0 $false]] | find -p { |it| $it.patch }
+> [[version patch]; [0.1.0 $false] [0.1.1 $true] [0.2.0 $false]] | find -p { |it| $it.patch }
+```
+
+Find using regex
+```shell
+> [abc bde arc abf] | find --regex "ab"
+```
+
+Find using regex case insensitive
+```shell
+> [aBc bde Arc abf] | find --regex "ab" -i
+```
+
+Find value in records
+```shell
+> [[version name]; [0.1.0 nushell] [0.1.1 fish] [0.2.0 zsh]] | find -r "nu"
 ```

src/default_config.nu

@@ -134,6 +134,7 @@ let default_theme = {
 
     # shapes are used to change the cli syntax highlighting
     shape_garbage: { fg: "#FFFFFF" bg: "#FF0000" attr: b}
+    shape_binary: purple_bold
     shape_bool: light_cyan
     shape_int: purple_bold
     shape_float: purple_bold