From 56a9eab7ebfddf440aacb4e46c66b5d69e755887 Mon Sep 17 00:00:00 2001
From: mike <98623181+1Kinoti@users.noreply.github.com>
Date: Sun, 15 Jan 2023 18:03:57 +0300
Subject: [PATCH] Allow underscores in integers and floats (#7759)

# Description

This PR makes changes that allow underscores in numbers.

Example:
```nu
# allows underscores to be placed arbitrarily to enhance readability.
let pi = 3.1415_9265_3589_793

# works with integers
let num = 1_000_000_000_000
let fav_color = 0x68_9d_6a
```
---
 crates/nu-parser/src/parser.rs        | 73 +++++++++------------------
 crates/nu-parser/tests/test_parser.rs | 23 +++++++++
 src/tests/test_parser.rs              | 25 +++++++++
 3 files changed, 73 insertions(+), 48 deletions(-)

diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs
index 2a4898bff4..778a926b32 100644
--- a/crates/nu-parser/src/parser.rs
+++ b/crates/nu-parser/src/parser.rs
@@ -1329,12 +1329,19 @@ fn decode_with_base(s: &str, base: u32, digits_per_byte: usize) -> Result<Vec<u8
         .collect()
 }
 
+fn strip_underscores(token: &[u8]) -> String {
+    String::from_utf8_lossy(token)
+        .chars()
+        .filter(|c| *c != '_')
+        .collect()
+}
+
 pub fn parse_int(token: &[u8], span: Span) -> (Expression, Option<ParseError>) {
-    if let Some(token) = token.strip_prefix(b"0x") {
-        if let Ok(v) = i64::from_str_radix(&String::from_utf8_lossy(token), 16) {
+    fn extract_int(token: &str, span: Span, radix: u32) -> (Expression, Option<ParseError>) {
+        if let Ok(num) = i64::from_str_radix(token, radix) {
             (
                 Expression {
-                    expr: Expr::Int(v),
+                    expr: Expr::Int(num),
                     span,
                     ty: Type::Int,
                     custom_completion: None,
@@ -1351,52 +1358,20 @@ pub fn parse_int(token: &[u8], span: Span) -> (Expression, Option<ParseError>) {
                 )),
             )
         }
-    } else if let Some(token) = token.strip_prefix(b"0b") {
-        if let Ok(v) = i64::from_str_radix(&String::from_utf8_lossy(token), 2) {
-            (
-                Expression {
-                    expr: Expr::Int(v),
-                    span,
-                    ty: Type::Int,
-                    custom_completion: None,
-                },
-                None,
-            )
-        } else {
-            (
-                garbage(span),
-                Some(ParseError::Mismatch(
-                    "int".into(),
-                    "incompatible int".into(),
-                    span,
-                )),
-            )
-        }
-    } else if let Some(token) = token.strip_prefix(b"0o") {
-        if let Ok(v) = i64::from_str_radix(&String::from_utf8_lossy(token), 8) {
-            (
-                Expression {
-                    expr: Expr::Int(v),
-                    span,
-                    ty: Type::Int,
-                    custom_completion: None,
-                },
-                None,
-            )
-        } else {
-            (
-                garbage(span),
-                Some(ParseError::Mismatch(
-                    "int".into(),
-                    "incompatible int".into(),
-                    span,
-                )),
-            )
-        }
-    } else if let Ok(x) = String::from_utf8_lossy(token).parse::<i64>() {
+    }
+
+    let token = strip_underscores(token);
+
+    if let Some(num) = token.strip_prefix("0b") {
+        extract_int(num, span, 2)
+    } else if let Some(num) = token.strip_prefix("0o") {
+        extract_int(num, span, 8)
+    } else if let Some(num) = token.strip_prefix("0x") {
+        extract_int(num, span, 16)
+    } else if let Ok(num) = token.parse::<i64>() {
         (
             Expression {
-                expr: Expr::Int(x),
+                expr: Expr::Int(num),
                 span,
                 ty: Type::Int,
                 custom_completion: None,
@@ -1412,7 +1387,9 @@ pub fn parse_int(token: &[u8], span: Span) -> (Expression, Option<ParseError>) {
 }
 
 pub fn parse_float(token: &[u8], span: Span) -> (Expression, Option<ParseError>) {
-    if let Ok(x) = String::from_utf8_lossy(token).parse::<f64>() {
+    let token = strip_underscores(token);
+
+    if let Ok(x) = token.parse::<f64>() {
         (
             Expression {
                 expr: Expr::Float(x),
diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs
index 2a84bc47ec..3d6d6ea302 100644
--- a/crates/nu-parser/tests/test_parser.rs
+++ b/crates/nu-parser/tests/test_parser.rs
@@ -64,6 +64,29 @@ pub fn parse_int() {
     ))
 }
 
+#[test]
+pub fn parse_int_with_underscores() {
+    let engine_state = EngineState::new();
+    let mut working_set = StateWorkingSet::new(&engine_state);
+
+    let (block, err) = parse(&mut working_set, None, b"420_69_2023", true, &[]);
+
+    assert!(err.is_none());
+    assert_eq!(block.len(), 1);
+    let expressions = &block[0];
+    assert_eq!(expressions.len(), 1);
+    assert!(matches!(
+        expressions[0],
+        PipelineElement::Expression(
+            _,
+            Expression {
+                expr: Expr::Int(420692023),
+                ..
+            }
+        )
+    ))
+}
+
 #[test]
 pub fn parse_binary_with_hex_format() {
     let engine_state = EngineState::new();
diff --git a/src/tests/test_parser.rs b/src/tests/test_parser.rs
index 3fe54c42b5..7657715005 100644
--- a/src/tests/test_parser.rs
+++ b/src/tests/test_parser.rs
@@ -18,6 +18,31 @@ fn alias_1() -> TestResult {
     run_test("def foo [$x] { $x + 10 }; alias f = foo; f 100", "110")
 }
 
+#[test]
+fn ints_with_underscores() -> TestResult {
+    run_test("1_0000_0000_0000 + 10", "1000000000010")
+}
+
+#[test]
+fn floats_with_underscores() -> TestResult {
+    run_test("3.1415_9265_3589_793 * 2", "6.283185307179586")
+}
+
+#[test]
+fn bin_ints_with_underscores() -> TestResult {
+    run_test("0b_10100_11101_10010", "21426")
+}
+
+#[test]
+fn oct_ints_with_underscores() -> TestResult {
+    run_test("0o2443_6442_7652_0044", "90422533333028")
+}
+
+#[test]
+fn hex_ints_with_underscores() -> TestResult {
+    run_test("0x68__9d__6a", "6856042")
+}
+
 #[test]
 fn alias_2() -> TestResult {
     run_test(