Fix bits ror/bits rol implementation (#13673)

# Description
`bits rol` and `bits ror` were both undefined for the full byte rotates
and panicked when exceeding the byte rotation range.
`bits ror` further more produced nonsensical results by pulling bits
from the following byte instead of the preceding byte.

Those bugs are now fixed

# User-Facing Changes
Sound Nushell `IncorrectValue` error when exceeding the available bits

# Tests + Formatting
Added the necessary tests

crates/nu-cmd-extra/src/extra/bits/rotate_left.rs

@@ -1,11 +1,10 @@
 use super::{get_input_num_type, get_number_bytes, InputNumType, NumberBytes};
-use itertools::Itertools;
 use nu_cmd_base::input_handler::{operate, CmdArgument};
 use nu_engine::command_prelude::*;
 
 struct Arguments {
     signed: bool,
-    bits: usize,
+    bits: Spanned<usize>,
     number_size: NumberBytes,
 }
 
@@ -69,7 +68,7 @@ impl Command for BitsRol {
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let head = call.head;
-        let bits: usize = call.req(engine_state, stack, 0)?;
+        let bits = call.req(engine_state, stack, 0)?;
         let signed = call.has_flag(engine_state, stack, "signed")?;
         let number_bytes: Option<Spanned<usize>> =
             call.get_flag(engine_state, stack, "number-bytes")?;
@@ -119,6 +118,8 @@ fn action(input: &Value, args: &Arguments, span: Span) -> Value {
         number_size,
         bits,
     } = *args;
+    let bits_span = bits.span;
+    let bits = bits.item;
 
     match input {
         Value::Int { val, .. } => {
@@ -127,6 +128,19 @@ fn action(input: &Value, args: &Arguments, span: Span) -> Value {
             let bits = bits as u32;
             let input_num_type = get_input_num_type(val, signed, number_size);
 
+            if bits > input_num_type.num_bits() {
+                return Value::error(
+                    ShellError::IncorrectValue {
+                        msg: format!(
+                            "Trying to rotate by more than the available bits ({})",
+                            input_num_type.num_bits()
+                        ),
+                        val_span: bits_span,
+                        call_span: span,
+                    },
+                    span,
+                );
+            }
             let int = match input_num_type {
                 One => (val as u8).rotate_left(bits) as i64,
                 Two => (val as u16).rotate_left(bits) as i64,
@@ -157,16 +171,28 @@ fn action(input: &Value, args: &Arguments, span: Span) -> Value {
             Value::int(int, span)
         }
         Value::Binary { val, .. } => {
+            let len = val.len();
+            if bits > len * 8 {
+                return Value::error(
+                    ShellError::IncorrectValue {
+                        msg: format!(
+                            "Trying to rotate by more than the available bits ({})",
+                            len * 8
+                        ),
+                        val_span: bits_span,
+                        call_span: span,
+                    },
+                    span,
+                );
+            }
             let byte_shift = bits / 8;
             let bit_rotate = bits % 8;
 
-            let mut bytes = val
+            let bytes = if bit_rotate == 0 {
-                .iter()
+                rotate_bytes_left(val, byte_shift)
-                .copied()
+            } else {
-                .circular_tuple_windows::<(u8, u8)>()
+                rotate_bytes_and_bits_left(val, byte_shift, bit_rotate)
-                .map(|(lhs, rhs)| (lhs << bit_rotate) | (rhs >> (8 - bit_rotate)))
+            };
-                .collect::<Vec<u8>>();
-            bytes.rotate_left(byte_shift);
 
             Value::binary(bytes, span)
         }
@@ -184,6 +210,34 @@ fn action(input: &Value, args: &Arguments, span: Span) -> Value {
     }
 }
 
+fn rotate_bytes_left(data: &[u8], byte_shift: usize) -> Vec<u8> {
+    let len = data.len();
+    let mut output = vec![0; len];
+    output[..len - byte_shift].copy_from_slice(&data[byte_shift..]);
+    output[len - byte_shift..].copy_from_slice(&data[..byte_shift]);
+    output
+}
+
+fn rotate_bytes_and_bits_left(data: &[u8], byte_shift: usize, bit_shift: usize) -> Vec<u8> {
+    debug_assert!(byte_shift < data.len());
+    debug_assert!(
+        (1..8).contains(&bit_shift),
+        "Bit shifts of 0 can't be handled by this impl and everything else should be part of the byteshift");
+    let mut bytes = Vec::with_capacity(data.len());
+    let mut next_index = byte_shift;
+    for _ in 0..data.len() {
+        let curr_byte = data[next_index];
+        next_index += 1;
+        if next_index == data.len() {
+            next_index = 0;
+        }
+        let next_byte = data[next_index];
+        let new_byte = (curr_byte << bit_shift) | (next_byte >> (8 - bit_shift));
+        bytes.push(new_byte);
+    }
+    bytes
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

crates/nu-cmd-extra/src/extra/bits/rotate_right.rs

@@ -1,11 +1,10 @@
 use super::{get_input_num_type, get_number_bytes, InputNumType, NumberBytes};
-use itertools::Itertools;
 use nu_cmd_base::input_handler::{operate, CmdArgument};
 use nu_engine::command_prelude::*;
 
 struct Arguments {
     signed: bool,
-    bits: usize,
+    bits: Spanned<usize>,
     number_size: NumberBytes,
 }
 
@@ -69,7 +68,7 @@ impl Command for BitsRor {
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let head = call.head;
-        let bits: usize = call.req(engine_state, stack, 0)?;
+        let bits = call.req(engine_state, stack, 0)?;
         let signed = call.has_flag(engine_state, stack, "signed")?;
         let number_bytes: Option<Spanned<usize>> =
             call.get_flag(engine_state, stack, "number-bytes")?;
@@ -123,6 +122,8 @@ fn action(input: &Value, args: &Arguments, span: Span) -> Value {
         number_size,
         bits,
     } = *args;
+    let bits_span = bits.span;
+    let bits = bits.item;
 
     match input {
         Value::Int { val, .. } => {
@@ -131,6 +132,19 @@ fn action(input: &Value, args: &Arguments, span: Span) -> Value {
             let bits = bits as u32;
             let input_num_type = get_input_num_type(val, signed, number_size);
 
+            if bits > input_num_type.num_bits() {
+                return Value::error(
+                    ShellError::IncorrectValue {
+                        msg: format!(
+                            "Trying to rotate by more than the available bits ({})",
+                            input_num_type.num_bits()
+                        ),
+                        val_span: bits_span,
+                        call_span: span,
+                    },
+                    span,
+                );
+            }
             let int = match input_num_type {
                 One => (val as u8).rotate_right(bits) as i64,
                 Two => (val as u16).rotate_right(bits) as i64,
@@ -161,16 +175,28 @@ fn action(input: &Value, args: &Arguments, span: Span) -> Value {
             Value::int(int, span)
         }
         Value::Binary { val, .. } => {
+            let len = val.len();
+            if bits > len * 8 {
+                return Value::error(
+                    ShellError::IncorrectValue {
+                        msg: format!(
+                            "Trying to rotate by more than the available bits ({})",
+                            len * 8
+                        ),
+                        val_span: bits_span,
+                        call_span: span,
+                    },
+                    span,
+                );
+            }
             let byte_shift = bits / 8;
             let bit_rotate = bits % 8;
 
-            let mut bytes = val
+            let bytes = if bit_rotate == 0 {
-                .iter()
+                rotate_bytes_right(val, byte_shift)
-                .copied()
+            } else {
-                .circular_tuple_windows::<(u8, u8)>()
+                rotate_bytes_and_bits_right(val, byte_shift, bit_rotate)
-                .map(|(lhs, rhs)| (lhs >> bit_rotate) | (rhs << (8 - bit_rotate)))
+            };
-                .collect::<Vec<u8>>();
-            bytes.rotate_right(byte_shift);
 
             Value::binary(bytes, span)
         }
@@ -188,6 +214,35 @@ fn action(input: &Value, args: &Arguments, span: Span) -> Value {
     }
 }
 
+fn rotate_bytes_right(data: &[u8], byte_shift: usize) -> Vec<u8> {
+    let len = data.len();
+    let mut output = vec![0; len];
+    output[byte_shift..].copy_from_slice(&data[..len - byte_shift]);
+    output[..byte_shift].copy_from_slice(&data[len - byte_shift..]);
+    output
+}
+
+fn rotate_bytes_and_bits_right(data: &[u8], byte_shift: usize, bit_shift: usize) -> Vec<u8> {
+    debug_assert!(byte_shift < data.len());
+    debug_assert!(
+        (1..8).contains(&bit_shift),
+        "Bit shifts of 0 can't be handled by this impl and everything else should be part of the byteshift"
+    );
+    let mut bytes = Vec::with_capacity(data.len());
+    let mut previous_index = data.len() - byte_shift - 1;
+    for _ in 0..data.len() {
+        let previous_byte = data[previous_index];
+        previous_index += 1;
+        if previous_index == data.len() {
+            previous_index = 0;
+        }
+        let curr_byte = data[previous_index];
+        let rotated_byte = (curr_byte >> bit_shift) | (previous_byte << (8 - bit_shift));
+        bytes.push(rotated_byte);
+    }
+
+    bytes
+}
 #[cfg(test)]
 mod test {
     use super::*;

tests/repl/test_bits.rs

@@ -241,20 +241,124 @@ fn bits_rotate_left_list() -> TestResult {
     )
 }
 
+#[test]
+fn bits_rotate_left_negative_operand() -> TestResult {
+    fail_test("8 | bits rol -2", "positive value")
+}
+#[test]
+fn bits_rotate_left_exceeding1() -> TestResult {
+    // We have no type accepting more than 64 bits so guaranteed fail
+    fail_test("8 | bits rol 65", "more than the available bits (8)")
+}
+
+#[test]
+fn bits_rotate_left_exceeding2() -> TestResult {
+    // This is purely down to the current autodetect feature limiting to the smallest integer
+    // type thus assuming a u8
+    fail_test("8 | bits rol 9", "more than the available bits (8)")
+}
+
+#[test]
+fn bits_rotate_left_binary1() -> TestResult {
+    run_test(
+        "0x[01 30 80] | bits rol 3 | into bits",
+        "00001001 10000100 00000000",
+    )
+}
+
+#[test]
+fn bits_rotate_left_binary2() -> TestResult {
+    // Whole byte case
+    run_test(
+        "0x[01 30 80] | bits rol 8 | into bits",
+        "00110000 10000000 00000001",
+    )
+}
+
+#[test]
+fn bits_rotate_left_binary3() -> TestResult {
+    // Compared to the int case this is made inclusive of the bit count
+    run_test(
+        "0x[01 30 80] | bits rol 24 | into bits",
+        "00000001 00110000 10000000",
+    )
+}
+
+#[test]
+fn bits_rotate_left_binary4() -> TestResult {
+    // Shifting by both bytes and bits
+    run_test(
+        "0x[01 30 80] | bits rol 15 | into bits",
+        "01000000 00000000 10011000",
+    )
+}
+
 #[test]
 fn bits_rotate_right() -> TestResult {
-    run_test("2 | bits ror 62", "8")
+    run_test("2 | bits ror 6", "8")
 }
 
 #[test]
 fn bits_rotate_right_negative() -> TestResult {
-    run_test("-3 | bits ror 60", "-33")
+    run_test("-3 | bits ror 4", "-33")
 }
 
 #[test]
 fn bits_rotate_right_list() -> TestResult {
     run_test(
-        "[1 2 7 32 23 10] | bits ror 60 | str join '.'",
+        "[1 2 7 32 23 10] | bits ror 4 | str join '.'",
         "16.32.112.2.113.160",
     )
 }
+#[test]
+fn bits_rotate_right_negative_operand() -> TestResult {
+    fail_test("8 | bits ror -2", "positive value")
+}
+
+#[test]
+fn bits_rotate_right_exceeding1() -> TestResult {
+    // We have no type accepting more than 64 bits so guaranteed fail
+    fail_test("8 | bits ror 65", "more than the available bits (8)")
+}
+
+#[test]
+fn bits_rotate_right_exceeding2() -> TestResult {
+    // This is purely down to the current autodetect feature limiting to the smallest integer
+    // type thus assuming a u8
+    fail_test("8 | bits ror 9", "more than the available bits (8)")
+}
+
+#[test]
+fn bits_rotate_right_binary1() -> TestResult {
+    run_test(
+        "0x[01 30 80] | bits ror 3 | into bits",
+        "00000000 00100110 00010000",
+    )
+}
+
+#[test]
+fn bits_rotate_right_binary2() -> TestResult {
+    // Whole byte case
+    run_test(
+        "0x[01 30 80] | bits ror 8 | into bits",
+        "10000000 00000001 00110000",
+    )
+}
+
+#[test]
+fn bits_rotate_right_binary3() -> TestResult {
+    // Compared to the int case this is made inclusive of the bit count
+    run_test(
+        "0x[01 30 80] | bits ror 24 | into bits",
+        "00000001 00110000 10000000",
+    )
+}
+
+#[test]
+fn bits_rotate_right_binary4() -> TestResult {
+    // Shifting by both bytes and bits
+    run_test(
+        "0x[01 30 80] | bits ror 15 | into bits",
+        "01100001 00000000 00000010",
+    )
+}