Fix panic in unit parsing with non-UTF8 code (#16355)

# Description
Trying to parse non-UTF8 data as a value with unit (part of every
literal parse) introduced a replacement character which shifted the span
information so the indices where incorrect and triggered
a panic.

This has been resolved by removing a bad `String::from_utf8_lossy`

# User-Facing Changes
One less possible panic

# Tests + Formatting
Added a test with the original reproducer from fuzzing:

File with `0\xffB` where the `\xff` represents the non utf-8 char `FF`
run as a script to trigger
This commit is contained in:
Stefan Holderbach
2025-08-05 22:08:33 +02:00
committed by GitHub
parent 61a89c1834
commit 3a82c6c88d
2 changed files with 17 additions and 1 deletions

View File

@@ -2768,7 +2768,8 @@ pub fn parse_unit_value<'res>(
return None; return None;
} }
let value = transform(String::from_utf8_lossy(bytes).into()); // Bail if not UTF-8
let value = transform(str::from_utf8(bytes).ok()?.into());
if let Some((unit, name, convert)) = unit_groups.iter().find(|x| value.ends_with(x.1)) { if let Some((unit, name, convert)) = unit_groups.iter().find(|x| value.ends_with(x.1)) {
let lhs_len = value.len() - name.len(); let lhs_len = value.len() - name.len();

View File

@@ -292,6 +292,21 @@ pub fn parse_filesize() {
assert_eq!(value.unit.item, Unit::Filesize(FilesizeUnit::B)); assert_eq!(value.unit.item, Unit::Filesize(FilesizeUnit::B));
} }
#[test]
pub fn parse_non_utf8_fails() {
let engine_state = EngineState::new();
let mut working_set = StateWorkingSet::new(&engine_state);
// Panic when parsing units was triggered by non-UTF8 characters
// due to bad handling via `String::from_utf8_lossy`
//
// See https://github.com/nushell/nushell/pull/16355
let _block = parse(&mut working_set, None, b"0\xffB", true);
// Asserting on the exact error doesn't make as much sense as
assert!(!working_set.parse_errors.is_empty());
}
#[test] #[test]
pub fn parse_cell_path() { pub fn parse_cell_path() {
let engine_state = EngineState::new(); let engine_state = EngineState::new();