diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index 1cf764e23d..7cf2b12ba9 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -2768,7 +2768,8 @@ pub fn parse_unit_value<'res>( return None; } - let value = transform(String::from_utf8_lossy(bytes).into()); + // Bail if not UTF-8 + let value = transform(str::from_utf8(bytes).ok()?.into()); if let Some((unit, name, convert)) = unit_groups.iter().find(|x| value.ends_with(x.1)) { let lhs_len = value.len() - name.len(); diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index 2267916322..b93762c865 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -292,6 +292,21 @@ pub fn parse_filesize() { assert_eq!(value.unit.item, Unit::Filesize(FilesizeUnit::B)); } +#[test] +pub fn parse_non_utf8_fails() { + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); + + // Panic when parsing units was triggered by non-UTF8 characters + // due to bad handling via `String::from_utf8_lossy` + // + // See https://github.com/nushell/nushell/pull/16355 + let _block = parse(&mut working_set, None, b"0\xffB", true); + + // Asserting on the exact error doesn't make as much sense as + assert!(!working_set.parse_errors.is_empty()); +} + #[test] pub fn parse_cell_path() { let engine_state = EngineState::new();