mirror of
https://github.com/nushell/nushell.git
synced 2025-06-30 14:40:06 +02:00
Support extended unicode escapes in strings: "\u{10fff}" (#7883)
# Description Support extended unicode escapes in strings with same syntax as Rust: `"\u{6e}"`. # User-Facing Changes New syntax in string literals, `\u{NNNNNN}`, to go along with the existing `\uNNNN`. New syntax accepts 1-6 hex digits and rejects values greater than 0x10FFFF (max Unicode char).. _(List of all changes that impact the user experience here. This helps us keep track of breaking changes.)_ Won't break existing scripts, since this is new syntax. We might consider deprecating `char -u`, since users can now embed unicode chars > 0xFFFF with the new escape. # Tests + Formatting Several unit tests and one integration test added. - [x] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) Done - [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style Done - [x] `cargo test --workspace` to check that all tests pass Done # After Submitting - [ ] If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
This commit is contained in:
@ -412,30 +412,6 @@ mod string {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
pub fn parse_escaped_string() {
|
||||
let engine_state = EngineState::new();
|
||||
let mut working_set = StateWorkingSet::new(&engine_state);
|
||||
|
||||
let (block, err) = parse(
|
||||
&mut working_set,
|
||||
None,
|
||||
b"\"hello \\u006e\\u0075\\u0073hell\"",
|
||||
true,
|
||||
&[],
|
||||
);
|
||||
|
||||
assert!(err.is_none());
|
||||
assert_eq!(block.len(), 1);
|
||||
let expressions = &block[0];
|
||||
assert_eq!(expressions.len(), 1);
|
||||
if let PipelineElement::Expression(_, expr) = &expressions[0] {
|
||||
assert_eq!(expr.expr, Expr::String("hello nushell".to_string()))
|
||||
} else {
|
||||
panic!("Not an expression")
|
||||
}
|
||||
}
|
||||
|
||||
mod interpolation {
|
||||
use nu_protocol::Span;
|
||||
|
||||
|
91
crates/nu-parser/tests/test_parser_unicode_escapes.rs
Normal file
91
crates/nu-parser/tests/test_parser_unicode_escapes.rs
Normal file
@ -0,0 +1,91 @@
|
||||
#![cfg(test)]
|
||||
|
||||
//use nu_parser::ParseError;
|
||||
use nu_parser::*;
|
||||
use nu_protocol::{
|
||||
//ast::{Expr, Expression, PipelineElement},
|
||||
ast::{Expr, PipelineElement},
|
||||
//engine::{Command, EngineState, Stack, StateWorkingSet},
|
||||
engine::{EngineState, StateWorkingSet},
|
||||
//Signature, SyntaxShape,
|
||||
};
|
||||
|
||||
pub fn do_test(test: &[u8], expected: &str, error_contains: Option<&str>) {
|
||||
let engine_state = EngineState::new();
|
||||
let mut working_set = StateWorkingSet::new(&engine_state);
|
||||
|
||||
let (block, err) = parse(&mut working_set, None, test, true, &[]);
|
||||
|
||||
match err {
|
||||
None => {
|
||||
assert_eq!(block.len(), 1);
|
||||
let expressions = &block[0];
|
||||
assert_eq!(expressions.len(), 1);
|
||||
if let PipelineElement::Expression(_, expr) = &expressions[0] {
|
||||
assert_eq!(expr.expr, Expr::String(expected.to_string()))
|
||||
} else {
|
||||
panic!("Not an expression")
|
||||
}
|
||||
}
|
||||
Some(pev) => match error_contains {
|
||||
None => {
|
||||
panic!("Err:{:#?}", pev);
|
||||
}
|
||||
Some(contains_string) => {
|
||||
let full_err = format!("{:#?}", pev);
|
||||
assert!(
|
||||
full_err.contains(contains_string),
|
||||
"Expected error containing {}, instead got {}",
|
||||
contains_string,
|
||||
full_err
|
||||
);
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// cases that all should work
|
||||
#[test]
|
||||
pub fn unicode_escapes_in_strings() {
|
||||
pub struct Tc(&'static [u8], &'static str);
|
||||
|
||||
let test_vec = vec![
|
||||
Tc(b"\"hello \\u{6e}\\u{000075}\\u{073}hell\"", "hello nushell"),
|
||||
// template: Tc(br#""<string literal without #'s>"", "<Rust literal comparand>")
|
||||
//deprecated Tc(br#""\u006enu\u0075\u0073\u0073""#, "nnuuss"),
|
||||
Tc(br#""hello \u{6e}\u{000075}\u{073}hell""#, "hello nushell"),
|
||||
Tc(br#""\u{39}8\u{10ffff}""#, "98\u{10ffff}"),
|
||||
Tc(br#""abc\u{41}""#, "abcA"), // at end of string
|
||||
Tc(br#""\u{41}abc""#, "Aabc"), // at start of string
|
||||
Tc(br#""\u{a}""#, "\n"), // single digit
|
||||
];
|
||||
|
||||
for tci in test_vec {
|
||||
println!("Expecting: {}", tci.1);
|
||||
do_test(tci.0, tci.1, None);
|
||||
}
|
||||
}
|
||||
|
||||
// cases that all should fail (in expected way)
|
||||
#[test]
|
||||
pub fn unicode_escapes_in_strings_expected_failures() {
|
||||
// input, substring of expected failure
|
||||
pub struct Tc(&'static [u8], &'static str);
|
||||
|
||||
let test_vec = vec![
|
||||
// template: Tc(br#""<string literal without #'s>"", "<pattern in expected error>")
|
||||
//deprecated Tc(br#""\u06e""#, "any shape"), // 4digit too short, next char is EOF
|
||||
//deprecatedTc(br#""\u06ex""#, "any shape"), // 4digit too short, next char is non-hex-digit
|
||||
Tc(br#""hello \u{6e""#, "any shape"), // extended, missing close delim
|
||||
Tc(
|
||||
br#""\u{39}8\u{000000000000000000000000000000000000000000000037}""#,
|
||||
"any shape",
|
||||
), // hex too long, but small value
|
||||
Tc(br#""\u{110000}""#, "any shape"), // max unicode <= 0x10ffff
|
||||
];
|
||||
|
||||
for tci in test_vec {
|
||||
println!("Expecting failure containing: {}", tci.1);
|
||||
do_test(tci.0, "--success not expected--", Some(tci.1));
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user