forked from extern/nushell
e616b2e247
# Description
Support extended unicode escapes in strings with same syntax as Rust:
`"\u{6e}"`.
# User-Facing Changes
New syntax in string literals, `\u{NNNNNN}`, to go along with the
existing `\uNNNN`.
New syntax accepts 1-6 hex digits and rejects values greater than
0x10FFFF (max Unicode char)..
_(List of all changes that impact the user experience here. This helps
us keep track of breaking changes.)_
Won't break existing scripts, since this is new syntax.
We might consider deprecating `char -u`, since users can now embed
unicode chars > 0xFFFF with the new escape.
# Tests + Formatting
Several unit tests and one integration test added.
- [x] `cargo fmt --all -- --check` to check standard code formatting
(`cargo fmt --all` applies these changes)
Done
- [x] `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A
clippy::needless_collect` to check that you're using the standard code
style
Done
- [x] `cargo test --workspace` to check that all tests pass
Done
# After Submitting
- [ ] If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
92 lines
3.2 KiB
Rust
92 lines
3.2 KiB
Rust
#![cfg(test)]
|
|
|
|
//use nu_parser::ParseError;
|
|
use nu_parser::*;
|
|
use nu_protocol::{
|
|
//ast::{Expr, Expression, PipelineElement},
|
|
ast::{Expr, PipelineElement},
|
|
//engine::{Command, EngineState, Stack, StateWorkingSet},
|
|
engine::{EngineState, StateWorkingSet},
|
|
//Signature, SyntaxShape,
|
|
};
|
|
|
|
pub fn do_test(test: &[u8], expected: &str, error_contains: Option<&str>) {
|
|
let engine_state = EngineState::new();
|
|
let mut working_set = StateWorkingSet::new(&engine_state);
|
|
|
|
let (block, err) = parse(&mut working_set, None, test, true, &[]);
|
|
|
|
match err {
|
|
None => {
|
|
assert_eq!(block.len(), 1);
|
|
let expressions = &block[0];
|
|
assert_eq!(expressions.len(), 1);
|
|
if let PipelineElement::Expression(_, expr) = &expressions[0] {
|
|
assert_eq!(expr.expr, Expr::String(expected.to_string()))
|
|
} else {
|
|
panic!("Not an expression")
|
|
}
|
|
}
|
|
Some(pev) => match error_contains {
|
|
None => {
|
|
panic!("Err:{:#?}", pev);
|
|
}
|
|
Some(contains_string) => {
|
|
let full_err = format!("{:#?}", pev);
|
|
assert!(
|
|
full_err.contains(contains_string),
|
|
"Expected error containing {}, instead got {}",
|
|
contains_string,
|
|
full_err
|
|
);
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
// cases that all should work
|
|
#[test]
|
|
pub fn unicode_escapes_in_strings() {
|
|
pub struct Tc(&'static [u8], &'static str);
|
|
|
|
let test_vec = vec![
|
|
Tc(b"\"hello \\u{6e}\\u{000075}\\u{073}hell\"", "hello nushell"),
|
|
// template: Tc(br#""<string literal without #'s>"", "<Rust literal comparand>")
|
|
//deprecated Tc(br#""\u006enu\u0075\u0073\u0073""#, "nnuuss"),
|
|
Tc(br#""hello \u{6e}\u{000075}\u{073}hell""#, "hello nushell"),
|
|
Tc(br#""\u{39}8\u{10ffff}""#, "98\u{10ffff}"),
|
|
Tc(br#""abc\u{41}""#, "abcA"), // at end of string
|
|
Tc(br#""\u{41}abc""#, "Aabc"), // at start of string
|
|
Tc(br#""\u{a}""#, "\n"), // single digit
|
|
];
|
|
|
|
for tci in test_vec {
|
|
println!("Expecting: {}", tci.1);
|
|
do_test(tci.0, tci.1, None);
|
|
}
|
|
}
|
|
|
|
// cases that all should fail (in expected way)
|
|
#[test]
|
|
pub fn unicode_escapes_in_strings_expected_failures() {
|
|
// input, substring of expected failure
|
|
pub struct Tc(&'static [u8], &'static str);
|
|
|
|
let test_vec = vec![
|
|
// template: Tc(br#""<string literal without #'s>"", "<pattern in expected error>")
|
|
//deprecated Tc(br#""\u06e""#, "any shape"), // 4digit too short, next char is EOF
|
|
//deprecatedTc(br#""\u06ex""#, "any shape"), // 4digit too short, next char is non-hex-digit
|
|
Tc(br#""hello \u{6e""#, "any shape"), // extended, missing close delim
|
|
Tc(
|
|
br#""\u{39}8\u{000000000000000000000000000000000000000000000037}""#,
|
|
"any shape",
|
|
), // hex too long, but small value
|
|
Tc(br#""\u{110000}""#, "any shape"), // max unicode <= 0x10ffff
|
|
];
|
|
|
|
for tci in test_vec {
|
|
println!("Expecting failure containing: {}", tci.1);
|
|
do_test(tci.0, "--success not expected--", Some(tci.1));
|
|
}
|
|
}
|