holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

str length, str substring, str index-of, split words and split chars now use graphemes instead of UTF-8 bytes (#7752)

Browse Source 
Closes https://github.com/nushell/nushell/issues/7742
This commit is contained in:
Leon
2023-01-20 17:16:18 +10:00
committed by GitHub
parent 0fe2884397
commit ea9ca8b4ed
6 changed files with 323 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
crates/nu-command/src/strings/split/chars.rs
View File

@ -1,8 +1,10 @@
use crate::grapheme_flags;
use nu_protocol::{
ast::Call,
engine::{Command, EngineState, Stack},
Category, Example, PipelineData, ShellError, Signature, Span, Type, Value,
};
use unicode_segmentation::UnicodeSegmentation;
#[derive(Clone)]
pub struct SubCommand;
@ -15,6 +17,12 @@ impl Command for SubCommand {
fn signature(&self) -> Signature {
Signature::build("split chars")
.input_output_types(vec![(Type::String, Type::List(Box::new(Type::String)))])
.switch("grapheme-clusters", "split on grapheme clusters", Some('g'))
.switch(
"code-points",
"split on code points (default; splits combined characters)",
Some('c'),
)
.vectorizes_over_list(true)
.category(Category::Strings)
}
@ -28,20 +36,34 @@ impl Command for SubCommand {
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Split the string into a list of characters",
example: "'hello' | split chars",
result: Some(Value::List {
vals: vec![
Value::test_string("h"),
Value::test_string("e"),
Value::test_string("l"),
Value::test_string("l"),
Value::test_string("o"),
],
span: Span::test_data(),
}),
}]
vec![
Example {
description: "Split the string into a list of characters",
example: "'hello' | split chars",
result: Some(Value::List {
vals: vec![
Value::test_string("h"),
Value::test_string("e"),
Value::test_string("l"),
Value::test_string("l"),
Value::test_string("o"),
],
span: Span::test_data(),
}),
},
Example {
description: "Split on grapheme clusters",
example: "'🇯🇵ほげ' | split chars -g",
result: Some(Value::List {
vals: vec![
Value::test_string("🇯🇵"),
Value::test_string("ほ"),
Value::test_string("げ"),
],
span: Span::test_data(),
}),
},
]
}
fn run(
@ -62,21 +84,30 @@ fn split_chars(
) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
let span = call.head;
let graphemes = grapheme_flags(call)?;
input.flat_map(
move |x| split_chars_helper(&x, span),
move |x| split_chars_helper(&x, span, graphemes),
engine_state.ctrlc.clone(),
)
}
fn split_chars_helper(v: &Value, name: Span) -> Vec<Value> {
fn split_chars_helper(v: &Value, name: Span, graphemes: bool) -> Vec<Value> {
match v.span() {
Ok(v_span) => {
if let Ok(s) = v.as_string() {
s.chars()
.collect::<Vec<_>>()
.into_iter()
.map(move |x| Value::string(x, v_span))
.collect()
if graphemes {
s.graphemes(true)
.collect::<Vec<_>>()
.into_iter()
.map(move |x| Value::string(x, v_span))
.collect()
} else {
s.chars()
.collect::<Vec<_>>()
.into_iter()
.map(move |x| Value::string(x, v_span))
.collect()
}
} else {
vec![Value::Error {
error: ShellError::PipelineMismatch("string".into(), name, v_span),

59
crates/nu-command/src/strings/split/words.rs
View File

@ -1,3 +1,4 @@
use crate::grapheme_flags;
use fancy_regex::Regex;
use nu_engine::CallExt;
use nu_protocol::{
@ -5,6 +6,7 @@ use nu_protocol::{
engine::{Command, EngineState, Stack},
Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
};
use unicode_segmentation::UnicodeSegmentation;
#[derive(Clone)]
pub struct SubCommand;
@ -40,6 +42,16 @@ impl Command for SubCommand {
"The minimum word length",
Some('l'),
)
.switch(
"grapheme-clusters",
"measure word length in grapheme clusters (requires -l)",
Some('g'),
)
.switch(
"utf-8-bytes",
"measure word length in UTF-8 bytes (default; requires -l; non-ASCII chars are length 2+)",
Some('b'),
)
}
fn usage(&self) -> &str {
@ -105,13 +117,34 @@ fn split_words(
// let ignore_punctuation = call.has_flag("ignore-punctuation");
let word_length: Option<usize> = call.get_flag(engine_state, stack, "min-word-length")?;
if matches!(word_length, None) {
if call.has_flag("grapheme-clusters") {
return Err(ShellError::IncompatibleParametersSingle(
"--grapheme-clusters (-g) requires --min-word-length (-l)".to_string(),
span,
));
}
if call.has_flag("utf-8-bytes") {
return Err(ShellError::IncompatibleParametersSingle(
"--utf-8-bytes (-b) requires --min-word-length (-l)".to_string(),
span,
));
}
}
let graphemes = grapheme_flags(call)?;
input.flat_map(
move |x| split_words_helper(&x, word_length, span),
move |x| split_words_helper(&x, word_length, span, graphemes),
engine_state.ctrlc.clone(),
)
}
fn split_words_helper(v: &Value, word_length: Option<usize>, span: Span) -> Vec<Value> {
fn split_words_helper(
v: &Value,
word_length: Option<usize>,
span: Span,
graphemes: bool,
) -> Vec<Value> {
// There are some options here with this regex.
// [^A-Za-z\'] = do not match uppercase or lowercase letters or apostrophes
// [^[:alpha:]\'] = do not match any uppercase or lowercase letters or apostrophes
@ -132,7 +165,12 @@ fn split_words_helper(v: &Value, word_length: Option<usize>, span: Span) -> Vec<
.filter_map(|s| {
if s.trim() != "" {
if let Some(len) = word_length {
if s.chars().count() >= len {
if if graphemes {
s.graphemes(true).count()
} else {
s.len()
} >= len
{
Some(Value::string(s, v_span))
} else {
None
@ -144,7 +182,7 @@ fn split_words_helper(v: &Value, word_length: Option<usize>, span: Span) -> Vec<
None
}
})
.collect()
.collect::<Vec<Value>>()
} else {
vec![Value::Error {
error: ShellError::PipelineMismatch("string".into(), span, v_span),
@ -319,6 +357,19 @@ fn split_words_helper(v: &Value, word_length: Option<usize>, span: Span) -> Vec<
#[cfg(test)]
mod test {
use super::*;
use nu_test_support::{nu, pipeline};
#[test]
fn test_incompat_flags() {
let out = nu!(cwd: ".", pipeline("'a' | split words -bg -l 2"));
assert!(out.err.contains("incompatible_parameters"));
}
#[test]
fn test_incompat_flags_2() {
let out = nu!(cwd: ".", pipeline("'a' | split words -g"));
assert!(out.err.contains("incompatible_parameters"));
}
#[test]
fn test_examples() {