Merge pull request #1126 from jonathandturner/utf8_fix

UTF8 fix for twitter-reported issue

crates/nu-parser/src/parse/parser.rs

@@ -357,7 +357,10 @@ fn word<'a, T, U, V>(
 
 pub fn matches(cond: fn(char) -> bool) -> impl Fn(NomSpan) -> IResult<NomSpan, NomSpan> + Copy {
     move |input: NomSpan| match input.iter_elements().next() {
-        Option::Some(c) if cond(c) => Ok((input.slice(1..), input.slice(0..1))),
+        Option::Some(c) if cond(c) => {
+            let len_utf8 = c.len_utf8();
+            Ok((input.slice(len_utf8..), input.slice(0..len_utf8)))
+        }
         _ => Err(nom::Err::Error(nom::error::ParseError::from_error_kind(
             input,
             nom::error::ErrorKind::Many0,

tests/shell/mod.rs

@@ -29,6 +29,21 @@ mod pipeline {
         })
     }
 
+    #[test]
+    fn doesnt_break_on_utf8_command() {
+        let actual = nu!(
+            cwd: ".", pipeline(
+            r#"
+                echo ö
+            "#
+        ));
+
+        assert!(
+            actual.contains("ö"),
+            format!("'{}' should contain ö", actual)
+        );
+    }
+
     #[test]
     fn can_process_row_as_it_argument_to_an_external_command_given_the_it_data_is_one_string_line()
     {