Add string stream and binary stream, add text decoding (#570)

* WIP

* Add binary/string streams and text decoding

* Make string collection fallible

* Oops, forgot pretty hex

* Oops, forgot pretty hex

* clippy

crates/nu-command/src/strings/decode.rs

@@ -0,0 +1,107 @@
+use encoding_rs::Encoding;
+use nu_engine::CallExt;
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Spanned, SyntaxShape,
+    Value,
+};
+
+#[derive(Clone)]
+pub struct Decode;
+
+impl Command for Decode {
+    fn name(&self) -> &str {
+        "decode"
+    }
+
+    fn usage(&self) -> &str {
+        "Decode bytes as a string."
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("decode")
+            .required("encoding", SyntaxShape::String, "the text encoding to use")
+            .category(Category::Strings)
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Decode the output of an external command",
+            example: "cat myfile.q | decode utf-8",
+            result: None,
+        }]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let head = call.head;
+        let encoding: Spanned<String> = call.req(engine_state, stack, 0)?;
+
+        match input {
+            PipelineData::ByteStream(stream, ..) => {
+                let bytes: Vec<u8> = stream.flatten().collect();
+
+                let encoding = match Encoding::for_label(encoding.item.as_bytes()) {
+                    None => Err(ShellError::SpannedLabeledError(
+                        format!(
+                            r#"{} is not a valid encoding, refer to https://docs.rs/encoding_rs/0.8.23/encoding_rs/#statics for a valid list of encodings"#,
+                            encoding.item
+                        ),
+                        "invalid encoding".into(),
+                        encoding.span,
+                    )),
+                    Some(encoding) => Ok(encoding),
+                }?;
+
+                let result = encoding.decode(&bytes);
+
+                Ok(Value::String {
+                    val: result.0.to_string(),
+                    span: head,
+                }
+                .into_pipeline_data())
+            }
+            PipelineData::Value(Value::Binary { val: bytes, .. }, ..) => {
+                let encoding = match Encoding::for_label(encoding.item.as_bytes()) {
+                    None => Err(ShellError::SpannedLabeledError(
+                        format!(
+                            r#"{} is not a valid encoding, refer to https://docs.rs/encoding_rs/0.8.23/encoding_rs/#statics for a valid list of encodings"#,
+                            encoding.item
+                        ),
+                        "invalid encoding".into(),
+                        encoding.span,
+                    )),
+                    Some(encoding) => Ok(encoding),
+                }?;
+
+                let result = encoding.decode(&bytes);
+
+                Ok(Value::String {
+                    val: result.0.to_string(),
+                    span: head,
+                }
+                .into_pipeline_data())
+            }
+            _ => Err(ShellError::UnsupportedInput(
+                "non-binary input".into(),
+                head,
+            )),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        crate::test_examples(Decode)
+    }
+}

crates/nu-command/src/strings/format/command.rs

@@ -151,7 +151,7 @@ fn format(
                 }
             }
 
-            Ok(PipelineData::Stream(
+            Ok(PipelineData::ListStream(
                 ValueStream::from_stream(list.into_iter(), None),
                 None,
             ))

crates/nu-command/src/strings/mod.rs

@@ -1,5 +1,6 @@
 mod build_string;
 mod char_;
+mod decode;
 mod format;
 mod parse;
 mod size;
@@ -8,6 +9,7 @@ mod str_;
 
 pub use build_string::BuildString;
 pub use char_::Char;
+pub use decode::*;
 pub use format::*;
 pub use parse::*;
 pub use size::Size;

crates/nu-command/src/strings/parse.rs

@@ -126,7 +126,7 @@ fn operate(
         }
     }
 
-    Ok(PipelineData::Stream(
+    Ok(PipelineData::ListStream(
         ValueStream::from_stream(parsed.into_iter(), ctrlc),
         None,
     ))