Nucli refactor crate stream (#2828)

* nu-stream is building on its own, now clean up Cargo.toml

* replace the stream crate in nu-cli

* cc

* since we moved stream out of the nu-cli crate and into its own crate we need to remove pub(crate) and just make it pub

* clean up the prelude and hand merge everything together

* clean up Cargo.tom

* cargo fmt along with Cargo.lock

Cargo.lock

@@ -3181,6 +3181,7 @@ dependencies = [
  "nu-plugin",
  "nu-protocol",
  "nu-source",
+ "nu-stream",
  "nu-table",
  "nu-test-support",
  "nu-value-ext",
@@ -3358,6 +3359,16 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "nu-stream"
+version = "0.24.2"
+dependencies = [
+ "futures 0.3.8",
+ "nu-errors",
+ "nu-protocol",
+ "nu-source",
+]
+
 [[package]]
 name = "nu-table"
 version = "0.24.2"

crates/nu-cli/Cargo.toml

@@ -18,6 +18,7 @@ nu-parser = {version = "0.24.2", path = "../nu-parser"}
 nu-plugin = {version = "0.24.2", path = "../nu-plugin"}
 nu-protocol = {version = "0.24.2", path = "../nu-protocol"}
 nu-source = {version = "0.24.2", path = "../nu-source"}
+nu-stream = {version = "0.24.2", path = "../nu-stream"}
 nu-table = {version = "0.24.2", path = "../nu-table"}
 nu-test-support = {version = "0.24.2", path = "../nu-test-support"}
 nu-value-ext = {version = "0.24.2", path = "../nu-value-ext"}

crates/nu-cli/src/commands/classified/block.rs

@@ -2,7 +2,6 @@ use crate::commands::classified::expr::run_expression_block;
 use crate::commands::classified::internal::run_internal_command;
 use crate::evaluation_context::EvaluationContext;
 use crate::prelude::*;
-use crate::stream::InputStream;
 use async_recursion::async_recursion;
 use futures::stream::TryStreamExt;
 use nu_errors::ShellError;
@@ -10,6 +9,7 @@ use nu_protocol::hir::{
     Block, Call, ClassifiedCommand, Expression, Pipeline, SpannedExpression, Synthetic,
 };
 use nu_protocol::{ReturnSuccess, UntaggedValue, Value};
+use nu_stream::InputStream;
 use std::sync::atomic::Ordering;
 
 #[async_recursion]

crates/nu-cli/src/evaluation_context.rs

@@ -2,9 +2,9 @@ use crate::commands::{command::CommandArgs, Command, UnevaluatedCallInfo};
 use crate::env::host::Host;
 use crate::prelude::*;
 use crate::shell::shell_manager::ShellManager;
-use crate::stream::{InputStream, OutputStream};
 use nu_protocol::hir;
 use nu_source::{Tag, Text};
+use nu_stream::{InputStream, OutputStream};
 use parking_lot::Mutex;
 use std::error::Error;
 use std::sync::atomic::AtomicBool;

crates/nu-cli/src/examples.rs

@@ -17,7 +17,7 @@ use crate::commands::{
     StrCollect, WholeStreamCommand, Wrap,
 };
 use crate::evaluation_context::EvaluationContext;
-use crate::stream::{InputStream, OutputStream};
+use nu_stream::{InputStream, OutputStream};
 
 use async_trait::async_trait;
 use futures::executor::block_on;

crates/nu-cli/src/lib.rs

@@ -31,7 +31,6 @@ mod keybinding;
 mod path;
 mod plugin;
 mod shell;
-mod stream;
 pub mod types;
 pub mod utils;
 
@@ -54,11 +53,11 @@ pub use crate::env::environment_syncer::EnvironmentSyncer;
 pub use crate::env::host::BasicHost;
 pub use crate::evaluation_context::EvaluationContext;
 pub use crate::prelude::ToOutputStream;
-pub use crate::stream::{InputStream, InterruptibleStream, OutputStream};
 pub use nu_data::config;
 pub use nu_data::dict::TaggedListBuilder;
 pub use nu_data::primitive;
 pub use nu_data::value;
+pub use nu_stream::{InputStream, InterruptibleStream, OutputStream};
 pub use nu_value_ext::ValueExt;
 pub use num_traits::cast::ToPrimitive;
 

crates/nu-cli/src/prelude.rs

@@ -36,7 +36,7 @@ macro_rules! trace_stream {
                 );
             });
 
-            $crate::stream::InputStream::from_stream(objects.boxed())
+            nu_stream::InputStream::from_stream(objects.boxed())
         } else {
             $expr
         }
@@ -61,7 +61,7 @@ macro_rules! trace_out_stream {
                 );
             });
 
-            $crate::stream::OutputStream::new(objects)
+            nu_stream::OutputStream::new(objects)
         } else {
             $expr
         }
@@ -84,7 +84,6 @@ pub(crate) use crate::shell::filesystem_shell::FilesystemShell;
 pub(crate) use crate::shell::help_shell::HelpShell;
 pub(crate) use crate::shell::shell_manager::ShellManager;
 pub(crate) use crate::shell::value_shell::ValueShell;
-pub(crate) use crate::stream::{InputStream, InterruptibleStream, OutputStream};
 pub(crate) use bigdecimal::BigDecimal;
 pub(crate) use futures::stream::BoxStream;
 pub(crate) use futures::{Stream, StreamExt};
@@ -94,6 +93,7 @@ pub(crate) use nu_source::{
     b, AnchorLocation, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource, Span, SpannedItem, Tag,
     TaggedItem, Text,
 };
+pub(crate) use nu_stream::{InputStream, InterruptibleStream, OutputStream};
 pub(crate) use nu_value_ext::ValueExt;
 pub(crate) use num_bigint::BigInt;
 pub(crate) use num_traits::cast::ToPrimitive;

crates/nu-cli/src/shell/shell.rs

@@ -7,7 +7,7 @@ use crate::commands::mkdir::MkdirArgs;
 use crate::commands::move_::mv::Arguments as MvArgs;
 use crate::commands::rm::RemoveArgs;
 use crate::prelude::*;
-use crate::stream::OutputStream;
+use nu_stream::OutputStream;
 
 use encoding_rs::Encoding;
 use nu_errors::ShellError;

crates/nu-cli/src/shell/shell_manager.rs

@@ -9,7 +9,7 @@ use crate::commands::rm::RemoveArgs;
 use crate::prelude::*;
 use crate::shell::filesystem_shell::FilesystemShell;
 use crate::shell::shell::Shell;
-use crate::stream::OutputStream;
+use nu_stream::OutputStream;
 
 use encoding_rs::Encoding;
 use nu_errors::ShellError;

crates/nu-stream/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+authors = ["The Nu Project Contributors"]
+description = "Nushell stream"
+edition = "2018"
+license = "MIT"
+name = "nu-stream"
+version = "0.24.2"
+
+[dependencies]
+nu-errors = {version = "0.24.2", path = "../nu-errors"}
+nu-protocol = {version = "0.24.2", path = "../nu-protocol"}
+nu-source = {version = "0.24.2", path = "../nu-source"}
+
+futures = {version = "0.3.5", features = ["compat", "io-compat"]}
+
+[features]
+stable = []
+trace = []

crates/nu-stream/src/input.rs

@@ -2,7 +2,7 @@ use crate::prelude::*;
 use futures::stream::{iter, once};
 use nu_errors::ShellError;
 use nu_protocol::{Primitive, Type, UntaggedValue, Value};
-use nu_source::{Tagged, TaggedItem};
+use nu_source::{PrettyDebug, Tag, Tagged, TaggedItem};
 
 pub struct InputStream {
     values: BoxStream<'static, Value>,

crates/nu-stream/src/lib.rs

@@ -1,3 +1,5 @@
+mod prelude;
+
 mod input;
 mod interruptible;
 mod output;

crates/nu-stream/src/output.rs

@@ -4,7 +4,7 @@ use nu_protocol::{ReturnSuccess, ReturnValue, Value};
 use std::iter::IntoIterator;
 
 pub struct OutputStream {
-    pub(crate) values: BoxStream<'static, ReturnValue>,
+    pub values: BoxStream<'static, ReturnValue>,
 }
 
 impl OutputStream {

crates/nu-stream/src/prelude.rs

@@ -0,0 +1,93 @@
+#[macro_export]
+macro_rules! return_err {
+    ($expr:expr) => {
+        match $expr {
+            Err(_) => return,
+            Ok(expr) => expr,
+        };
+    };
+}
+
+#[macro_export]
+macro_rules! stream {
+    ($($expr:expr),*) => {{
+        let mut v = VecDeque::new();
+
+        $(
+            v.push_back($expr);
+        )*
+
+        v
+    }}
+}
+
+#[macro_export]
+macro_rules! trace_stream {
+    (target: $target:tt, $desc:tt = $expr:expr) => {{
+        if log::log_enabled!(target: $target, log::Level::Trace) {
+            use futures::stream::StreamExt;
+
+            let objects = $expr.inspect(move |o| {
+                trace!(
+                    target: $target,
+                    "{} = {}",
+                    $desc,
+                    nu_source::PrettyDebug::plain_string(o, 70)
+                );
+            });
+
+            $crate::stream::InputStream::from_stream(objects.boxed())
+        } else {
+            $expr
+        }
+    }};
+}
+
+#[macro_export]
+macro_rules! trace_out_stream {
+    (target: $target:tt, $desc:tt = $expr:expr) => {{
+        if log::log_enabled!(target: $target, log::Level::Trace) {
+            use futures::stream::StreamExt;
+
+            let objects = $expr.inspect(move |o| {
+                trace!(
+                    target: $target,
+                    "{} = {}",
+                    $desc,
+                    match o {
+                        Err(err) => format!("{:?}", err),
+                        Ok(value) => value.display(),
+                    }
+                );
+            });
+
+            $crate::stream::OutputStream::new(objects)
+        } else {
+            $expr
+        }
+    }};
+}
+
+pub(crate) use futures::stream::BoxStream;
+pub(crate) use futures::{Stream, StreamExt};
+pub(crate) use std::collections::VecDeque;
+pub(crate) use std::future::Future;
+pub(crate) use std::sync::Arc;
+
+pub(crate) use crate::{InputStream, OutputStream};
+
+pub trait ToOutputStream {
+    fn to_output_stream(self) -> OutputStream;
+}
+
+impl<T, U> ToOutputStream for T
+where
+    T: Stream<Item = U> + Send + 'static,
+    U: Into<nu_protocol::ReturnValue>,
+{
+    fn to_output_stream(self) -> OutputStream {
+        OutputStream {
+            values: self.map(|item| item.into()).boxed(),
+        }
+    }
+}