Add support for optional list stream output formatting (#6325)

* add support for optional list stream output formatting

* cargo fmt

* table: add ValueFormatter test

crates/nu-protocol/src/pipeline_data.rs

@@ -4,6 +4,7 @@ use crate::{
     format_error, Config, ListStream, RawStream, ShellError, Span, Value,
 };
 use nu_utils::{stderr_write_all_and_flush, stdout_write_all_and_flush};
+use std::fmt;
 use std::sync::{atomic::AtomicBool, Arc};
 
 /// The foundational abstraction for input and output to commands
@@ -94,7 +95,7 @@ impl PipelineData {
             PipelineData::Value(Value::Nothing { .. }, ..) => Value::nothing(span),
             PipelineData::Value(v, ..) => v,
             PipelineData::ListStream(s, ..) => Value::List {
-                vals: s.collect(),
+                vals: s.map(|(value, _)| value).collect(),
                 span, // FIXME?
             },
             PipelineData::ExternalStream {
@@ -222,7 +223,7 @@ impl PipelineData {
         match self {
             // FIXME: there are probably better ways of doing this
             PipelineData::ListStream(stream, ..) => Value::List {
-                vals: stream.collect(),
+                vals: stream.map(|(value, _)| value).collect(),
                 span: head,
             }
             .follow_cell_path(cell_path, insensitive),
@@ -240,7 +241,7 @@ impl PipelineData {
         match self {
             // FIXME: there are probably better ways of doing this
             PipelineData::ListStream(stream, ..) => Value::List {
-                vals: stream.collect(),
+                vals: stream.map(|(value, _)| value).collect(),
                 span: head,
             }
             .upsert_cell_path(cell_path, callback),
@@ -263,7 +264,9 @@ impl PipelineData {
             PipelineData::Value(Value::List { vals, .. }, ..) => {
                 Ok(vals.into_iter().map(f).into_pipeline_data(ctrlc))
             }
-            PipelineData::ListStream(stream, ..) => Ok(stream.map(f).into_pipeline_data(ctrlc)),
+            PipelineData::ListStream(stream, ..) => Ok(stream
+                .map(move |(value, _)| f(value))
+                .into_pipeline_data(ctrlc)),
             PipelineData::ExternalStream { stdout: None, .. } => {
                 Ok(PipelineData::new(Span { start: 0, end: 0 }))
             }
@@ -315,9 +318,9 @@ impl PipelineData {
             PipelineData::Value(Value::List { vals, .. }, ..) => {
                 Ok(vals.into_iter().flat_map(f).into_pipeline_data(ctrlc))
             }
-            PipelineData::ListStream(stream, ..) => {
-                Ok(stream.flat_map(f).into_pipeline_data(ctrlc))
-            }
+            PipelineData::ListStream(stream, ..) => Ok(stream
+                .flat_map(move |(value, _)| f(value))
+                .into_pipeline_data(ctrlc)),
             PipelineData::ExternalStream { stdout: None, .. } => {
                 Ok(PipelineData::new(Span { start: 0, end: 0 }))
             }
@@ -366,7 +369,10 @@ impl PipelineData {
             PipelineData::Value(Value::List { vals, .. }, ..) => {
                 Ok(vals.into_iter().filter(f).into_pipeline_data(ctrlc))
             }
-            PipelineData::ListStream(stream, ..) => Ok(stream.filter(f).into_pipeline_data(ctrlc)),
+            PipelineData::ListStream(stream, ..) => Ok(stream
+                .filter(move |(value, _)| f(value))
+                .map(|(value, _)| value)
+                .into_pipeline_data(ctrlc)),
             PipelineData::ExternalStream { stdout: None, .. } => {
                 Ok(PipelineData::new(Span { start: 0, end: 0 }))
             }
@@ -510,6 +516,31 @@ impl PipelineData {
     }
 }
 
+#[derive(Clone)]
+pub struct ValueFormatter(Arc<dyn Fn(Value) -> Value + Send + Sync>);
+
+impl ValueFormatter {
+    pub fn from_fn<F>(f: F) -> Self
+    where
+        F: Fn(Value) -> Value,
+        F: Send + Sync + 'static,
+    {
+        Self(Arc::new(f))
+    }
+
+    pub fn format(&self, value: Value) -> Value {
+        self.0(value)
+    }
+}
+
+impl fmt::Debug for ValueFormatter {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_tuple("PipelineDataFormatter")
+            .field(&"<formatter>")
+            .finish()
+    }
+}
+
 pub struct PipelineIterator(PipelineData);
 
 impl IntoIterator for PipelineData {
@@ -522,7 +553,7 @@ impl IntoIterator for PipelineData {
             PipelineData::Value(Value::List { vals, .. }, metadata) => {
                 PipelineIterator(PipelineData::ListStream(
                     ListStream {
-                        stream: Box::new(vals.into_iter()),
+                        stream: Box::new(vals.into_iter().map(|v| (v, None))),
                         ctrlc: None,
                     },
                     metadata,
@@ -532,14 +563,14 @@ impl IntoIterator for PipelineData {
                 match val.into_range_iter(None) {
                     Ok(iter) => PipelineIterator(PipelineData::ListStream(
                         ListStream {
-                            stream: Box::new(iter),
+                            stream: Box::new(iter.map(|v| (v, None))),
                             ctrlc: None,
                         },
                         metadata,
                     )),
                     Err(error) => PipelineIterator(PipelineData::ListStream(
                         ListStream {
-                            stream: Box::new(std::iter::once(Value::Error { error })),
+                            stream: Box::new(std::iter::once((Value::Error { error }, None))),
                             ctrlc: None,
                         },
                         metadata,
@@ -558,7 +589,7 @@ impl Iterator for PipelineIterator {
         match &mut self.0 {
             PipelineData::Value(Value::Nothing { .. }, ..) => None,
             PipelineData::Value(v, ..) => Some(std::mem::take(v)),
-            PipelineData::ListStream(stream, ..) => stream.next(),
+            PipelineData::ListStream(stream, ..) => stream.next().map(|(value, _)| value),
             PipelineData::ExternalStream { stdout: None, .. } => None,
             PipelineData::ExternalStream {
                 stdout: Some(stream),
@@ -577,10 +608,12 @@ pub trait IntoPipelineData {
 
 impl<V> IntoPipelineData for V
 where
-    V: Into<Value>,
+    V: Into<(Value, Option<ValueFormatter>)>,
 {
     fn into_pipeline_data(self) -> PipelineData {
-        PipelineData::Value(self.into(), None)
+        let (value, _formatter) = self.into();
+
+        PipelineData::Value(value, None)
     }
 }
 
@@ -597,7 +630,7 @@ impl<I> IntoInterruptiblePipelineData for I
 where
     I: IntoIterator + Send + 'static,
     I::IntoIter: Send + 'static,
-    <I::IntoIter as Iterator>::Item: Into<Value>,
+    <I::IntoIter as Iterator>::Item: Into<(Value, Option<ValueFormatter>)>,
 {
     fn into_pipeline_data(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineData {
         PipelineData::ListStream(

crates/nu-protocol/src/value/mod.rs

@@ -7,8 +7,8 @@ mod unit;
 
 use crate::ast::Operator;
 use crate::ast::{CellPath, PathMember};
-use crate::ShellError;
 use crate::{did_you_mean, BlockId, Config, Span, Spanned, Type, VarId};
+use crate::{ShellError, ValueFormatter};
 use byte_unit::ByteUnit;
 use chrono::{DateTime, Duration, FixedOffset};
 use chrono_humanize::HumanTime;
@@ -1127,6 +1127,12 @@ impl Default for Value {
     }
 }
 
+impl From<Value> for (Value, Option<ValueFormatter>) {
+    fn from(val: Value) -> Self {
+        (val, None)
+    }
+}
+
 impl PartialOrd for Value {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         // Compare two floating point numbers. The decision interval for equality is dynamically

crates/nu-protocol/src/value/stream.rs

@@ -171,23 +171,25 @@ impl Iterator for RawStream {
 /// Like other iterators in Rust, observing values from this stream will drain the items as you view them
 /// and the stream cannot be replayed.
 pub struct ListStream {
-    pub stream: Box<dyn Iterator<Item = Value> + Send + 'static>,
+    pub stream: Box<dyn Iterator<Item = (Value, Option<ValueFormatter>)> + Send + 'static>,
     pub ctrlc: Option<Arc<AtomicBool>>,
 }
 
 impl ListStream {
     pub fn into_string(self, separator: &str, config: &Config) -> String {
-        self.map(|x: Value| x.into_string(", ", config))
+        self.map(|(x, _): (Value, _)| x.into_string(", ", config))
             .collect::<Vec<String>>()
             .join(separator)
     }
 
     pub fn from_stream(
-        input: impl Iterator<Item = Value> + Send + 'static,
+        input: impl Iterator<Item = impl Into<(Value, Option<ValueFormatter>)> + 'static>
+            + Send
+            + 'static,
         ctrlc: Option<Arc<AtomicBool>>,
     ) -> ListStream {
         ListStream {
-            stream: Box::new(input),
+            stream: Box::new(input.map(Into::into)),
             ctrlc,
         }
     }
@@ -200,7 +202,7 @@ impl Debug for ListStream {
 }
 
 impl Iterator for ListStream {
-    type Item = Value;
+    type Item = (Value, Option<ValueFormatter>);
 
     fn next(&mut self) -> Option<Self::Item> {
         if let Some(ctrlc) = &self.ctrlc {