WIP

TODO.md

@@ -30,6 +30,7 @@
 - [x] Handling rows with missing columns during a cell path
 - [x] finish operator type-checking
 - [x] Config file loading
+- [x] block variable captures
 - [ ] Input/output types
 - [ ] Support for `$in`
 - [ ] ctrl-c support

crates/nu-cli/src/completions.rs

@@ -77,6 +77,7 @@ impl Completer for NuCompleter {
 
                         let v: Vec<_> = match result {
                             Ok(pd) => pd
+                                .into_iter()
                                 .map(move |x| {
                                     let s = x.as_string().expect(
                                         "FIXME: better error handling for custom completions",

crates/nu-command/src/conversions/into/binary.rs

@@ -1,7 +1,7 @@
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
+    Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
 };
 
 #[derive(Clone)]
@@ -92,23 +92,21 @@ fn into_binary(
     let head = call.head;
     // let column_paths: Vec<CellPath> = call.rest(context, 0)?;
 
-    Ok(input
+    input.map(move |v| {
-        .map(move |v| {
+        action(v, head)
-            action(v, head)
+        // FIXME: Add back in cell_path support
-            // FIXME: Add back in cell_path support
+        // if column_paths.is_empty() {
-            // if column_paths.is_empty() {
+        //     action(v, head)
-            //     action(v, head)
+        // } else {
-            // } else {
+        //     let mut ret = v;
-            //     let mut ret = v;
+        //     for path in &column_paths {
-            //     for path in &column_paths {
+        //         ret =
-            //         ret =
+        //             ret.swap_data_by_cell_path(path, Box::new(move |old| action(old, old.tag())))?;
-            //             ret.swap_data_by_cell_path(path, Box::new(move |old| action(old, old.tag())))?;
+        //     }
-            //     }
 
-            //     Ok(ret)
+        //     Ok(ret)
-            // }
+        // }
-        })
+    })
-        .into_pipeline_data())
 }
 
 fn int_to_endian(n: i64) -> Vec<u8> {

crates/nu-command/src/conversions/into/filesize.rs

@@ -1,7 +1,7 @@
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
+    Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
 };
 
 #[derive(Clone)]
@@ -120,26 +120,24 @@ fn into_filesize(
     let head = call.head;
     // let call_paths: Vec<ColumnPath> = args.rest(0)?;
 
-    Ok(input
+    input.map(move |v| {
-        .map(move |v| {
+        action(v, head)
-            action(v, head)
 
-            // FIXME: Add back cell_path support
+        // FIXME: Add back cell_path support
-            // if column_paths.is_empty() {
+        // if column_paths.is_empty() {
-            //     action(&v, v.tag())
+        //     action(&v, v.tag())
-            // } else {
+        // } else {
-            //     let mut ret = v;
+        //     let mut ret = v;
-            //     for path in &column_paths {
+        //     for path in &column_paths {
-            //         ret = ret.swap_data_by_column_path(
+        //         ret = ret.swap_data_by_column_path(
-            //             path,
+        //             path,
-            //             Box::new(move |old| action(old, old.tag())),
+        //             Box::new(move |old| action(old, old.tag())),
-            //         )?;
+        //         )?;
-            //     }
+        //     }
 
-            //     Ok(ret)
+        //     Ok(ret)
-            // }
+        // }
-        })
+    })
-        .into_pipeline_data())
 }
 
 pub fn action(input: Value, span: Span) -> Value {

crates/nu-command/src/conversions/into/int.rs

@@ -1,7 +1,7 @@
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
+    Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
 };
 
 #[derive(Clone)]
@@ -96,23 +96,21 @@ fn into_int(
     let head = call.head;
     // let column_paths: Vec<CellPath> = call.rest(context, 0)?;
 
-    Ok(input
+    input.map(move |v| {
-        .map(move |v| {
+        action(v, head)
-            action(v, head)
+        // FIXME: Add back cell_path support
-            // FIXME: Add back cell_path support
+        // if column_paths.is_empty() {
-            // if column_paths.is_empty() {
+        //     action(&v, v.tag())
-            //     action(&v, v.tag())
+        // } else {
-            // } else {
+        //     let mut ret = v;
-            //     let mut ret = v;
+        //     for path in &column_paths {
-            //     for path in &column_paths {
+        //         ret = ret
-            //         ret = ret
+        //             .swap_data_by_column_path(path, Box::new(move |old| action(old, old.tag())))?;
-            //             .swap_data_by_column_path(path, Box::new(move |old| action(old, old.tag())))?;
+        //     }
-            //     }
 
-            //     Ok(ret)
+        //     Ok(ret)
-            // }
+        // }
-        })
+    })
-        .into_pipeline_data())
 }
 
 pub fn action(input: Value, span: Span) -> Value {

crates/nu-command/src/core_commands/for_.rs

@@ -60,7 +60,7 @@ impl Command for For {
         let mut stack = stack.collect_captures(&block.captures);
 
         match values {
-            Value::List { vals, span } => Ok(vals
+            Value::List { vals, .. } => Ok(vals
                 .into_iter()
                 .map(move |x| {
                     let mut stack = stack.clone();
@@ -68,25 +68,19 @@ impl Command for For {
 
                     let block = engine_state.get_block(block_id);
                     match eval_block(&engine_state, &mut stack, block, PipelineData::new()) {
-                        Ok(value) => Value::List {
+                        Ok(pipeline_data) => pipeline_data.into_value(),
-                            vals: value.collect(),
-                            span,
-                        },
                         Err(error) => Value::Error { error },
                     }
                 })
                 .into_pipeline_data()),
-            Value::Range { val, span } => Ok(val
+            Value::Range { val, .. } => Ok(val
                 .into_range_iter()?
                 .map(move |x| {
                     stack.add_var(var_id, x);
 
                     let block = engine_state.get_block(block_id);
                     match eval_block(&engine_state, &mut stack, block, PipelineData::new()) {
-                        Ok(value) => Value::List {
+                        Ok(pipeline_data) => pipeline_data.into_value(),
-                            vals: value.collect(),
-                            span,
-                        },
                         Err(error) => Value::Error { error },
                     }
                 })

crates/nu-command/src/filters/length.rs

@@ -32,7 +32,7 @@ impl Command for Length {
             }
             .into_pipeline_data()),
             _ => Ok(Value::Int {
-                val: input.count() as i64,
+                val: input.into_iter().count() as i64,
                 span: call.head,
             }
             .into_pipeline_data()),

crates/nu-command/src/strings/size.rs

@@ -4,9 +4,7 @@ use unicode_segmentation::UnicodeSegmentation;
 
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
-use nu_protocol::{
+use nu_protocol::{Example, PipelineData, ShellError, Signature, Span, Type, Value};
-    Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Type, Value,
-};
 
 #[derive(Clone)]
 pub struct Size;
@@ -104,18 +102,16 @@ impl Command for Size {
 
 fn size(call: &Call, input: PipelineData) -> Result<PipelineData, ShellError> {
     let span = call.head;
-    Ok(input
+    input.map(move |v| match v.as_string() {
-        .map(move |v| match v.as_string() {
+        Ok(s) => count(&s, span),
-            Ok(s) => count(&s, span),
+        Err(_) => Value::Error {
-            Err(_) => Value::Error {
+            error: ShellError::PipelineMismatch {
-                error: ShellError::PipelineMismatch {
+                expected: Type::String,
-                    expected: Type::String,
+                expected_span: span,
-                    expected_span: span,
+                origin: span,
-                    origin: span,
-                },
             },
-        })
+        },
-        .into_pipeline_data())
+    })
 }
 
 fn count(contents: &str, span: Span) -> Value {

crates/nu-command/src/strings/split/chars.rs

@@ -1,7 +1,7 @@
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Type, Value,
+    Example, PipelineData, ShellError, Signature, Span, Type, Value,
 };
 
 #[derive(Clone)]
@@ -54,9 +54,7 @@ fn split_chars(
 ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
     let span = call.head;
 
-    Ok(input
+    input.flat_map(move |x| split_chars_helper(&x, span))
-        .flat_map(move |x| split_chars_helper(&x, span))
-        .into_pipeline_data())
 }
 
 fn split_chars_helper(v: &Value, name: Span) -> Vec<Value> {

crates/nu-command/src/strings/split/column.rs

@@ -2,7 +2,7 @@ use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    IntoPipelineData, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Type, Value,
+    PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Type, Value,
 };
 
 #[derive(Clone)]
@@ -54,9 +54,7 @@ fn split_column(
     let rest: Vec<Spanned<String>> = call.rest(engine_state, stack, 1)?;
     let collapse_empty = call.has_flag("collapse-empty");
 
-    Ok(input
+    input.map(move |x| split_column_helper(&x, &separator, &rest, collapse_empty, name_span))
-        .map(move |x| split_column_helper(&x, &separator, &rest, collapse_empty, name_span))
-        .into_pipeline_data())
 }
 
 fn split_column_helper(

crates/nu-command/src/strings/split/row.rs

@@ -2,7 +2,7 @@ use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    IntoPipelineData, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Type, Value,
+    PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Type, Value,
 };
 
 #[derive(Clone)]
@@ -45,9 +45,7 @@ fn split_row(
     let name_span = call.head;
     let separator: Spanned<String> = call.req(engine_state, stack, 0)?;
 
-    Ok(input
+    input.flat_map(move |x| split_row_helper(&x, &separator, name_span))
-        .flat_map(move |x| split_row_helper(&x, &separator, name_span))
-        .into_pipeline_data())
 }
 
 fn split_row_helper(v: &Value, separator: &Spanned<String>, name: Span) -> Vec<Value> {

crates/nu-engine/src/eval.rs

@@ -226,26 +226,20 @@ pub fn eval_expression(
         Expr::RowCondition(_, expr) => eval_expression(engine_state, stack, expr),
         Expr::Call(call) => {
             // FIXME: protect this collect with ctrl-c
-            Ok(Value::List {
+            Ok(eval_call(engine_state, stack, call, PipelineData::new())?.into_value())
-                vals: eval_call(engine_state, stack, call, PipelineData::new())?.collect(),
-                span: expr.span,
-            })
         }
         Expr::ExternalCall(name, span, args) => {
             // FIXME: protect this collect with ctrl-c
-            Ok(Value::List {
+            Ok(eval_external(
-                vals: eval_external(
+                engine_state,
-                    engine_state,
+                stack,
-                    stack,
+                name,
-                    name,
+                span,
-                    span,
+                args,
-                    args,
+                PipelineData::new(),
-                    PipelineData::new(),
+                true,
-                    true,
+            )?
-                )?
+            .into_value())
-                .collect(),
-                span: expr.span,
-            })
         }
         Expr::Operator(_) => Ok(Value::Nothing { span: expr.span }),
         Expr::BinaryOp(lhs, op, rhs) => {
@@ -278,13 +272,8 @@ pub fn eval_expression(
         Expr::Subexpression(block_id) => {
             let block = engine_state.get_block(*block_id);
 
-            let mut stack = stack.collect_captures(&block.captures);
-
             // FIXME: protect this collect with ctrl-c
-            Ok(Value::List {
+            Ok(eval_block(engine_state, stack, block, PipelineData::new())?.into_value())
-                vals: eval_block(engine_state, &mut stack, block, PipelineData::new())?.collect(),
-                span: expr.span,
-            })
         }
         Expr::Block(block_id) => Ok(Value::Block {
             val: *block_id,

crates/nu-protocol/src/engine/stack.rs

@@ -40,11 +40,11 @@ impl Stack {
         let mut output = Stack::new();
 
         for capture in captures {
-            output.vars.insert(
+            // Note: this assumes we have calculated captures correctly and that commands
-                *capture,
+            // that take in a var decl will manually set this into scope when running the blocks
-                self.get_var(*capture)
+            if let Ok(value) = self.get_var(*capture) {
-                    .expect("internal error: capture of missing variable"),
+                output.vars.insert(*capture, value);
-            );
+            }
         }
 
         output

crates/nu-protocol/src/pipeline_data.rs

@@ -38,6 +38,51 @@ impl PipelineData {
             PipelineData::Value(v) => v.follow_cell_path(cell_path),
         }
     }
+
+    /// Simplified mapper to help with simple values also. For full iterator support use `.into_iter()` instead
+    pub fn map<F>(self, mut f: F) -> Result<PipelineData, ShellError>
+    where
+        Self: Sized,
+        F: FnMut(Value) -> Value + 'static + Send,
+    {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }) => {
+                Ok(vals.into_iter().map(f).into_pipeline_data())
+            }
+            PipelineData::Stream(stream) => Ok(stream.map(f).into_pipeline_data()),
+            PipelineData::Value(Value::Range { val, .. }) => {
+                Ok(val.into_range_iter()?.map(f).into_pipeline_data())
+            }
+            PipelineData::Value(v) => {
+                let output = f(v);
+                match output {
+                    Value::Error { error } => Err(error),
+                    v => Ok(v.into_pipeline_data()),
+                }
+            }
+        }
+    }
+
+    /// Simplified flatmapper. For full iterator support use `.into_iter()` instead
+    pub fn flat_map<U, F>(self, mut f: F) -> Result<PipelineData, ShellError>
+    where
+        Self: Sized,
+        U: IntoIterator<Item = Value>,
+        <U as IntoIterator>::IntoIter: 'static + Send,
+        F: FnMut(Value) -> U + 'static + Send,
+    {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }) => {
+                Ok(vals.into_iter().map(f).flatten().into_pipeline_data())
+            }
+            PipelineData::Stream(stream) => Ok(stream.map(f).flatten().into_pipeline_data()),
+            PipelineData::Value(Value::Range { val, .. }) => match val.into_range_iter() {
+                Ok(iter) => Ok(iter.map(f).flatten().into_pipeline_data()),
+                Err(error) => Err(error),
+            },
+            PipelineData::Value(v) => Ok(f(v).into_iter().into_pipeline_data()),
+        }
+    }
 }
 
 impl Default for PipelineData {
@@ -46,11 +91,28 @@ impl Default for PipelineData {
     }
 }
 
-impl Iterator for PipelineData {
+pub struct PipelineIterator(PipelineData);
+
+impl IntoIterator for PipelineData {
+    type Item = Value;
+
+    type IntoIter = PipelineIterator;
+
+    fn into_iter(self) -> Self::IntoIter {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }) => PipelineIterator(
+                PipelineData::Stream(ValueStream(Box::new(vals.into_iter()))),
+            ),
+            x => PipelineIterator(x),
+        }
+    }
+}
+
+impl Iterator for PipelineIterator {
     type Item = Value;
 
     fn next(&mut self) -> Option<Self::Item> {
-        match self {
+        match &mut self.0 {
             PipelineData::Value(Value::Nothing { .. }) => None,
             PipelineData::Value(v) => {
                 let prev = std::mem::take(v);