Split nu-cli into nu-cli/nu-engine (#2898)

We split off the evaluation engine part of nu-cli into its own crate. This helps improve build times for nu-cli by 17% in my tests. It also helps us see a bit better what's the core engine portion vs the part specific to the interactive CLI piece.

There's more than can be done here, but I think it's a good start in the right direction.

crates/nu-engine/src/call_info.rs

@@ -0,0 +1,27 @@
+use crate::evaluate::evaluate_args::evaluate_args;
+use crate::evaluation_context::EvaluationContext;
+use nu_errors::ShellError;
+use nu_protocol::hir;
+use nu_protocol::CallInfo;
+use nu_source::Tag;
+
+#[derive(Debug, Clone)]
+pub struct UnevaluatedCallInfo {
+    pub args: hir::Call,
+    pub name_tag: Tag,
+}
+
+impl UnevaluatedCallInfo {
+    pub async fn evaluate(self, ctx: &EvaluationContext) -> Result<CallInfo, ShellError> {
+        let args = evaluate_args(&self.args, ctx).await?;
+
+        Ok(CallInfo {
+            args,
+            name_tag: self.name_tag,
+        })
+    }
+
+    pub fn switch_present(&self, switch: &str) -> bool {
+        self.args.switch_preset(switch)
+    }
+}

crates/nu-engine/src/command_args.rs

@@ -0,0 +1,173 @@
+use crate::call_info::UnevaluatedCallInfo;
+use crate::deserializer::ConfigDeserializer;
+use crate::env::host::Host;
+use crate::evaluate::scope::Scope;
+use crate::evaluation_context::EvaluationContext;
+use crate::shell::shell_manager::ShellManager;
+use derive_new::new;
+use getset::Getters;
+use nu_errors::ShellError;
+use nu_protocol::EvaluatedArgs;
+use nu_protocol::{CallInfo, Value};
+use nu_source::Tag;
+use nu_stream::InputStream;
+use parking_lot::Mutex;
+use serde::Deserialize;
+use std::ops::Deref;
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
+
+#[derive(Getters)]
+#[get = "pub"]
+pub struct CommandArgs {
+    pub host: Arc<parking_lot::Mutex<Box<dyn Host>>>,
+    pub ctrl_c: Arc<AtomicBool>,
+    pub current_errors: Arc<Mutex<Vec<ShellError>>>,
+    pub shell_manager: ShellManager,
+    pub call_info: UnevaluatedCallInfo,
+    pub scope: Scope,
+    pub input: InputStream,
+}
+
+#[derive(Getters, Clone)]
+#[get = "pub"]
+pub struct RawCommandArgs {
+    pub host: Arc<parking_lot::Mutex<Box<dyn Host>>>,
+    pub ctrl_c: Arc<AtomicBool>,
+    pub current_errors: Arc<Mutex<Vec<ShellError>>>,
+    pub shell_manager: ShellManager,
+    pub scope: Scope,
+    pub call_info: UnevaluatedCallInfo,
+}
+
+impl RawCommandArgs {
+    pub fn with_input(self, input: impl Into<InputStream>) -> CommandArgs {
+        CommandArgs {
+            host: self.host,
+            ctrl_c: self.ctrl_c,
+            current_errors: self.current_errors,
+            shell_manager: self.shell_manager,
+            call_info: self.call_info,
+            scope: self.scope,
+            input: input.into(),
+        }
+    }
+}
+
+impl std::fmt::Debug for CommandArgs {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.call_info.fmt(f)
+    }
+}
+
+impl CommandArgs {
+    pub async fn evaluate_once(self) -> Result<EvaluatedWholeStreamCommandArgs, ShellError> {
+        let ctx = EvaluationContext::from_args(&self);
+        let host = self.host.clone();
+        let ctrl_c = self.ctrl_c.clone();
+        let shell_manager = self.shell_manager.clone();
+        let input = self.input;
+        let call_info = self.call_info.evaluate(&ctx).await?;
+        let scope = self.scope.clone();
+
+        Ok(EvaluatedWholeStreamCommandArgs::new(
+            host,
+            ctrl_c,
+            shell_manager,
+            call_info,
+            input,
+            scope,
+        ))
+    }
+
+    pub async fn process<'de, T: Deserialize<'de>>(self) -> Result<(T, InputStream), ShellError> {
+        let args = self.evaluate_once().await?;
+        let call_info = args.call_info.clone();
+
+        let mut deserializer = ConfigDeserializer::from_call_info(call_info);
+
+        Ok((T::deserialize(&mut deserializer)?, args.input))
+    }
+}
+
+pub struct EvaluatedWholeStreamCommandArgs {
+    pub args: EvaluatedCommandArgs,
+    pub input: InputStream,
+}
+
+impl Deref for EvaluatedWholeStreamCommandArgs {
+    type Target = EvaluatedCommandArgs;
+    fn deref(&self) -> &Self::Target {
+        &self.args
+    }
+}
+
+impl EvaluatedWholeStreamCommandArgs {
+    pub fn new(
+        host: Arc<parking_lot::Mutex<dyn Host>>,
+        ctrl_c: Arc<AtomicBool>,
+        shell_manager: ShellManager,
+        call_info: CallInfo,
+        input: impl Into<InputStream>,
+        scope: Scope,
+    ) -> EvaluatedWholeStreamCommandArgs {
+        EvaluatedWholeStreamCommandArgs {
+            args: EvaluatedCommandArgs {
+                host,
+                ctrl_c,
+                shell_manager,
+                call_info,
+                scope,
+            },
+            input: input.into(),
+        }
+    }
+
+    pub fn name_tag(&self) -> Tag {
+        self.args.call_info.name_tag.clone()
+    }
+
+    pub fn parts(self) -> (InputStream, EvaluatedArgs) {
+        let EvaluatedWholeStreamCommandArgs { args, input } = self;
+
+        (input, args.call_info.args)
+    }
+
+    pub fn split(self) -> (InputStream, EvaluatedCommandArgs) {
+        let EvaluatedWholeStreamCommandArgs { args, input } = self;
+
+        (input, args)
+    }
+}
+
+#[derive(Getters, new)]
+#[get = "pub(crate)"]
+pub struct EvaluatedCommandArgs {
+    pub host: Arc<parking_lot::Mutex<dyn Host>>,
+    pub ctrl_c: Arc<AtomicBool>,
+    pub shell_manager: ShellManager,
+    pub call_info: CallInfo,
+    pub scope: Scope,
+}
+
+impl EvaluatedCommandArgs {
+    pub fn nth(&self, pos: usize) -> Option<&Value> {
+        self.call_info.args.nth(pos)
+    }
+
+    /// Get the nth positional argument, error if not possible
+    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
+        self.call_info
+            .args
+            .nth(pos)
+            .ok_or_else(|| ShellError::unimplemented("Better error: expect_nth"))
+    }
+
+    pub fn get(&self, name: &str) -> Option<&Value> {
+        self.call_info.args.get(name)
+    }
+
+    pub fn has(&self, name: &str) -> bool {
+        self.call_info.args.has(name)
+    }
+}

crates/nu-engine/src/deserializer.rs

@@ -0,0 +1,609 @@
+use log::trace;
+use nu_errors::{CoerceInto, ShellError};
+use nu_protocol::{
+    hir::CapturedBlock, CallInfo, ColumnPath, Primitive, RangeInclusion, ShellTypeName,
+    UntaggedValue, Value,
+};
+use nu_source::{HasSpan, Spanned, SpannedItem, Tagged, TaggedItem};
+use nu_value_ext::ValueExt;
+use serde::de;
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+#[derive(Copy, Clone, Deserialize, Serialize)]
+pub struct NumericRange {
+    pub from: (Option<Spanned<u64>>, RangeInclusion),
+    pub to: (Option<Spanned<u64>>, RangeInclusion),
+}
+
+impl NumericRange {
+    pub fn min(self) -> u64 {
+        match self.from.1 {
+            RangeInclusion::Inclusive => self.from.0.map(|from| *from).unwrap_or(0),
+            RangeInclusion::Exclusive => {
+                self.from.0.map(|from| *from).unwrap_or(0).saturating_add(1)
+            }
+        }
+    }
+
+    pub fn max(self) -> u64 {
+        match self.to.1 {
+            RangeInclusion::Inclusive => self.to.0.map(|to| *to).unwrap_or(u64::MAX),
+            RangeInclusion::Exclusive => self
+                .to
+                .0
+                .map(|to| *to)
+                .unwrap_or(u64::MAX)
+                .saturating_sub(1),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct DeserializerItem<'de> {
+    key_struct_field: Option<(String, &'de str)>,
+    val: Value,
+}
+
+pub struct ConfigDeserializer<'de> {
+    call: CallInfo,
+    stack: Vec<DeserializerItem<'de>>,
+    saw_root: bool,
+    position: usize,
+}
+
+impl<'de> ConfigDeserializer<'de> {
+    pub fn from_call_info(call: CallInfo) -> ConfigDeserializer<'de> {
+        ConfigDeserializer {
+            call,
+            stack: vec![],
+            saw_root: false,
+            position: 0,
+        }
+    }
+
+    pub fn push_val(&mut self, val: Value) {
+        self.stack.push(DeserializerItem {
+            key_struct_field: None,
+            val,
+        });
+    }
+
+    pub fn push(&mut self, name: &'static str) -> Result<(), ShellError> {
+        let value: Option<Value> = if name == "rest" {
+            let positional = self.call.args.slice_from(self.position);
+            self.position += positional.len();
+            Some(UntaggedValue::Table(positional).into_untagged_value()) // TODO: correct tag
+        } else if self.call.args.has(name) {
+            self.call.args.get(name).cloned()
+        } else {
+            let position = self.position;
+            self.position += 1;
+            self.call.args.nth(position).cloned()
+        };
+
+        trace!("pushing {:?}", value);
+
+        self.stack.push(DeserializerItem {
+            key_struct_field: Some((name.to_string(), name)),
+            val: value.unwrap_or_else(|| UntaggedValue::nothing().into_value(&self.call.name_tag)),
+        });
+
+        Ok(())
+    }
+
+    pub fn top(&mut self) -> &DeserializerItem {
+        let value = self.stack.last();
+        trace!("inspecting top value :: {:?}", value);
+        value.expect("Can't get top element of an empty stack")
+    }
+
+    pub fn pop(&mut self) -> DeserializerItem {
+        let value = self.stack.pop();
+        trace!("popping value :: {:?}", value);
+        value.expect("Can't pop an empty stack")
+    }
+}
+
+use de::Visitor;
+
+impl<'de, 'a> de::Deserializer<'de> for &'a mut ConfigDeserializer<'de> {
+    type Error = ShellError;
+    fn deserialize_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_any")
+    }
+    fn deserialize_bool<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        let value = self.pop();
+        trace!("Extracting {:?} for bool", value.val);
+
+        match &value.val {
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Boolean(b)),
+                ..
+            } => visitor.visit_bool(*b),
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Nothing),
+                ..
+            } => visitor.visit_bool(false),
+            other => Err(ShellError::type_error(
+                "Boolean",
+                other.type_name().spanned(other.span()),
+            )),
+        }
+    }
+    fn deserialize_i8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_i8")
+    }
+    fn deserialize_i16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_i16")
+    }
+    fn deserialize_i32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_i32")
+    }
+    fn deserialize_i64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_i64")
+    }
+    fn deserialize_u8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_u8")
+    }
+    fn deserialize_u16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_u16")
+    }
+    fn deserialize_u32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_u32")
+    }
+    fn deserialize_u64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_u64")
+    }
+    fn deserialize_f32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_f32")
+    }
+    fn deserialize_f64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_f64")
+    }
+    fn deserialize_char<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_char")
+    }
+    fn deserialize_str<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_str")
+    }
+    fn deserialize_string<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_string")
+    }
+    fn deserialize_bytes<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_bytes")
+    }
+    fn deserialize_byte_buf<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_byte_buf")
+    }
+
+    fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        let value = self.top();
+        let name = std::any::type_name::<V::Value>();
+        trace!("<Option> Extracting {:?} for Option<{}>", value, name);
+        match &value.val.value {
+            UntaggedValue::Primitive(Primitive::Nothing) => visitor.visit_none(),
+            _ => visitor.visit_some(self),
+        }
+    }
+
+    fn deserialize_unit<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_unit")
+    }
+    fn deserialize_unit_struct<V>(
+        self,
+        _name: &'static str,
+        _visitor: V,
+    ) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_unit_struct")
+    }
+    fn deserialize_newtype_struct<V>(
+        self,
+        _name: &'static str,
+        _visitor: V,
+    ) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_newtype_struct")
+    }
+    fn deserialize_seq<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        let value = self.pop();
+        trace!("<Vec> Extracting {:?} for vec", value.val);
+
+        match value.val.into_parts() {
+            (UntaggedValue::Table(items), _) => {
+                let de = SeqDeserializer::new(&mut self, items.into_iter());
+                visitor.visit_seq(de)
+            }
+            (other, tag) => Err(ShellError::type_error(
+                "Vec",
+                other.type_name().spanned(tag),
+            )),
+        }
+    }
+    fn deserialize_tuple<V>(mut self, len: usize, visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        let value = self.pop();
+        trace!(
+            "<Tuple> Extracting {:?} for tuple with {} elements",
+            value.val,
+            len
+        );
+
+        match value.val.into_parts() {
+            (UntaggedValue::Table(items), _) => {
+                let de = SeqDeserializer::new(&mut self, items.into_iter());
+                visitor.visit_seq(de)
+            }
+            (other, tag) => Err(ShellError::type_error(
+                "Tuple",
+                other.type_name().spanned(tag),
+            )),
+        }
+    }
+    fn deserialize_tuple_struct<V>(
+        self,
+        _name: &'static str,
+        _len: usize,
+        _visitor: V,
+    ) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_tuple_struct")
+    }
+    fn deserialize_map<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_map")
+    }
+    fn deserialize_struct<V>(
+        mut self,
+        name: &'static str,
+        fields: &'static [&'static str],
+        visitor: V,
+    ) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        fn visit<'de, T, V>(
+            val: T,
+            name: &'static str,
+            fields: &'static [&'static str],
+            visitor: V,
+        ) -> Result<V::Value, ShellError>
+        where
+            T: serde::Serialize,
+            V: Visitor<'de>,
+        {
+            let json = serde_json::to_string(&val)?;
+            let json_cursor = std::io::Cursor::new(json.into_bytes());
+            let mut json_de = serde_json::Deserializer::from_reader(json_cursor);
+            let r = json_de.deserialize_struct(name, fields, visitor)?;
+            Ok(r)
+        }
+
+        trace!(
+            "deserializing struct {:?} {:?} (saw_root={} stack={:?})",
+            name,
+            fields,
+            self.saw_root,
+            self.stack
+        );
+
+        if !self.saw_root {
+            self.saw_root = true;
+            return visitor.visit_seq(StructDeserializer::new(&mut self, fields));
+        }
+
+        let value = self.pop();
+
+        let type_name = std::any::type_name::<V::Value>();
+        let tagged_val_name = std::any::type_name::<Value>();
+
+        trace!(
+            "name={} type_name={} tagged_val_name={}",
+            name,
+            type_name,
+            tagged_val_name
+        );
+
+        if type_name == tagged_val_name {
+            return visit::<Value, _>(value.val, name, fields, visitor);
+        }
+
+        if name == "CapturedBlock" {
+            let block = match value.val {
+                Value {
+                    value: UntaggedValue::Block(block),
+                    ..
+                } => block,
+                other => {
+                    return Err(ShellError::type_error(
+                        "Block",
+                        other.type_name().spanned(other.span()),
+                    ))
+                }
+            };
+            return visit::<CapturedBlock, _>(*block, name, fields, visitor);
+        }
+
+        if name == "ColumnPath" {
+            let path = match value.val {
+                Value {
+                    value: UntaggedValue::Primitive(Primitive::ColumnPath(path)),
+                    ..
+                } => path,
+                other => {
+                    return Err(ShellError::type_error(
+                        "column path",
+                        other.type_name().spanned(other.span()),
+                    ))
+                }
+            };
+
+            return visit::<ColumnPath, _>(path, name, fields, visitor);
+        }
+
+        trace!("Extracting {:?} for {:?}", value.val, type_name);
+
+        let tag = value.val.tag();
+        match value.val {
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Boolean(b)),
+                ..
+            } => visit::<Tagged<bool>, _>(b.tagged(tag), name, fields, visitor),
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Nothing),
+                ..
+            } => visit::<Tagged<bool>, _>(false.tagged(tag), name, fields, visitor),
+            Value {
+                value: UntaggedValue::Primitive(Primitive::FilePath(p)),
+                ..
+            } => visit::<Tagged<PathBuf>, _>(p.tagged(tag), name, fields, visitor),
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Int(int)),
+                ..
+            } => {
+                let i: i64 = int.tagged(value.val.tag).coerce_into("converting to i64")?;
+                visit::<Tagged<i64>, _>(i.tagged(tag), name, fields, visitor)
+            }
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Duration(big_int)),
+                ..
+            } => {
+                let u_int: u64 = big_int
+                    .tagged(value.val.tag)
+                    .coerce_into("converting to u64")?;
+                visit::<Tagged<u64>, _>(u_int.tagged(tag), name, fields, visitor)
+            }
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Decimal(decimal)),
+                ..
+            } => {
+                let i: f64 = decimal
+                    .tagged(value.val.tag)
+                    .coerce_into("converting to f64")?;
+                visit::<Tagged<f64>, _>(i.tagged(tag), name, fields, visitor)
+            }
+            Value {
+                value: UntaggedValue::Primitive(Primitive::String(string)),
+                ..
+            } => visit::<Tagged<String>, _>(string.tagged(tag), name, fields, visitor),
+            Value {
+                value: UntaggedValue::Primitive(Primitive::Range(range)),
+                ..
+            } => {
+                let (left, left_inclusion) = range.from;
+                let (right, right_inclusion) = range.to;
+                let left_span = left.span;
+                let right_span = right.span;
+
+                let left = match left.item {
+                    Primitive::Nothing => None,
+                    _ => Some(left.as_u64(left_span)?),
+                };
+                let right = match right.item {
+                    Primitive::Nothing => None,
+                    _ => Some(right.as_u64(right_span)?),
+                };
+
+                let numeric_range = NumericRange {
+                    from: (left.map(|left| left.spanned(left_span)), left_inclusion),
+                    to: (
+                        right.map(|right| right.spanned(right_span)),
+                        right_inclusion,
+                    ),
+                };
+
+                visit::<Tagged<NumericRange>, _>(numeric_range.tagged(tag), name, fields, visitor)
+            }
+
+            other => Err(ShellError::type_error(
+                name,
+                other.type_name().spanned(other.span()),
+            )),
+        }
+    }
+    fn deserialize_enum<V>(
+        self,
+        _name: &'static str,
+        _variants: &'static [&'static str],
+        _visitor: V,
+    ) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_enum")
+    }
+    fn deserialize_identifier<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_identifier")
+    }
+    fn deserialize_ignored_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+    where
+        V: Visitor<'de>,
+    {
+        unimplemented!("deserialize_ignored_any")
+    }
+}
+
+struct SeqDeserializer<'a, 'de: 'a, I: Iterator<Item = Value>> {
+    de: &'a mut ConfigDeserializer<'de>,
+    vals: I,
+}
+
+impl<'a, 'de: 'a, I: Iterator<Item = Value>> SeqDeserializer<'a, 'de, I> {
+    fn new(de: &'a mut ConfigDeserializer<'de>, vals: I) -> Self {
+        SeqDeserializer { de, vals }
+    }
+}
+
+impl<'a, 'de: 'a, I: Iterator<Item = Value>> de::SeqAccess<'de> for SeqDeserializer<'a, 'de, I> {
+    type Error = ShellError;
+
+    fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>
+    where
+        T: de::DeserializeSeed<'de>,
+    {
+        let next = if let Some(next) = self.vals.next() {
+            next
+        } else {
+            return Ok(None);
+        };
+
+        self.de.push_val(next);
+        seed.deserialize(&mut *self.de).map(Some)
+    }
+
+    fn size_hint(&self) -> Option<usize> {
+        self.vals.size_hint().1
+    }
+}
+
+struct StructDeserializer<'a, 'de: 'a> {
+    de: &'a mut ConfigDeserializer<'de>,
+    fields: &'static [&'static str],
+}
+
+impl<'a, 'de: 'a> StructDeserializer<'a, 'de> {
+    fn new(de: &'a mut ConfigDeserializer<'de>, fields: &'static [&'static str]) -> Self {
+        StructDeserializer { de, fields }
+    }
+}
+
+impl<'a, 'de: 'a> de::SeqAccess<'de> for StructDeserializer<'a, 'de> {
+    type Error = ShellError;
+
+    fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>
+    where
+        T: de::DeserializeSeed<'de>,
+    {
+        if self.fields.is_empty() {
+            return Ok(None);
+        }
+
+        trace!("Processing {}", self.fields[0]);
+
+        self.de.push(self.fields[0])?;
+        self.fields = &self.fields[1..];
+        seed.deserialize(&mut *self.de).map(Some)
+    }
+
+    fn size_hint(&self) -> Option<usize> {
+        Some(self.fields.len())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::any::type_name;
+    #[test]
+    fn check_type_name_properties() {
+        // This ensures that certain properties for the
+        // std::any::type_name function hold, that
+        // this code relies on. The type_name docs explicitly
+        // mention that the actual format of the output
+        // is unspecified and change is likely.
+        // This test makes sure that such change is detected
+        // by this test failing, and not things silently breaking.
+        // Specifically, we rely on this behavior further above
+        // in the file for the Value special case parsing.
+        let tuple = type_name::<()>();
+        let tagged_tuple = type_name::<Tagged<()>>();
+        let tagged_value = type_name::<Value>();
+        assert!(tuple != tagged_tuple);
+        assert!(tuple != tagged_value);
+        assert!(tagged_tuple != tagged_value);
+    }
+}

crates/nu-engine/src/documentation.rs

@@ -0,0 +1,296 @@
+use crate::evaluate::scope::Scope;
+use crate::whole_stream_command::WholeStreamCommand;
+use indexmap::IndexMap;
+use itertools::Itertools;
+use nu_protocol::{NamedType, PositionalType, Signature, UntaggedValue, Value};
+use nu_source::PrettyDebug;
+use std::collections::HashMap;
+
+const COMMANDS_DOCS_DIR: &str = "docs/commands";
+
+pub struct DocumentationConfig {
+    no_subcommands: bool,
+    no_colour: bool,
+}
+
+impl Default for DocumentationConfig {
+    fn default() -> Self {
+        DocumentationConfig {
+            no_subcommands: false,
+            no_colour: false,
+        }
+    }
+}
+
+fn generate_doc(name: &str, scope: &Scope) -> IndexMap<String, Value> {
+    let mut row_entries = IndexMap::new();
+    let command = scope
+        .get_command(name)
+        .unwrap_or_else(|| panic!("Expected command '{}' from names to be in registry", name));
+    row_entries.insert(
+        "name".to_owned(),
+        UntaggedValue::string(name).into_untagged_value(),
+    );
+    row_entries.insert(
+        "usage".to_owned(),
+        UntaggedValue::string(command.usage()).into_untagged_value(),
+    );
+    retrieve_doc_link(name).and_then(|link| {
+        row_entries.insert(
+            "doc_link".to_owned(),
+            UntaggedValue::string(link).into_untagged_value(),
+        )
+    });
+    row_entries.insert(
+        "documentation".to_owned(),
+        UntaggedValue::string(get_documentation(
+            command.stream_command(),
+            scope,
+            &DocumentationConfig {
+                no_subcommands: true,
+                no_colour: true,
+            },
+        ))
+        .into_untagged_value(),
+    );
+    row_entries
+}
+
+// generate_docs gets the documentation from each command and returns a Table as output
+pub fn generate_docs(scope: &Scope) -> Value {
+    let mut sorted_names = scope.get_command_names();
+    sorted_names.sort();
+
+    // cmap will map parent commands to it's subcommands e.g. to -> [to csv, to yaml, to bson]
+    let mut cmap: HashMap<String, Vec<String>> = HashMap::new();
+    for name in &sorted_names {
+        if name.contains(' ') {
+            let split_name = name.split_whitespace().collect_vec();
+            let parent_name = split_name.first().expect("Expected a parent command name");
+            let sub_names = cmap
+                .get_mut(*parent_name)
+                .expect("Expected a entry for parent");
+            sub_names.push(name.to_owned());
+        } else {
+            cmap.insert(name.to_owned(), Vec::new());
+        };
+    }
+    // Return documentation for each command
+    // Sub-commands are nested under their respective parent commands
+    let mut table = Vec::new();
+    for name in sorted_names.iter() {
+        // Must be a sub-command, skip since it's being handled underneath when we hit the parent command
+        if !cmap.contains_key(name) {
+            continue;
+        }
+        let mut row_entries = generate_doc(name, scope);
+        // Iterate over all the subcommands of the parent command
+        let mut sub_table = Vec::new();
+        for sub_name in cmap.get(name).unwrap_or(&Vec::new()).iter() {
+            let sub_row = generate_doc(sub_name, scope);
+            sub_table.push(UntaggedValue::row(sub_row).into_untagged_value());
+        }
+
+        if !sub_table.is_empty() {
+            row_entries.insert(
+                "subcommands".to_owned(),
+                UntaggedValue::table(&sub_table).into_untagged_value(),
+            );
+        }
+        table.push(UntaggedValue::row(row_entries).into_untagged_value());
+    }
+    UntaggedValue::table(&table).into_untagged_value()
+}
+
+fn retrieve_doc_link(name: &str) -> Option<String> {
+    let doc_name = name.split_whitespace().join("_"); // Because .replace(" ", "_") didn't work
+    let mut entries =
+        std::fs::read_dir(COMMANDS_DOCS_DIR).expect("Directory for command docs are missing!");
+    entries.find_map(|r| {
+        r.map_or(None, |de| {
+            if de.file_name().to_string_lossy() == format!("{}.{}", &doc_name, "md") {
+                Some(format!("/commands/{}.{}", &doc_name, "html"))
+            } else {
+                None
+            }
+        })
+    })
+}
+
+#[allow(clippy::cognitive_complexity)]
+pub fn get_documentation(
+    cmd: &dyn WholeStreamCommand,
+    scope: &Scope,
+    config: &DocumentationConfig,
+) -> String {
+    let cmd_name = cmd.name();
+    let signature = cmd.signature();
+    let mut long_desc = String::new();
+
+    long_desc.push_str(&cmd.usage());
+    long_desc.push('\n');
+
+    let mut subcommands = vec![];
+    if !config.no_subcommands {
+        for name in scope.get_command_names() {
+            if name.starts_with(&format!("{} ", cmd_name)) {
+                let subcommand = scope.get_command(&name).expect("This shouldn't happen");
+
+                subcommands.push(format!("  {} - {}", name, subcommand.usage()));
+            }
+        }
+    }
+
+    let mut one_liner = String::new();
+    one_liner.push_str(&signature.name);
+    one_liner.push(' ');
+
+    for positional in &signature.positional {
+        match &positional.0 {
+            PositionalType::Mandatory(name, _m) => {
+                one_liner.push_str(&format!("<{}> ", name));
+            }
+            PositionalType::Optional(name, _o) => {
+                one_liner.push_str(&format!("({}) ", name));
+            }
+        }
+    }
+
+    if signature.rest_positional.is_some() {
+        one_liner.push_str(" ...args");
+    }
+
+    if !subcommands.is_empty() {
+        one_liner.push_str("<subcommand> ");
+    }
+
+    if !signature.named.is_empty() {
+        one_liner.push_str("{flags} ");
+    }
+
+    long_desc.push_str(&format!("\nUsage:\n  > {}\n", one_liner));
+
+    if !subcommands.is_empty() {
+        long_desc.push_str("\nSubcommands:\n");
+        subcommands.sort();
+        long_desc.push_str(&subcommands.join("\n"));
+        long_desc.push('\n');
+    }
+
+    if !signature.positional.is_empty() || signature.rest_positional.is_some() {
+        long_desc.push_str("\nParameters:\n");
+        for positional in &signature.positional {
+            match &positional.0 {
+                PositionalType::Mandatory(name, _m) => {
+                    long_desc.push_str(&format!("  <{}> {}\n", name, positional.1));
+                }
+                PositionalType::Optional(name, _o) => {
+                    long_desc.push_str(&format!("  ({}) {}\n", name, positional.1));
+                }
+            }
+        }
+
+        if let Some(rest_positional) = &signature.rest_positional {
+            long_desc.push_str(&format!("  ...args: {}\n", rest_positional.1));
+        }
+    }
+    if !signature.named.is_empty() {
+        long_desc.push_str(&get_flags_section(&signature))
+    }
+
+    let palette = crate::shell::palette::DefaultPalette {};
+    let examples = cmd.examples();
+    if !examples.is_empty() {
+        long_desc.push_str("\nExamples:");
+    }
+    for example in examples {
+        long_desc.push('\n');
+        long_desc.push_str("  ");
+        long_desc.push_str(example.description);
+
+        if config.no_colour {
+            long_desc.push_str(&format!("\n  > {}\n", example.example));
+        } else {
+            let colored_example =
+                crate::shell::painter::Painter::paint_string(example.example, scope, &palette);
+            long_desc.push_str(&format!("\n  > {}\n", colored_example));
+        }
+    }
+
+    long_desc.push('\n');
+
+    long_desc
+}
+
+fn get_flags_section(signature: &Signature) -> String {
+    let mut long_desc = String::new();
+    long_desc.push_str("\nFlags:\n");
+    for (flag, ty) in &signature.named {
+        let msg = match ty.0 {
+            NamedType::Switch(s) => {
+                if let Some(c) = s {
+                    format!(
+                        "  -{}, --{}{} {}\n",
+                        c,
+                        flag,
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                } else {
+                    format!(
+                        "  --{}{} {}\n",
+                        flag,
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                }
+            }
+            NamedType::Mandatory(s, m) => {
+                if let Some(c) = s {
+                    format!(
+                        "  -{}, --{} <{}> (required parameter){} {}\n",
+                        c,
+                        flag,
+                        m.display(),
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                } else {
+                    format!(
+                        "  --{} <{}> (required parameter){} {}\n",
+                        flag,
+                        m.display(),
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                }
+            }
+            NamedType::Optional(s, o) => {
+                if let Some(c) = s {
+                    format!(
+                        "  -{}, --{} <{}>{} {}\n",
+                        c,
+                        flag,
+                        o.display(),
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                } else {
+                    format!(
+                        "  --{} <{}>{} {}\n",
+                        flag,
+                        o.display(),
+                        if !ty.1.is_empty() { ":" } else { "" },
+                        ty.1
+                    )
+                }
+            }
+        };
+        long_desc.push_str(&msg);
+    }
+    long_desc
+}
+
+pub fn get_help(cmd: &dyn WholeStreamCommand, scope: &Scope) -> String {
+    get_documentation(cmd, scope, &DocumentationConfig::default())
+}

crates/nu-engine/src/env/environment.rs

@@ -0,0 +1,30 @@
+use nu_protocol::Value;
+use std::ffi::OsString;
+
+use std::fmt::Debug;
+
+pub trait Env: Debug + Send {
+    fn env(&self) -> Option<Value>;
+    fn path(&self) -> Option<Value>;
+
+    fn add_env(&mut self, key: &str, value: &str);
+    fn add_path(&mut self, new_path: OsString);
+}
+
+impl Env for Box<dyn Env> {
+    fn env(&self) -> Option<Value> {
+        (**self).env()
+    }
+
+    fn path(&self) -> Option<Value> {
+        (**self).path()
+    }
+
+    fn add_env(&mut self, key: &str, value: &str) {
+        (**self).add_env(key, value);
+    }
+
+    fn add_path(&mut self, new_path: OsString) {
+        (**self).add_path(new_path);
+    }
+}

crates/nu-engine/src/env/host.rs

@@ -0,0 +1,127 @@
+use indexmap::IndexMap;
+use std::ffi::OsString;
+use std::fmt::Debug;
+
+pub trait Host: Debug + Send {
+    fn out_termcolor(&self) -> termcolor::StandardStream;
+    fn err_termcolor(&self) -> termcolor::StandardStream;
+
+    fn stdout(&mut self, out: &str);
+    fn stderr(&mut self, out: &str);
+
+    fn vars(&mut self) -> Vec<(String, String)>;
+    fn env_get(&mut self, key: OsString) -> Option<OsString>;
+    fn env_set(&mut self, k: OsString, v: OsString);
+    fn env_rm(&mut self, k: OsString);
+
+    fn width(&self) -> usize;
+}
+
+impl Host for Box<dyn Host> {
+    fn stdout(&mut self, out: &str) {
+        (**self).stdout(out)
+    }
+
+    fn stderr(&mut self, out: &str) {
+        (**self).stderr(out)
+    }
+
+    fn vars(&mut self) -> Vec<(String, String)> {
+        (**self).vars()
+    }
+
+    fn env_get(&mut self, key: OsString) -> Option<OsString> {
+        (**self).env_get(key)
+    }
+
+    fn env_set(&mut self, key: OsString, value: OsString) {
+        (**self).env_set(key, value);
+    }
+
+    fn env_rm(&mut self, key: OsString) {
+        (**self).env_rm(key)
+    }
+
+    fn out_termcolor(&self) -> termcolor::StandardStream {
+        (**self).out_termcolor()
+    }
+
+    fn err_termcolor(&self) -> termcolor::StandardStream {
+        (**self).err_termcolor()
+    }
+
+    fn width(&self) -> usize {
+        (**self).width()
+    }
+}
+
+#[derive(Debug)]
+pub struct FakeHost {
+    line_written: String,
+    env_vars: IndexMap<String, String>,
+}
+
+impl FakeHost {
+    pub fn new() -> FakeHost {
+        FakeHost {
+            line_written: String::from(""),
+            env_vars: IndexMap::default(),
+        }
+    }
+}
+
+impl Default for FakeHost {
+    fn default() -> Self {
+        FakeHost::new()
+    }
+}
+
+impl Host for FakeHost {
+    fn stdout(&mut self, out: &str) {
+        self.line_written = out.to_string();
+    }
+
+    fn stderr(&mut self, out: &str) {
+        self.line_written = out.to_string();
+    }
+
+    fn vars(&mut self) -> Vec<(String, String)> {
+        self.env_vars
+            .iter()
+            .map(|(k, v)| (k.clone(), v.clone()))
+            .collect::<Vec<_>>()
+    }
+
+    fn env_get(&mut self, key: OsString) -> Option<OsString> {
+        let key = key.into_string().expect("Couldn't convert to string.");
+
+        match self.env_vars.get(&key) {
+            Some(env) => Some(OsString::from(env)),
+            None => None,
+        }
+    }
+
+    fn env_set(&mut self, key: OsString, value: OsString) {
+        self.env_vars.insert(
+            key.into_string().expect("Couldn't convert to string."),
+            value.into_string().expect("Couldn't convert to string."),
+        );
+    }
+
+    fn env_rm(&mut self, key: OsString) {
+        self.env_vars
+            .shift_remove(&key.into_string().expect("Couldn't convert to string."));
+    }
+
+    fn out_termcolor(&self) -> termcolor::StandardStream {
+        termcolor::StandardStream::stdout(termcolor::ColorChoice::Auto)
+    }
+
+    fn err_termcolor(&self) -> termcolor::StandardStream {
+        termcolor::StandardStream::stderr(termcolor::ColorChoice::Auto)
+    }
+
+    fn width(&self) -> usize {
+        1
+    }
+}

crates/nu-engine/src/env/mod.rs

@@ -0,0 +1,2 @@
+pub(crate) mod environment;
+pub(crate) mod host;

crates/nu-engine/src/evaluate/block.rs

@@ -0,0 +1,216 @@
+use crate::evaluate::expr::run_expression_block;
+use crate::evaluate::internal::run_internal_command;
+use crate::evaluation_context::EvaluationContext;
+use async_recursion::async_recursion;
+use futures::stream::TryStreamExt;
+use nu_errors::ShellError;
+use nu_parser::ParserScope;
+use nu_protocol::hir::{
+    Block, Call, ClassifiedCommand, Expression, Pipeline, SpannedExpression, Synthetic,
+};
+use nu_protocol::{ReturnSuccess, UntaggedValue, Value};
+use nu_source::{Span, Tag};
+use nu_stream::InputStream;
+use nu_stream::ToOutputStream;
+use std::sync::atomic::Ordering;
+
+#[async_recursion]
+pub async fn run_block(
+    block: &Block,
+    ctx: &EvaluationContext,
+    mut input: InputStream,
+) -> Result<InputStream, ShellError> {
+    let mut output: Result<InputStream, ShellError> = Ok(InputStream::empty());
+    for (_, definition) in block.definitions.iter() {
+        ctx.scope.add_definition(definition.clone());
+    }
+
+    for group in &block.block {
+        match output {
+            Ok(inp) if inp.is_empty() => {}
+            Ok(inp) => {
+                // Run autoview on the values we've seen so far
+                // We may want to make this configurable for other kinds of hosting
+                if let Some(autoview) = ctx.get_command("autoview") {
+                    let mut output_stream = match ctx
+                        .run_command(
+                            autoview,
+                            Tag::unknown(),
+                            Call::new(
+                                Box::new(SpannedExpression::new(
+                                    Expression::Synthetic(Synthetic::String("autoview".into())),
+                                    Span::unknown(),
+                                )),
+                                Span::unknown(),
+                            ),
+                            inp,
+                        )
+                        .await
+                    {
+                        Ok(x) => x,
+                        Err(e) => {
+                            return Err(e);
+                        }
+                    };
+                    match output_stream.try_next().await {
+                        Ok(Some(ReturnSuccess::Value(Value {
+                            value: UntaggedValue::Error(e),
+                            ..
+                        }))) => {
+                            return Err(e);
+                        }
+                        Ok(Some(_item)) => {
+                            if let Some(err) = ctx.get_errors().get(0) {
+                                ctx.clear_errors();
+                                return Err(err.clone());
+                            }
+                            if ctx.ctrl_c.load(Ordering::SeqCst) {
+                                return Ok(InputStream::empty());
+                            }
+                        }
+                        Ok(None) => {
+                            if let Some(err) = ctx.get_errors().get(0) {
+                                ctx.clear_errors();
+                                return Err(err.clone());
+                            }
+                        }
+                        Err(e) => {
+                            return Err(e);
+                        }
+                    }
+                }
+            }
+            Err(e) => {
+                return Err(e);
+            }
+        }
+        output = Ok(InputStream::empty());
+        for pipeline in &group.pipelines {
+            match output {
+                Ok(inp) if inp.is_empty() => {}
+                Ok(inp) => {
+                    let mut output_stream = inp.to_output_stream();
+
+                    match output_stream.try_next().await {
+                        Ok(Some(ReturnSuccess::Value(Value {
+                            value: UntaggedValue::Error(e),
+                            ..
+                        }))) => {
+                            return Err(e);
+                        }
+                        Ok(Some(_item)) => {
+                            if let Some(err) = ctx.get_errors().get(0) {
+                                ctx.clear_errors();
+                                return Err(err.clone());
+                            }
+                            if ctx.ctrl_c.load(Ordering::SeqCst) {
+                                // This early return doesn't return the result
+                                // we have so far, but breaking out of this loop
+                                // causes lifetime issues. A future contribution
+                                // could attempt to return the current output.
+                                // https://github.com/nushell/nushell/pull/2830#discussion_r550319687
+                                return Ok(InputStream::empty());
+                            }
+                        }
+                        Ok(None) => {
+                            if let Some(err) = ctx.get_errors().get(0) {
+                                ctx.clear_errors();
+                                return Err(err.clone());
+                            }
+                        }
+                        Err(e) => {
+                            return Err(e);
+                        }
+                    }
+                }
+                Err(e) => {
+                    return Err(e);
+                }
+            }
+            output = run_pipeline(pipeline, ctx, input).await;
+
+            input = InputStream::empty();
+        }
+    }
+
+    output
+}
+
+#[async_recursion]
+async fn run_pipeline(
+    commands: &Pipeline,
+    ctx: &EvaluationContext,
+    mut input: InputStream,
+) -> Result<InputStream, ShellError> {
+    for item in commands.list.clone() {
+        input = match item {
+            ClassifiedCommand::Dynamic(call) => {
+                let mut args = vec![];
+                if let Some(positional) = call.positional {
+                    for pos in &positional {
+                        let result = run_expression_block(pos, ctx).await?.into_vec().await;
+                        args.push(result);
+                    }
+                }
+
+                match &call.head.expr {
+                    Expression::Block(block) => {
+                        ctx.scope.enter_scope();
+                        for (param, value) in block.params.positional.iter().zip(args.iter()) {
+                            ctx.scope.add_var(param.0.name(), value[0].clone());
+                        }
+                        let result = run_block(&block, ctx, input).await;
+                        ctx.scope.exit_scope();
+
+                        let result = result?;
+                        return Ok(result);
+                    }
+                    Expression::Variable(v, span) => {
+                        if let Some(value) = ctx.scope.get_var(v) {
+                            match &value.value {
+                                UntaggedValue::Block(captured_block) => {
+                                    ctx.scope.enter_scope();
+                                    ctx.scope.add_vars(&captured_block.captured.entries);
+                                    for (param, value) in captured_block
+                                        .block
+                                        .params
+                                        .positional
+                                        .iter()
+                                        .zip(args.iter())
+                                    {
+                                        ctx.scope.add_var(param.0.name(), value[0].clone());
+                                    }
+                                    let result = run_block(&captured_block.block, ctx, input).await;
+                                    ctx.scope.exit_scope();
+
+                                    let result = result?;
+                                    return Ok(result);
+                                }
+                                _ => {
+                                    return Err(ShellError::labeled_error("Dynamic commands must start with a block (or variable pointing to a block)", "needs to be a block", call.head.span));
+                                }
+                            }
+                        } else {
+                            return Err(ShellError::labeled_error(
+                                "Variable not found",
+                                "variable not found",
+                                span,
+                            ));
+                        }
+                    }
+                    _ => {
+                        return Err(ShellError::labeled_error("Dynamic commands must start with a block (or variable pointing to a block)", "needs to be a block", call.head.span));
+                    }
+                }
+            }
+
+            ClassifiedCommand::Expr(expr) => run_expression_block(&*expr, ctx).await?,
+
+            ClassifiedCommand::Error(err) => return Err(err.into()),
+
+            ClassifiedCommand::Internal(left) => run_internal_command(left, ctx, input).await?,
+        };
+    }
+
+    Ok(input)
+}

crates/nu-engine/src/evaluate/evaluate_args.rs

@@ -0,0 +1,50 @@
+// TODO: Temporary redirect
+use crate::evaluate::evaluator::evaluate_baseline_expr;
+use crate::evaluation_context::EvaluationContext;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{hir, EvaluatedArgs, UntaggedValue, Value};
+
+pub(crate) async fn evaluate_args(
+    call: &hir::Call,
+    ctx: &EvaluationContext,
+) -> Result<EvaluatedArgs, ShellError> {
+    let mut positional_args: Vec<Value> = vec![];
+
+    if let Some(positional) = &call.positional {
+        for pos in positional {
+            let result = evaluate_baseline_expr(pos, ctx).await?;
+            positional_args.push(result);
+        }
+    }
+
+    let positional = if !positional_args.is_empty() {
+        Some(positional_args)
+    } else {
+        None
+    };
+
+    let mut named_args = IndexMap::new();
+
+    if let Some(named) = &call.named {
+        for (name, value) in named.iter() {
+            match value {
+                hir::NamedValue::PresentSwitch(tag) => {
+                    named_args.insert(name.clone(), UntaggedValue::boolean(true).into_value(tag));
+                }
+                hir::NamedValue::Value(_, expr) => {
+                    named_args.insert(name.clone(), evaluate_baseline_expr(expr, ctx).await?);
+                }
+                _ => {}
+            };
+        }
+    }
+
+    let named = if !named_args.is_empty() {
+        Some(named_args)
+    } else {
+        None
+    };
+
+    Ok(EvaluatedArgs::new(positional, named))
+}

crates/nu-engine/src/evaluate/evaluator.rs

@@ -0,0 +1,286 @@
+use crate::evaluate::block::run_block;
+use crate::evaluate::operator::apply_operator;
+use crate::evaluation_context::EvaluationContext;
+use async_recursion::async_recursion;
+use indexmap::IndexMap;
+use log::trace;
+use nu_errors::{ArgumentError, ShellError};
+use nu_protocol::did_you_mean;
+use nu_protocol::{
+    hir::{self, CapturedBlock, Expression, ExternalRedirection, RangeOperator, SpannedExpression},
+    Dictionary,
+};
+use nu_protocol::{
+    ColumnPath, Primitive, RangeInclusion, UnspannedPathMember, UntaggedValue, Value,
+};
+use nu_source::{Span, SpannedItem, Tag};
+use nu_stream::InputStream;
+use nu_value_ext::ValueExt;
+
+#[async_recursion]
+pub async fn evaluate_baseline_expr(
+    expr: &SpannedExpression,
+    ctx: &EvaluationContext,
+) -> Result<Value, ShellError> {
+    let tag = Tag {
+        span: expr.span,
+        anchor: None,
+    };
+    let span = expr.span;
+    match &expr.expr {
+        Expression::Literal(literal) => Ok(evaluate_literal(&literal, span)),
+        Expression::ExternalWord => Err(ShellError::argument_error(
+            "Invalid external word".spanned(tag.span),
+            ArgumentError::InvalidExternalWord,
+        )),
+        Expression::FilePath(path) => Ok(UntaggedValue::filepath(path.clone()).into_value(tag)),
+        Expression::Synthetic(hir::Synthetic::String(s)) => {
+            Ok(UntaggedValue::string(s).into_untagged_value())
+        }
+        Expression::Variable(var, _) => evaluate_reference(&var, ctx, tag),
+        Expression::Command => unimplemented!(),
+        Expression::Invocation(block) => evaluate_invocation(block, ctx).await,
+        Expression::ExternalCommand(_) => unimplemented!(),
+        Expression::Binary(binary) => {
+            // TODO: If we want to add short-circuiting, we'll need to move these down
+            let left = evaluate_baseline_expr(&binary.left, ctx).await?;
+            let right = evaluate_baseline_expr(&binary.right, ctx).await?;
+
+            trace!("left={:?} right={:?}", left.value, right.value);
+
+            match binary.op.expr {
+                Expression::Literal(hir::Literal::Operator(op)) => {
+                    match apply_operator(op, &left, &right) {
+                        Ok(result) => match result {
+                            UntaggedValue::Error(shell_err) => Err(shell_err),
+                            _ => Ok(result.into_value(tag)),
+                        },
+                        Err((left_type, right_type)) => Err(ShellError::coerce_error(
+                            left_type.spanned(binary.left.span),
+                            right_type.spanned(binary.right.span),
+                        )),
+                    }
+                }
+                _ => unreachable!(),
+            }
+        }
+        Expression::Range(range) => {
+            let left = if let Some(left) = &range.left {
+                evaluate_baseline_expr(&left, ctx).await?
+            } else {
+                Value::nothing()
+            };
+
+            let right = if let Some(right) = &range.right {
+                evaluate_baseline_expr(&right, ctx).await?
+            } else {
+                Value::nothing()
+            };
+
+            let left_span = left.tag.span;
+            let right_span = right.tag.span;
+
+            let left = (
+                left.as_primitive()?.spanned(left_span),
+                RangeInclusion::Inclusive,
+            );
+            let right = (
+                right.as_primitive()?.spanned(right_span),
+                match &range.operator.item {
+                    RangeOperator::Inclusive => RangeInclusion::Inclusive,
+                    RangeOperator::RightExclusive => RangeInclusion::Exclusive,
+                },
+            );
+
+            Ok(UntaggedValue::range(left, right).into_value(tag))
+        }
+        Expression::Table(headers, cells) => {
+            let mut output_headers = vec![];
+
+            for expr in headers {
+                let val = evaluate_baseline_expr(&expr, ctx).await?;
+
+                let header = val.as_string()?;
+                output_headers.push(header);
+            }
+
+            let mut output_table = vec![];
+
+            for row in cells {
+                if row.len() != headers.len() {
+                    match (row.first(), row.last()) {
+                        (Some(first), Some(last)) => {
+                            return Err(ShellError::labeled_error(
+                                "Cell count doesn't match header count",
+                                format!("expected {} columns", headers.len()),
+                                Span::new(first.span.start(), last.span.end()),
+                            ));
+                        }
+                        _ => {
+                            return Err(ShellError::untagged_runtime_error(
+                                "Cell count doesn't match header count",
+                            ));
+                        }
+                    }
+                }
+
+                let mut row_output = IndexMap::new();
+                for cell in output_headers.iter().zip(row.iter()) {
+                    let val = evaluate_baseline_expr(&cell.1, ctx).await?;
+                    row_output.insert(cell.0.clone(), val);
+                }
+                output_table.push(UntaggedValue::row(row_output).into_value(tag.clone()));
+            }
+
+            Ok(UntaggedValue::Table(output_table).into_value(tag))
+        }
+        Expression::List(list) => {
+            let mut exprs = vec![];
+
+            for expr in list {
+                let expr = evaluate_baseline_expr(&expr, ctx).await?;
+                exprs.push(expr);
+            }
+
+            Ok(UntaggedValue::Table(exprs).into_value(tag))
+        }
+        Expression::Block(block) => {
+            // Capture the current values of all free variables
+            let mut known_variables = vec![];
+            let free_variables = block.get_free_variables(&mut known_variables);
+
+            let mut captured = Dictionary::new(IndexMap::new());
+            for free_variable in &free_variables {
+                if let Some(v) = ctx.scope.get_var(free_variable) {
+                    captured.insert(free_variable.into(), v.clone());
+                }
+            }
+
+            let mut block = block.clone();
+            block.infer_params();
+
+            Ok(
+                UntaggedValue::Block(Box::new(CapturedBlock::new(block, captured)))
+                    .into_value(&tag),
+            )
+        }
+        Expression::Path(path) => {
+            let value = evaluate_baseline_expr(&path.head, ctx).await?;
+            let mut item = value;
+
+            for member in &path.tail {
+                let next = item.get_data_by_member(member);
+
+                match next {
+                    Err(err) => {
+                        if let UnspannedPathMember::String(_name) = &member.unspanned {
+                            let possible_matches = did_you_mean(&item, member.as_string());
+
+                            match possible_matches {
+                                Some(p) => {
+                                    return Err(ShellError::labeled_error(
+                                        "Unknown column",
+                                        format!("did you mean '{}'?", p[0]),
+                                        &member.span,
+                                    ));
+                                }
+                                None => return Err(err),
+                            }
+                        }
+                    }
+                    Ok(next) => {
+                        item = next.clone().value.into_value(&tag);
+                    }
+                };
+            }
+
+            Ok(item.value.into_value(tag))
+        }
+        Expression::Boolean(_boolean) => Ok(UntaggedValue::boolean(*_boolean).into_value(tag)),
+        Expression::Garbage => unimplemented!(),
+    }
+}
+
+fn evaluate_literal(literal: &hir::Literal, span: Span) -> Value {
+    match &literal {
+        hir::Literal::ColumnPath(path) => {
+            let members = path.iter().map(|member| member.to_path_member()).collect();
+
+            UntaggedValue::Primitive(Primitive::ColumnPath(ColumnPath::new(members)))
+                .into_value(span)
+        }
+        hir::Literal::Number(int) => match int {
+            nu_protocol::hir::Number::Int(i) => UntaggedValue::int(i.clone()).into_value(span),
+            nu_protocol::hir::Number::Decimal(d) => {
+                UntaggedValue::decimal(d.clone()).into_value(span)
+            }
+        },
+        hir::Literal::Size(int, unit) => unit.compute(&int).into_value(span),
+        hir::Literal::String(string) => UntaggedValue::string(string).into_value(span),
+        hir::Literal::GlobPattern(pattern) => UntaggedValue::glob_pattern(pattern).into_value(span),
+        hir::Literal::Bare(bare) => UntaggedValue::string(bare.clone()).into_value(span),
+        hir::Literal::Operator(_) => unimplemented!("Not sure what to do with operator yet"),
+    }
+}
+
+fn evaluate_reference(name: &str, ctx: &EvaluationContext, tag: Tag) -> Result<Value, ShellError> {
+    match name {
+        "$nu" => crate::evaluate::variables::nu(&ctx.scope.get_env_vars(), tag),
+
+        "$true" => Ok(Value {
+            value: UntaggedValue::boolean(true),
+            tag,
+        }),
+
+        "$false" => Ok(Value {
+            value: UntaggedValue::boolean(false),
+            tag,
+        }),
+
+        "$it" => match ctx.scope.get_var("$it") {
+            Some(v) => Ok(v),
+            None => Err(ShellError::labeled_error(
+                "Variable not in scope",
+                "missing '$it' (note: $it is only available inside of a block)",
+                tag.span,
+            )),
+        },
+
+        x => match ctx.scope.get_var(x) {
+            Some(v) => Ok(v),
+            None => Err(ShellError::labeled_error(
+                "Variable not in scope",
+                "unknown variable",
+                tag.span,
+            )),
+        },
+    }
+}
+
+async fn evaluate_invocation(
+    block: &hir::Block,
+    ctx: &EvaluationContext,
+) -> Result<Value, ShellError> {
+    // FIXME: we should use a real context here
+    let input = match ctx.scope.get_var("$it") {
+        Some(it) => InputStream::one(it),
+        None => InputStream::empty(),
+    };
+
+    let mut block = block.clone();
+    block.set_redirect(ExternalRedirection::Stdout);
+
+    let result = run_block(&block, ctx, input).await?;
+
+    let output = result.into_vec().await;
+
+    if let Some(e) = ctx.get_errors().get(0) {
+        return Err(e.clone());
+    }
+
+    match output.len() {
+        x if x > 1 => Ok(UntaggedValue::Table(output).into_value(Tag::unknown())),
+        1 => Ok(output[0].clone()),
+        _ => Ok(UntaggedValue::nothing().into_value(Tag::unknown())),
+    }
+}

crates/nu-engine/src/evaluate/expr.rs

@@ -0,0 +1,23 @@
+use crate::evaluate_baseline_expr;
+
+use log::{log_enabled, trace};
+
+use crate::evaluation_context::EvaluationContext;
+use futures::stream::once;
+use nu_errors::ShellError;
+use nu_protocol::hir::SpannedExpression;
+use nu_stream::{InputStream, ToInputStream};
+
+pub(crate) async fn run_expression_block(
+    expr: &SpannedExpression,
+    ctx: &EvaluationContext,
+) -> Result<InputStream, ShellError> {
+    if log_enabled!(log::Level::Trace) {
+        trace!(target: "nu::run::expr", "->");
+        trace!(target: "nu::run::expr", "{:?}", expr);
+    }
+
+    let output = evaluate_baseline_expr(expr, ctx).await?;
+
+    Ok(once(async { Ok(output) }).to_input_stream())
+}

crates/nu-engine/src/evaluate/internal.rs

@@ -0,0 +1,259 @@
+use crate::call_info::UnevaluatedCallInfo;
+use crate::command_args::RawCommandArgs;
+use crate::evaluation_context::EvaluationContext;
+use crate::filesystem::filesystem_shell::FilesystemShell;
+use crate::shell::help_shell::HelpShell;
+use crate::shell::value_shell::ValueShell;
+use futures::StreamExt;
+use log::{log_enabled, trace};
+use nu_errors::ShellError;
+use nu_protocol::hir::{ExternalRedirection, InternalCommand};
+use nu_protocol::{CommandAction, Primitive, ReturnSuccess, UntaggedValue, Value};
+use nu_source::{PrettyDebug, Span, Tag};
+use nu_stream::{trace_stream, InputStream, ToInputStream};
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+
+pub(crate) async fn run_internal_command(
+    command: InternalCommand,
+    context: &EvaluationContext,
+    input: InputStream,
+) -> Result<InputStream, ShellError> {
+    if log_enabled!(log::Level::Trace) {
+        trace!(target: "nu::run::internal", "->");
+        trace!(target: "nu::run::internal", "{}", command.name);
+    }
+
+    let objects: InputStream = trace_stream!(target: "nu::trace_stream::internal", "input" = input);
+
+    let internal_command = context.scope.expect_command(&command.name);
+
+    if command.name == "autoenv untrust" {
+        context
+            .user_recently_used_autoenv_untrust
+            .store(true, Ordering::SeqCst);
+    }
+
+    let result = {
+        context
+            .run_command(
+                internal_command?,
+                Tag::unknown_anchor(command.name_span),
+                command.args.clone(),
+                objects,
+            )
+            .await?
+    };
+
+    let head = Arc::new(command.args.head.clone());
+    let context = context.clone();
+    let command = Arc::new(command);
+
+    Ok(InputStream::from_stream(
+        result
+            .then(move |item| {
+                let head = head.clone();
+                let command = command.clone();
+                let context = context.clone();
+                async move {
+                    match item {
+                        Ok(ReturnSuccess::Action(action)) => match action {
+                            CommandAction::ChangePath(path) => {
+                                context.shell_manager.set_path(path);
+                                InputStream::empty()
+                            }
+                            CommandAction::Exit => std::process::exit(0), // TODO: save history.txt
+                            CommandAction::Error(err) => {
+                                context.error(err);
+                                InputStream::empty()
+                            }
+                            CommandAction::AutoConvert(tagged_contents, extension) => {
+                                let contents_tag = tagged_contents.tag.clone();
+                                let command_name = format!("from {}", extension);
+                                let command = command.clone();
+                                if let Some(converter) = context.scope.get_command(&command_name) {
+                                    let new_args = RawCommandArgs {
+                                        host: context.host.clone(),
+                                        ctrl_c: context.ctrl_c.clone(),
+                                        current_errors: context.current_errors.clone(),
+                                        shell_manager: context.shell_manager.clone(),
+                                        call_info: UnevaluatedCallInfo {
+                                            args: nu_protocol::hir::Call {
+                                                head: (&*head).clone(),
+                                                positional: None,
+                                                named: None,
+                                                span: Span::unknown(),
+                                                external_redirection: ExternalRedirection::Stdout,
+                                            },
+                                            name_tag: Tag::unknown_anchor(command.name_span),
+                                        },
+                                        scope: context.scope.clone(),
+                                    };
+                                    let result = converter
+                                        .run(new_args.with_input(vec![tagged_contents]))
+                                        .await;
+
+                                    match result {
+                                        Ok(mut result) => {
+                                            let result_vec: Vec<Result<ReturnSuccess, ShellError>> =
+                                                result.drain_vec().await;
+
+                                            let mut output = vec![];
+                                            for res in result_vec {
+                                                match res {
+                                                    Ok(ReturnSuccess::Value(Value {
+                                                        value: UntaggedValue::Table(list),
+                                                        ..
+                                                    })) => {
+                                                        for l in list {
+                                                            output.push(Ok(l));
+                                                        }
+                                                    }
+                                                    Ok(ReturnSuccess::Value(Value {
+                                                        value,
+                                                        ..
+                                                    })) => {
+                                                        output
+                                                            .push(Ok(value
+                                                                .into_value(contents_tag.clone())));
+                                                    }
+                                                    Err(e) => output.push(Err(e)),
+                                                    _ => {}
+                                                }
+                                            }
+
+                                            futures::stream::iter(output).to_input_stream()
+                                        }
+                                        Err(err) => {
+                                            context.error(err);
+                                            InputStream::empty()
+                                        }
+                                    }
+                                } else {
+                                    InputStream::one(tagged_contents)
+                                }
+                            }
+                            CommandAction::EnterHelpShell(value) => match value {
+                                Value {
+                                    value: UntaggedValue::Primitive(Primitive::String(cmd)),
+                                    tag,
+                                } => {
+                                    context.shell_manager.insert_at_current(Box::new(
+                                        match HelpShell::for_command(
+                                            UntaggedValue::string(cmd).into_value(tag),
+                                            &context.scope,
+                                        ) {
+                                            Ok(v) => v,
+                                            Err(err) => {
+                                                context.error(err);
+                                                return InputStream::empty();
+                                            }
+                                        },
+                                    ));
+                                    InputStream::from_stream(futures::stream::iter(vec![]))
+                                }
+                                _ => {
+                                    context.shell_manager.insert_at_current(Box::new(
+                                        match HelpShell::index(&context.scope) {
+                                            Ok(v) => v,
+                                            Err(err) => {
+                                                context.error(err);
+                                                return InputStream::empty();
+                                            }
+                                        },
+                                    ));
+                                    InputStream::from_stream(futures::stream::iter(vec![]))
+                                }
+                            },
+                            CommandAction::EnterValueShell(value) => {
+                                context
+                                    .shell_manager
+                                    .insert_at_current(Box::new(ValueShell::new(value)));
+                                InputStream::from_stream(futures::stream::iter(vec![]))
+                            }
+                            CommandAction::EnterShell(location) => {
+                                context.shell_manager.insert_at_current(Box::new(
+                                    match FilesystemShell::with_location(location) {
+                                        Ok(v) => v,
+                                        Err(err) => {
+                                            context.error(err.into());
+                                            return InputStream::empty();
+                                        }
+                                    },
+                                ));
+                                InputStream::from_stream(futures::stream::iter(vec![]))
+                            }
+                            CommandAction::AddPlugins(path) => {
+                                match crate::plugin::build_plugin::scan(vec![
+                                    std::path::PathBuf::from(path),
+                                ]) {
+                                    Ok(plugins) => {
+                                        context.add_commands(
+                                            plugins
+                                                .into_iter()
+                                                .filter(|p| {
+                                                    !context.is_command_registered(p.name())
+                                                })
+                                                .collect(),
+                                        );
+
+                                        InputStream::empty()
+                                    }
+                                    Err(reason) => {
+                                        context.error(reason);
+                                        InputStream::empty()
+                                    }
+                                }
+                            }
+                            CommandAction::PreviousShell => {
+                                context.shell_manager.prev();
+                                InputStream::empty()
+                            }
+                            CommandAction::NextShell => {
+                                context.shell_manager.next();
+                                InputStream::empty()
+                            }
+                            CommandAction::LeaveShell => {
+                                context.shell_manager.remove_at_current();
+                                if context.shell_manager.is_empty() {
+                                    std::process::exit(0); // TODO: save history.txt
+                                }
+                                InputStream::empty()
+                            }
+                        },
+
+                        Ok(ReturnSuccess::Value(Value {
+                            value: UntaggedValue::Error(err),
+                            ..
+                        })) => {
+                            context.error(err);
+                            InputStream::empty()
+                        }
+
+                        Ok(ReturnSuccess::Value(v)) => InputStream::one(v),
+
+                        Ok(ReturnSuccess::DebugValue(v)) => {
+                            let doc = PrettyDebug::pretty_doc(&v);
+                            let mut buffer = termcolor::Buffer::ansi();
+
+                            let _ = doc.render_raw(
+                                context.with_host(|host| host.width() - 5),
+                                &mut nu_source::TermColored::new(&mut buffer),
+                            );
+
+                            let value = String::from_utf8_lossy(buffer.as_slice());
+
+                            InputStream::one(UntaggedValue::string(value).into_untagged_value())
+                        }
+
+                        Err(err) => {
+                            context.error(err);
+                            InputStream::empty()
+                        }
+                    }
+                }
+            })
+            .flatten()
+            .take_while(|x| futures::future::ready(!x.is_error())),
+    ))
+}

crates/nu-engine/src/evaluate/mod.rs

@@ -0,0 +1,8 @@
+pub(crate) mod block;
+pub(crate) mod evaluate_args;
+pub(crate) mod evaluator;
+pub(crate) mod expr;
+pub(crate) mod internal;
+pub(crate) mod operator;
+pub(crate) mod scope;
+pub(crate) mod variables;

crates/nu-engine/src/evaluate/operator.rs

@@ -0,0 +1,87 @@
+use nu_data::value;
+use nu_errors::ShellError;
+use nu_protocol::hir::Operator;
+use nu_protocol::{Primitive, ShellTypeName, UntaggedValue, Value};
+use std::ops::Not;
+
+pub fn apply_operator(
+    op: Operator,
+    left: &Value,
+    right: &Value,
+) -> Result<UntaggedValue, (&'static str, &'static str)> {
+    match op {
+        Operator::Equal
+        | Operator::NotEqual
+        | Operator::LessThan
+        | Operator::GreaterThan
+        | Operator::LessThanOrEqual
+        | Operator::GreaterThanOrEqual => {
+            value::compare_values(op, left, right).map(UntaggedValue::boolean)
+        }
+        Operator::Contains => string_contains(left, right).map(UntaggedValue::boolean),
+        Operator::NotContains => string_contains(left, right)
+            .map(Not::not)
+            .map(UntaggedValue::boolean),
+        Operator::Plus => value::compute_values(op, left, right),
+        Operator::Minus => value::compute_values(op, left, right),
+        Operator::Multiply => value::compute_values(op, left, right),
+        Operator::Divide => value::compute_values(op, left, right).map(|res| match res {
+            UntaggedValue::Error(_) => UntaggedValue::Error(ShellError::labeled_error(
+                "Evaluation error",
+                "division by zero",
+                &right.tag.span,
+            )),
+            _ => res,
+        }),
+        Operator::Modulo => value::compute_values(op, left, right).map(|res| match res {
+            UntaggedValue::Error(_) => UntaggedValue::Error(ShellError::labeled_error(
+                "Evaluation error",
+                "division by zero",
+                &right.tag.span,
+            )),
+            _ => res,
+        }),
+        Operator::In => table_contains(left, right).map(UntaggedValue::boolean),
+        Operator::NotIn => table_contains(left, right).map(|x| UntaggedValue::boolean(!x)),
+        Operator::And => match (left.as_bool(), right.as_bool()) {
+            (Ok(left), Ok(right)) => Ok(UntaggedValue::boolean(left && right)),
+            _ => Err((left.type_name(), right.type_name())),
+        },
+        Operator::Or => match (left.as_bool(), right.as_bool()) {
+            (Ok(left), Ok(right)) => Ok(UntaggedValue::boolean(left || right)),
+            _ => Err((left.type_name(), right.type_name())),
+        },
+    }
+}
+
+fn string_contains(
+    left: &UntaggedValue,
+    right: &UntaggedValue,
+) -> Result<bool, (&'static str, &'static str)> {
+    match (left, right) {
+        (
+            UntaggedValue::Primitive(Primitive::String(l)),
+            UntaggedValue::Primitive(Primitive::String(r)),
+        ) => Ok(l.contains(r)),
+        (
+            UntaggedValue::Primitive(Primitive::FilePath(l)),
+            UntaggedValue::Primitive(Primitive::String(r)),
+        ) => Ok(l.as_path().display().to_string().contains(r)),
+        (
+            UntaggedValue::Primitive(Primitive::String(l)),
+            UntaggedValue::Primitive(Primitive::FilePath(r)),
+        ) => Ok(l.contains(&r.as_path().display().to_string())),
+        _ => Err((left.type_name(), right.type_name())),
+    }
+}
+
+fn table_contains(
+    left: &UntaggedValue,
+    right: &UntaggedValue,
+) -> Result<bool, (&'static str, &'static str)> {
+    let left = left.clone();
+    match right {
+        UntaggedValue::Table(values) => Ok(values.iter().any(|x| x.value == left)),
+        _ => Err((left.type_name(), right.type_name())),
+    }
+}

crates/nu-engine/src/evaluate/scope.rs

@@ -0,0 +1,369 @@
+use crate::whole_stream_command::{whole_stream_command, Command};
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_parser::ParserScope;
+use nu_protocol::{hir::Block, Value};
+use nu_source::Spanned;
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+pub struct Scope {
+    frames: Arc<parking_lot::Mutex<Vec<ScopeFrame>>>,
+}
+
+impl Default for Scope {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Scope {
+    pub fn new() -> Scope {
+        Scope {
+            frames: Arc::new(parking_lot::Mutex::new(vec![ScopeFrame::new()])),
+        }
+    }
+    pub fn get_command(&self, name: &str) -> Option<Command> {
+        for frame in self.frames.lock().iter().rev() {
+            if let Some(command) = frame.get_command(name) {
+                return Some(command);
+            }
+        }
+
+        None
+    }
+
+    pub fn get_aliases_with_name(&self, name: &str) -> Option<Vec<Vec<Spanned<String>>>> {
+        let aliases: Vec<_> = self
+            .frames
+            .lock()
+            .iter()
+            .rev()
+            .filter_map(|frame| frame.aliases.get(name).cloned())
+            .collect();
+        if aliases.is_empty() {
+            None
+        } else {
+            Some(aliases)
+        }
+    }
+
+    pub fn get_custom_commands_with_name(&self, name: &str) -> Option<Vec<Block>> {
+        let custom_commands: Vec<_> = self
+            .frames
+            .lock()
+            .iter()
+            .rev()
+            .filter_map(|frame| frame.custom_commands.get(name).cloned())
+            .collect();
+
+        if custom_commands.is_empty() {
+            None
+        } else {
+            Some(custom_commands)
+        }
+    }
+
+    pub fn add_command(&self, name: String, command: Command) {
+        // Note: this is assumed to always be true, as there is always a global top frame
+        if let Some(frame) = self.frames.lock().last_mut() {
+            frame.add_command(name, command)
+        }
+    }
+
+    pub fn get_command_names(&self) -> Vec<String> {
+        let mut names = vec![];
+
+        for frame in self.frames.lock().iter() {
+            let mut frame_command_names = frame.get_command_names();
+            names.append(&mut frame_command_names);
+        }
+
+        names.dedup();
+        names.sort();
+
+        names
+    }
+
+    pub fn len(&self) -> usize {
+        self.frames.lock().len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.frames.lock().is_empty()
+    }
+
+    fn has_cmd_helper(&self, name: &str, f: fn(&ScopeFrame, &str) -> bool) -> bool {
+        self.frames.lock().iter().any(|frame| f(frame, name))
+    }
+
+    pub fn has_command(&self, name: &str) -> bool {
+        self.has_cmd_helper(name, ScopeFrame::has_command)
+    }
+
+    pub fn has_custom_command(&self, name: &str) -> bool {
+        self.has_cmd_helper(name, ScopeFrame::has_custom_command)
+    }
+
+    pub fn has_alias(&self, name: &str) -> bool {
+        self.has_cmd_helper(name, ScopeFrame::has_alias)
+    }
+
+    pub fn expect_command(&self, name: &str) -> Result<Command, ShellError> {
+        if let Some(c) = self.get_command(name) {
+            Ok(c)
+        } else {
+            Err(ShellError::untagged_runtime_error(format!(
+                "Missing command '{}'",
+                name
+            )))
+        }
+    }
+
+    pub fn get_vars(&self) -> IndexMap<String, Value> {
+        //FIXME: should this be an iterator?
+        let mut output = IndexMap::new();
+
+        for frame in self.frames.lock().iter().rev() {
+            for v in frame.vars.iter() {
+                if !output.contains_key(v.0) {
+                    output.insert(v.0.clone(), v.1.clone());
+                }
+            }
+        }
+
+        output
+    }
+
+    pub fn get_env_vars(&self) -> IndexMap<String, String> {
+        //FIXME: should this be an iterator?
+        let mut output = IndexMap::new();
+
+        for frame in self.frames.lock().iter().rev() {
+            for v in frame.env.iter() {
+                if !output.contains_key(v.0) {
+                    output.insert(v.0.clone(), v.1.clone());
+                }
+            }
+        }
+
+        output
+    }
+
+    pub fn get_var(&self, name: &str) -> Option<Value> {
+        for frame in self.frames.lock().iter().rev() {
+            if let Some(v) = frame.vars.get(name) {
+                return Some(v.clone());
+            }
+        }
+
+        None
+    }
+
+    pub fn add_var(&self, name: impl Into<String>, value: Value) {
+        if let Some(frame) = self.frames.lock().last_mut() {
+            frame.vars.insert(name.into(), value);
+        }
+    }
+
+    pub fn add_vars(&self, vars: &IndexMap<String, Value>) {
+        if let Some(frame) = self.frames.lock().last_mut() {
+            frame
+                .vars
+                .extend(vars.iter().map(|(s, v)| (s.clone(), v.clone())))
+        }
+    }
+
+    pub fn add_env_var(&self, name: impl Into<String>, value: String) {
+        if let Some(frame) = self.frames.lock().last_mut() {
+            frame.env.insert(name.into(), value);
+        }
+    }
+
+    pub fn add_env(&self, env_vars: IndexMap<String, String>) {
+        if let Some(frame) = self.frames.lock().last_mut() {
+            frame.env.extend(env_vars)
+        }
+    }
+}
+
+impl ParserScope for Scope {
+    fn get_signature(&self, name: &str) -> Option<nu_protocol::Signature> {
+        self.get_command(name).map(|x| x.signature())
+    }
+
+    fn has_signature(&self, name: &str) -> bool {
+        self.get_command(name).is_some()
+    }
+
+    fn add_definition(&self, block: Block) {
+        if let Some(frame) = self.frames.lock().last_mut() {
+            let name = block.params.name.clone();
+            frame.custom_commands.insert(name.clone(), block.clone());
+            frame.commands.insert(name, whole_stream_command(block));
+        }
+    }
+
+    fn get_definitions(&self) -> Vec<Block> {
+        let mut blocks = vec![];
+        if let Some(frame) = self.frames.lock().last() {
+            for (_, custom_command) in &frame.custom_commands {
+                blocks.push(custom_command.clone());
+            }
+        }
+        blocks
+    }
+
+    fn get_alias(&self, name: &str) -> Option<Vec<Spanned<String>>> {
+        for frame in self.frames.lock().iter().rev() {
+            if let Some(x) = frame.aliases.get(name) {
+                return Some(x.clone());
+            }
+        }
+        None
+    }
+
+    fn add_alias(&self, name: &str, replacement: Vec<Spanned<String>>) {
+        // Note: this is assumed to always be true, as there is always a global top frame
+        if let Some(frame) = self.frames.lock().last_mut() {
+            frame.aliases.insert(name.to_string(), replacement);
+        }
+    }
+
+    fn enter_scope(&self) {
+        self.frames.lock().push(ScopeFrame::new());
+    }
+
+    fn exit_scope(&self) {
+        self.frames.lock().pop();
+    }
+}
+
+/// An evaluation scope. Scopes map variable names to Values and aid in evaluating blocks and expressions.
+#[derive(Debug, Clone)]
+pub struct ScopeFrame {
+    pub vars: IndexMap<String, Value>,
+    pub env: IndexMap<String, String>,
+    pub commands: IndexMap<String, Command>,
+    pub custom_commands: IndexMap<String, Block>,
+    pub aliases: IndexMap<String, Vec<Spanned<String>>>,
+}
+
+impl ScopeFrame {
+    pub fn has_command(&self, name: &str) -> bool {
+        self.commands.contains_key(name)
+    }
+
+    pub fn has_custom_command(&self, name: &str) -> bool {
+        self.custom_commands.contains_key(name)
+    }
+
+    pub fn has_alias(&self, name: &str) -> bool {
+        self.aliases.contains_key(name)
+    }
+
+    pub fn get_command_names(&self) -> Vec<String> {
+        self.commands.keys().map(|x| x.to_string()).collect()
+    }
+
+    pub fn add_command(&mut self, name: String, command: Command) {
+        self.commands.insert(name, command);
+    }
+
+    pub fn get_command(&self, name: &str) -> Option<Command> {
+        self.commands.get(name).cloned()
+    }
+
+    pub fn new() -> ScopeFrame {
+        ScopeFrame {
+            vars: IndexMap::new(),
+            env: IndexMap::new(),
+            commands: IndexMap::new(),
+            custom_commands: IndexMap::new(),
+            aliases: IndexMap::new(),
+        }
+    }
+}
+
+// impl Scope {
+//     pub fn vars(&self) -> IndexMap<String, Value> {
+//         //FIXME: should this be an iterator?
+
+//         let mut output = IndexMap::new();
+
+//         for v in &self.vars {
+//             output.insert(v.0.clone(), v.1.clone());
+//         }
+
+//         if let Some(parent) = &self.parent {
+//             for v in parent.vars() {
+//                 if !output.contains_key(&v.0) {
+//                     output.insert(v.0.clone(), v.1.clone());
+//                 }
+//             }
+//         }
+
+//         output
+//     }
+
+//     pub fn env(&self) -> IndexMap<String, String> {
+//         //FIXME: should this be an iterator?
+
+//         let mut output = IndexMap::new();
+
+//         for v in &self.env {
+//             output.insert(v.0.clone(), v.1.clone());
+//         }
+
+//         if let Some(parent) = &self.parent {
+//             for v in parent.env() {
+//                 if !output.contains_key(&v.0) {
+//                     output.insert(v.0.clone(), v.1.clone());
+//                 }
+//             }
+//         }
+
+//         output
+//     }
+
+//     pub fn var(&self, name: &str) -> Option<Value> {
+//         if let Some(value) = self.vars().get(name) {
+//             Some(value.clone())
+//         } else {
+//             None
+//         }
+//     }
+
+//     pub fn append_var(this: Arc<Self>, name: impl Into<String>, value: Value) -> Arc<Scope> {
+//         let mut vars = IndexMap::new();
+//         vars.insert(name.into(), value);
+//         Arc::new(Scope {
+//             vars,
+//             env: IndexMap::new(),
+//             commands: IndexMap::new(),
+//             aliases: IndexMap::new(),
+//             parent: Some(this),
+//         })
+//     }
+
+//     pub fn append_vars(this: Arc<Self>, vars: IndexMap<String, Value>) -> Arc<Scope> {
+//         Arc::new(Scope {
+//             vars,
+//             env: IndexMap::new(),
+//             commands: IndexMap::new(),
+//             aliases: IndexMap::new(),
+//             parent: Some(this),
+//         })
+//     }
+
+//     pub fn append_env(this: Arc<Self>, env: IndexMap<String, String>) -> Arc<Scope> {
+//         Arc::new(Scope {
+//             vars: IndexMap::new(),
+//             env,
+//             commands: IndexMap::new(),
+//             aliases: IndexMap::new(),
+//             parent: Some(this),
+//         })
+//     }
+
+// }

crates/nu-engine/src/evaluate/variables.rs

@@ -0,0 +1,65 @@
+use crate::history_path::history_path;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{TaggedDictBuilder, UntaggedValue, Value};
+use nu_source::Tag;
+
+pub fn nu(env: &IndexMap<String, String>, tag: impl Into<Tag>) -> Result<Value, ShellError> {
+    let tag = tag.into();
+
+    let mut nu_dict = TaggedDictBuilder::new(&tag);
+
+    let mut dict = TaggedDictBuilder::new(&tag);
+    for v in env.iter() {
+        if v.0 != "PATH" && v.0 != "Path" {
+            dict.insert_untagged(v.0, UntaggedValue::string(v.1));
+        }
+    }
+    nu_dict.insert_value("env", dict.into_value());
+
+    let config = nu_data::config::read(&tag, &None)?;
+    nu_dict.insert_value("config", UntaggedValue::row(config).into_value(&tag));
+
+    let mut table = vec![];
+    let path = std::env::var_os("PATH");
+    if let Some(paths) = path {
+        for path in std::env::split_paths(&paths) {
+            table.push(UntaggedValue::filepath(path).into_value(&tag));
+        }
+    }
+    nu_dict.insert_value("path", UntaggedValue::table(&table).into_value(&tag));
+
+    let path = std::env::current_dir()?;
+    nu_dict.insert_value("cwd", UntaggedValue::filepath(path).into_value(&tag));
+
+    if let Some(home) = crate::filesystem::filesystem_shell::homedir_if_possible() {
+        nu_dict.insert_value("home-dir", UntaggedValue::filepath(home).into_value(&tag));
+    }
+
+    let temp = std::env::temp_dir();
+    nu_dict.insert_value("temp-dir", UntaggedValue::filepath(temp).into_value(&tag));
+
+    let config = nu_data::config::default_path()?;
+    nu_dict.insert_value(
+        "config-path",
+        UntaggedValue::filepath(config).into_value(&tag),
+    );
+
+    #[cfg(feature = "rustyline-support")]
+    {
+        let keybinding_path = crate::keybinding::keybinding_path()?;
+        nu_dict.insert_value(
+            "keybinding-path",
+            UntaggedValue::filepath(keybinding_path).into_value(&tag),
+        );
+    }
+
+    let config: Box<dyn nu_data::config::Conf> = Box::new(nu_data::config::NuConfig::new());
+    let history = history_path(&config);
+    nu_dict.insert_value(
+        "history-path",
+        UntaggedValue::filepath(history).into_value(&tag),
+    );
+
+    Ok(nu_dict.into_value())
+}

crates/nu-engine/src/evaluation_context.rs

@@ -0,0 +1,135 @@
+use crate::call_info::UnevaluatedCallInfo;
+use crate::command_args::CommandArgs;
+use crate::env::host::Host;
+use crate::evaluate::scope::Scope;
+use crate::shell::shell_manager::ShellManager;
+use crate::whole_stream_command::Command;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::hir;
+use nu_source::Tag;
+use nu_stream::{InputStream, OutputStream};
+use parking_lot::Mutex;
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub struct EvaluationContext {
+    pub scope: Scope,
+    pub host: Arc<parking_lot::Mutex<Box<dyn Host>>>,
+    pub current_errors: Arc<Mutex<Vec<ShellError>>>,
+    pub ctrl_c: Arc<AtomicBool>,
+    pub user_recently_used_autoenv_untrust: Arc<AtomicBool>,
+    pub shell_manager: ShellManager,
+
+    /// Windows-specific: keep track of previous cwd on each drive
+    pub windows_drives_previous_cwd: Arc<Mutex<std::collections::HashMap<String, String>>>,
+}
+
+impl EvaluationContext {
+    pub fn from_raw(raw_args: &CommandArgs) -> EvaluationContext {
+        EvaluationContext {
+            scope: raw_args.scope.clone(),
+            host: raw_args.host.clone(),
+            current_errors: raw_args.current_errors.clone(),
+            ctrl_c: raw_args.ctrl_c.clone(),
+            shell_manager: raw_args.shell_manager.clone(),
+            user_recently_used_autoenv_untrust: Arc::new(AtomicBool::new(false)),
+            windows_drives_previous_cwd: Arc::new(Mutex::new(std::collections::HashMap::new())),
+        }
+    }
+
+    pub fn from_args(args: &CommandArgs) -> EvaluationContext {
+        EvaluationContext {
+            scope: args.scope.clone(),
+            host: args.host.clone(),
+            current_errors: args.current_errors.clone(),
+            ctrl_c: args.ctrl_c.clone(),
+            shell_manager: args.shell_manager.clone(),
+            user_recently_used_autoenv_untrust: Arc::new(AtomicBool::new(false)),
+            windows_drives_previous_cwd: Arc::new(Mutex::new(std::collections::HashMap::new())),
+        }
+    }
+
+    pub fn error(&self, error: ShellError) {
+        self.with_errors(|errors| errors.push(error))
+    }
+
+    pub fn clear_errors(&self) {
+        self.current_errors.lock().clear()
+    }
+
+    pub fn get_errors(&self) -> Vec<ShellError> {
+        self.current_errors.lock().clone()
+    }
+
+    pub fn configure<T>(
+        &mut self,
+        config: &dyn nu_data::config::Conf,
+        block: impl FnOnce(&dyn nu_data::config::Conf, &mut Self) -> T,
+    ) {
+        block(config, &mut *self);
+    }
+
+    pub fn with_host<T>(&self, block: impl FnOnce(&mut dyn Host) -> T) -> T {
+        let mut host = self.host.lock();
+
+        block(&mut *host)
+    }
+
+    pub fn with_errors<T>(&self, block: impl FnOnce(&mut Vec<ShellError>) -> T) -> T {
+        let mut errors = self.current_errors.lock();
+
+        block(&mut *errors)
+    }
+
+    pub fn add_commands(&self, commands: Vec<Command>) {
+        for command in commands {
+            self.scope.add_command(command.name().to_string(), command);
+        }
+    }
+
+    #[allow(unused)]
+    pub(crate) fn get_command(&self, name: &str) -> Option<Command> {
+        self.scope.get_command(name)
+    }
+
+    pub fn is_command_registered(&self, name: &str) -> bool {
+        self.scope.has_command(name)
+    }
+
+    pub(crate) async fn run_command(
+        &self,
+        command: Command,
+        name_tag: Tag,
+        args: hir::Call,
+        input: InputStream,
+    ) -> Result<OutputStream, ShellError> {
+        let command_args = self.command_args(args, input, name_tag);
+        command.run(command_args).await
+    }
+
+    fn call_info(&self, args: hir::Call, name_tag: Tag) -> UnevaluatedCallInfo {
+        UnevaluatedCallInfo { args, name_tag }
+    }
+
+    fn command_args(&self, args: hir::Call, input: InputStream, name_tag: Tag) -> CommandArgs {
+        CommandArgs {
+            host: self.host.clone(),
+            ctrl_c: self.ctrl_c.clone(),
+            current_errors: self.current_errors.clone(),
+            shell_manager: self.shell_manager.clone(),
+            call_info: self.call_info(args, name_tag),
+            scope: self.scope.clone(),
+            input,
+        }
+    }
+
+    pub fn get_env(&self) -> IndexMap<String, String> {
+        let mut output = IndexMap::new();
+        for (var, value) in self.host.lock().vars() {
+            output.insert(var, value);
+        }
+        output
+    }
+}

crates/nu-engine/src/example.rs

@@ -0,0 +1,7 @@
+use nu_protocol::Value;
+
+pub struct Example {
+    pub example: &'static str,
+    pub description: &'static str,
+    pub result: Option<Vec<Value>>,
+}

crates/nu-engine/src/filesystem/dir_info.rs

@@ -0,0 +1,267 @@
+use filesize::file_real_size_fast;
+use glob::Pattern;
+use indexmap::IndexMap;
+use nu_errors::ShellError;
+use nu_protocol::{UntaggedValue, Value};
+use nu_source::Tag;
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+pub struct DirBuilder {
+    pub tag: Tag,
+    pub min: Option<u64>,
+    pub deref: bool,
+    pub exclude: Option<Pattern>,
+    pub all: bool,
+}
+
+impl DirBuilder {
+    pub fn new(
+        tag: Tag,
+        min: Option<u64>,
+        deref: bool,
+        exclude: Option<Pattern>,
+        all: bool,
+    ) -> DirBuilder {
+        DirBuilder {
+            tag,
+            min,
+            deref,
+            exclude,
+            all,
+        }
+    }
+}
+
+pub struct DirInfo {
+    dirs: Vec<DirInfo>,
+    files: Vec<FileInfo>,
+    errors: Vec<ShellError>,
+    size: u64,
+    blocks: u64,
+    path: PathBuf,
+    tag: Tag,
+}
+
+pub struct FileInfo {
+    path: PathBuf,
+    size: u64,
+    blocks: Option<u64>,
+    tag: Tag,
+}
+
+impl FileInfo {
+    pub fn new(path: impl Into<PathBuf>, deref: bool, tag: Tag) -> Result<Self, ShellError> {
+        let path = path.into();
+        let m = if deref {
+            std::fs::metadata(&path)
+        } else {
+            std::fs::symlink_metadata(&path)
+        };
+
+        match m {
+            Ok(d) => {
+                let block_size = file_real_size_fast(&path, &d).ok();
+
+                Ok(FileInfo {
+                    path,
+                    blocks: block_size,
+                    size: d.len(),
+                    tag,
+                })
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+}
+
+impl DirInfo {
+    pub fn new(
+        path: impl Into<PathBuf>,
+        params: &DirBuilder,
+        depth: Option<u64>,
+        ctrl_c: Arc<AtomicBool>,
+    ) -> Self {
+        let path = path.into();
+
+        let mut s = Self {
+            dirs: Vec::new(),
+            errors: Vec::new(),
+            files: Vec::new(),
+            size: 0,
+            blocks: 0,
+            tag: params.tag.clone(),
+            path,
+        };
+
+        match std::fs::read_dir(&s.path) {
+            Ok(d) => {
+                for f in d {
+                    if ctrl_c.load(Ordering::SeqCst) {
+                        break;
+                    }
+
+                    match f {
+                        Ok(i) => match i.file_type() {
+                            Ok(t) if t.is_dir() => {
+                                s = s.add_dir(i.path(), depth, &params, ctrl_c.clone())
+                            }
+                            Ok(_t) => s = s.add_file(i.path(), &params),
+                            Err(e) => s = s.add_error(e.into()),
+                        },
+                        Err(e) => s = s.add_error(e.into()),
+                    }
+                }
+            }
+            Err(e) => s = s.add_error(e.into()),
+        }
+        s
+    }
+
+    fn add_dir(
+        mut self,
+        path: impl Into<PathBuf>,
+        mut depth: Option<u64>,
+        params: &DirBuilder,
+        ctrl_c: Arc<AtomicBool>,
+    ) -> Self {
+        if let Some(current) = depth {
+            if let Some(new) = current.checked_sub(1) {
+                depth = Some(new);
+            } else {
+                return self;
+            }
+        }
+
+        let d = DirInfo::new(path, &params, depth, ctrl_c);
+        self.size += d.size;
+        self.blocks += d.blocks;
+        self.dirs.push(d);
+        self
+    }
+
+    fn add_file(mut self, f: impl Into<PathBuf>, params: &DirBuilder) -> Self {
+        let f = f.into();
+        let include = params
+            .exclude
+            .as_ref()
+            .map_or(true, |x| !x.matches_path(&f));
+        if include {
+            match FileInfo::new(f, params.deref, self.tag.clone()) {
+                Ok(file) => {
+                    let inc = params.min.map_or(true, |s| file.size >= s);
+                    if inc {
+                        self.size += file.size;
+                        self.blocks += file.blocks.unwrap_or(0);
+                        if params.all {
+                            self.files.push(file);
+                        }
+                    }
+                }
+                Err(e) => self = self.add_error(e),
+            }
+        }
+        self
+    }
+
+    fn add_error(mut self, e: ShellError) -> Self {
+        self.errors.push(e);
+        self
+    }
+
+    pub fn get_size(&self) -> u64 {
+        self.size
+    }
+}
+
+impl From<DirInfo> for Value {
+    fn from(d: DirInfo) -> Self {
+        let mut r: IndexMap<String, Value> = IndexMap::new();
+
+        r.insert(
+            "path".to_string(),
+            UntaggedValue::filepath(d.path).into_value(&d.tag),
+        );
+
+        r.insert(
+            "apparent".to_string(),
+            UntaggedValue::filesize(d.size).into_value(&d.tag),
+        );
+
+        r.insert(
+            "physical".to_string(),
+            UntaggedValue::filesize(d.blocks).into_value(&d.tag),
+        );
+
+        r.insert("directories".to_string(), value_from_vec(d.dirs, &d.tag));
+
+        r.insert("files".to_string(), value_from_vec(d.files, &d.tag));
+
+        if !d.errors.is_empty() {
+            let v = UntaggedValue::Table(
+                d.errors
+                    .into_iter()
+                    .map(move |e| UntaggedValue::Error(e).into_untagged_value())
+                    .collect::<Vec<Value>>(),
+            )
+            .into_value(&d.tag);
+
+            r.insert("errors".to_string(), v);
+        }
+
+        Value {
+            value: UntaggedValue::row(r),
+            tag: d.tag,
+        }
+    }
+}
+
+impl From<FileInfo> for Value {
+    fn from(f: FileInfo) -> Self {
+        let mut r: IndexMap<String, Value> = IndexMap::new();
+
+        r.insert(
+            "path".to_string(),
+            UntaggedValue::filepath(f.path).into_value(&f.tag),
+        );
+
+        r.insert(
+            "apparent".to_string(),
+            UntaggedValue::filesize(f.size).into_value(&f.tag),
+        );
+
+        let b = f
+            .blocks
+            .map(UntaggedValue::filesize)
+            .unwrap_or_else(UntaggedValue::nothing)
+            .into_value(&f.tag);
+
+        r.insert("physical".to_string(), b);
+
+        r.insert(
+            "directories".to_string(),
+            UntaggedValue::nothing().into_value(&f.tag),
+        );
+
+        r.insert(
+            "files".to_string(),
+            UntaggedValue::nothing().into_value(&f.tag),
+        );
+
+        UntaggedValue::row(r).into_value(&f.tag)
+    }
+}
+
+fn value_from_vec<V>(vec: Vec<V>, tag: &Tag) -> Value
+where
+    V: Into<Value>,
+{
+    if vec.is_empty() {
+        UntaggedValue::nothing()
+    } else {
+        let values = vec.into_iter().map(Into::into).collect::<Vec<Value>>();
+        UntaggedValue::Table(values)
+    }
+    .into_value(tag)
+}

crates/nu-engine/src/filesystem/mod.rs

@@ -0,0 +1,4 @@
+pub(crate) mod dir_info;
+pub mod filesystem_shell;
+pub mod path;
+pub(crate) mod utils;

crates/nu-engine/src/filesystem/path.rs

@@ -0,0 +1,114 @@
+use std::io;
+use std::path::{Component, Path, PathBuf};
+
+pub fn absolutize<P, Q>(relative_to: P, path: Q) -> PathBuf
+where
+    P: AsRef<Path>,
+    Q: AsRef<Path>,
+{
+    let path = if path.as_ref() == Path::new(".") {
+        // Joining a Path with '.' appends a '.' at the end, making the prompt
+        // more ugly - so we don't do anything, which should result in an equal
+        // path on all supported systems.
+        relative_to.as_ref().to_owned()
+    } else {
+        relative_to.as_ref().join(path)
+    };
+
+    let (relative_to, path) = {
+        let components: Vec<_> = path.components().collect();
+        let separator = components
+            .iter()
+            .enumerate()
+            .find(|(_, c)| c == &&Component::CurDir || c == &&Component::ParentDir);
+
+        if let Some((index, _)) = separator {
+            let (absolute, relative) = components.split_at(index);
+            let absolute: PathBuf = absolute.iter().collect();
+            let relative: PathBuf = relative.iter().collect();
+
+            (absolute, relative)
+        } else {
+            (relative_to.as_ref().to_path_buf(), path)
+        }
+    };
+
+    let path = if path.is_relative() {
+        let mut result = relative_to;
+        path.components().for_each(|component| match component {
+            Component::ParentDir => {
+                result.pop();
+            }
+            Component::Normal(normal) => result.push(normal),
+            _ => {}
+        });
+
+        result
+    } else {
+        path
+    };
+
+    dunce::simplified(&path).to_path_buf()
+}
+
+pub fn canonicalize<P, Q>(relative_to: P, path: Q) -> io::Result<PathBuf>
+where
+    P: AsRef<Path>,
+    Q: AsRef<Path>,
+{
+    let absolutized = absolutize(&relative_to, path);
+    let path = match std::fs::read_link(&absolutized) {
+        Ok(resolved) => {
+            let parent = absolutized.parent().unwrap_or(&absolutized);
+            absolutize(parent, resolved)
+        }
+
+        Err(e) => {
+            if absolutized.exists() {
+                absolutized
+            } else {
+                return Err(e);
+            }
+        }
+    };
+
+    Ok(dunce::simplified(&path).to_path_buf())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io;
+
+    #[test]
+    fn absolutize_two_dots() {
+        let relative_to = Path::new("/foo/bar");
+        let path = Path::new("..");
+
+        assert_eq!(
+            PathBuf::from("/foo"), // missing path
+            absolutize(relative_to, path)
+        );
+    }
+
+    #[test]
+    fn canonicalize_should_succeed() -> io::Result<()> {
+        let relative_to = Path::new("/foo/bar");
+        let path = Path::new("../..");
+
+        assert_eq!(
+            PathBuf::from("/"), // existing path
+            canonicalize(relative_to, path)?,
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn canonicalize_should_fail() {
+        let relative_to = Path::new("/foo/bar/baz"); // '/foo' is missing
+        let path = Path::new("../..");
+
+        assert!(canonicalize(relative_to, path).is_err());
+    }
+}

crates/nu-engine/src/filesystem/utils.rs

@@ -0,0 +1,210 @@
+use crate::filesystem::path::canonicalize;
+use nu_errors::ShellError;
+use std::path::{Path, PathBuf};
+
+#[derive(Default)]
+pub struct FileStructure {
+    pub resources: Vec<Res>,
+}
+
+impl FileStructure {
+    pub fn new() -> FileStructure {
+        FileStructure {
+            resources: Vec::<Res>::new(),
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn contains_more_than_one_file(&self) -> bool {
+        self.resources.len() > 1
+    }
+
+    #[allow(dead_code)]
+    pub fn contains_files(&self) -> bool {
+        !self.resources.is_empty()
+    }
+
+    pub fn paths_applying_with<F>(
+        &mut self,
+        to: F,
+    ) -> Result<Vec<(PathBuf, PathBuf)>, Box<dyn std::error::Error>>
+    where
+        F: Fn((PathBuf, usize)) -> Result<(PathBuf, PathBuf), Box<dyn std::error::Error>>,
+    {
+        self.resources
+            .iter()
+            .map(|f| (PathBuf::from(&f.loc), f.at))
+            .map(|f| to(f))
+            .collect()
+    }
+
+    pub fn walk_decorate(&mut self, start_path: &Path) -> Result<(), ShellError> {
+        self.resources = Vec::<Res>::new();
+        self.build(start_path, 0)?;
+        self.resources.sort();
+
+        Ok(())
+    }
+
+    fn build(&mut self, src: &Path, lvl: usize) -> Result<(), ShellError> {
+        let source = canonicalize(std::env::current_dir()?, src)?;
+
+        if source.is_dir() {
+            for entry in std::fs::read_dir(src)? {
+                let entry = entry?;
+                let path = entry.path();
+
+                if path.is_dir() {
+                    self.build(&path, lvl + 1)?;
+                }
+
+                self.resources.push(Res {
+                    loc: path.to_path_buf(),
+                    at: lvl,
+                });
+            }
+        } else {
+            self.resources.push(Res {
+                loc: source,
+                at: lvl,
+            });
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, Eq, Ord, PartialEq, PartialOrd)]
+pub struct Res {
+    pub at: usize,
+    pub loc: PathBuf,
+}
+
+impl Res {}
+
+#[cfg(test)]
+mod tests {
+    use super::{FileStructure, Res};
+    use nu_protocol::{TaggedDictBuilder, UntaggedValue, Value, ValueResource, ValueStructure};
+    use nu_source::Tag;
+    use nu_test_support::{fs::Stub::EmptyFile, playground::Playground};
+    use std::path::PathBuf;
+
+    fn structured_sample_record(key: &str, value: &str) -> Value {
+        let mut record = TaggedDictBuilder::new(Tag::unknown());
+        record.insert_untagged(key, UntaggedValue::string(value));
+        record.into_value()
+    }
+
+    fn sample_nushell_source_code() -> Value {
+        /*
+            src
+             commands
+              plugins => "sys.rs"
+             tests
+              helpers => "mod.rs"
+        */
+
+        let mut src = TaggedDictBuilder::new(Tag::unknown());
+        let mut record = TaggedDictBuilder::new(Tag::unknown());
+
+        record.insert_value("commands", structured_sample_record("plugins", "sys.rs"));
+        record.insert_value("tests", structured_sample_record("helpers", "mod.rs"));
+        src.insert_value("src", record.into_value());
+
+        src.into_value()
+    }
+
+    #[test]
+    fn prepares_and_decorates_value_filesystemlike_sources() {
+        let mut res = ValueStructure::new();
+
+        res.walk_decorate(&sample_nushell_source_code())
+            .expect("Can not decorate values traversal.");
+
+        assert_eq!(
+            res.resources,
+            vec![
+                ValueResource {
+                    loc: PathBuf::from("src"),
+                    at: 0,
+                },
+                ValueResource {
+                    loc: PathBuf::from("commands"),
+                    at: 1,
+                },
+                ValueResource {
+                    loc: PathBuf::from("tests"),
+                    at: 1,
+                },
+                ValueResource {
+                    loc: PathBuf::from("helpers"),
+                    at: 2,
+                },
+                ValueResource {
+                    loc: PathBuf::from("plugins"),
+                    at: 2,
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn recognizes_if_path_exists_in_value_filesystemlike_sources() {
+        let mut res = ValueStructure::new();
+
+        res.walk_decorate(&sample_nushell_source_code())
+            .expect("Can not decorate values traversal.");
+
+        assert!(res.exists(&PathBuf::from("/")));
+
+        assert!(res.exists(&PathBuf::from("src/commands/plugins")));
+        assert!(res.exists(&PathBuf::from("src/commands")));
+        assert!(res.exists(&PathBuf::from("src/tests")));
+        assert!(res.exists(&PathBuf::from("src/tests/helpers")));
+        assert!(res.exists(&PathBuf::from("src")));
+
+        assert!(res.exists(&PathBuf::from("/src/commands/plugins")));
+        assert!(res.exists(&PathBuf::from("/src/commands")));
+        assert!(res.exists(&PathBuf::from("/src/tests")));
+        assert!(res.exists(&PathBuf::from("/src/tests/helpers")));
+        assert!(res.exists(&PathBuf::from("/src")));
+
+        assert!(!res.exists(&PathBuf::from("/not_valid")));
+        assert!(!res.exists(&PathBuf::from("/src/not_valid")));
+    }
+
+    #[test]
+    fn prepares_and_decorates_filesystem_source_files() {
+        Playground::setup("file_structure_test", |dirs, sandbox| {
+            sandbox.with_files(vec![
+                EmptyFile("sample.ini"),
+                EmptyFile("sample.eml"),
+                EmptyFile("cargo_sample.toml"),
+            ]);
+
+            let mut res = FileStructure::new();
+
+            res.walk_decorate(&dirs.test())
+                .expect("Can not decorate files traversal.");
+
+            assert_eq!(
+                res.resources,
+                vec![
+                    Res {
+                        loc: dirs.test().join("cargo_sample.toml"),
+                        at: 0
+                    },
+                    Res {
+                        loc: dirs.test().join("sample.eml"),
+                        at: 0
+                    },
+                    Res {
+                        loc: dirs.test().join("sample.ini"),
+                        at: 0
+                    }
+                ]
+            );
+        })
+    }
+}

crates/nu-engine/src/history_path.rs

@@ -0,0 +1,22 @@
+use nu_data::config::Conf;
+use std::path::PathBuf;
+
+const DEFAULT_LOCATION: &str = "history.txt";
+
+pub fn history_path(config: &dyn Conf) -> PathBuf {
+    let default_path = nu_data::config::user_data()
+        .map(|mut p| {
+            p.push(DEFAULT_LOCATION);
+            p
+        })
+        .unwrap_or_else(|_| PathBuf::from(DEFAULT_LOCATION));
+
+    config
+        .var("history-path")
+        .map_or(default_path.clone(), |custom_path| {
+            match custom_path.as_string() {
+                Ok(path) => PathBuf::from(path),
+                Err(_) => default_path,
+            }
+        })
+}

crates/nu-engine/src/lib.rs

@@ -0,0 +1,39 @@
+mod call_info;
+mod command_args;
+pub mod deserializer;
+pub mod documentation;
+mod env;
+mod evaluate;
+mod evaluation_context;
+mod example;
+pub mod filesystem;
+mod history_path;
+mod maybe_text_codec;
+pub mod plugin;
+pub mod shell;
+mod whole_stream_command;
+
+pub use crate::call_info::UnevaluatedCallInfo;
+pub use crate::command_args::{
+    CommandArgs, EvaluatedCommandArgs, EvaluatedWholeStreamCommandArgs, RawCommandArgs,
+};
+pub use crate::documentation::{generate_docs, get_documentation, get_help};
+pub use crate::env::environment::Env;
+pub use crate::env::host::FakeHost;
+pub use crate::env::host::Host;
+pub use crate::evaluate::block::run_block;
+pub use crate::evaluate::evaluator::evaluate_baseline_expr;
+pub use crate::evaluate::scope::Scope;
+pub use crate::evaluation_context::EvaluationContext;
+pub use crate::example::Example;
+pub use crate::filesystem::dir_info::{DirBuilder, DirInfo, FileInfo};
+pub use crate::filesystem::filesystem_shell::FilesystemShell;
+pub use crate::filesystem::path;
+pub use crate::history_path::history_path;
+pub use crate::maybe_text_codec::{MaybeTextCodec, StringOrBinary};
+pub use crate::shell::help_shell::{command_dict, HelpShell};
+pub use crate::shell::painter::Painter;
+pub use crate::shell::palette::{DefaultPalette, Palette};
+pub use crate::shell::shell_manager::ShellManager;
+pub use crate::shell::value_shell::ValueShell;
+pub use crate::whole_stream_command::{whole_stream_command, Command, WholeStreamCommand};

crates/nu-engine/src/maybe_text_codec.rs

@@ -0,0 +1,126 @@
+use bytes::{BufMut, Bytes, BytesMut};
+
+use nu_errors::ShellError;
+
+use encoding_rs::{CoderResult, Decoder, Encoding, UTF_8};
+
+#[cfg(not(test))]
+const OUTPUT_BUFFER_SIZE: usize = 8192;
+#[cfg(test)]
+const OUTPUT_BUFFER_SIZE: usize = 4;
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum StringOrBinary {
+    String(String),
+    Binary(Vec<u8>),
+}
+
+pub struct MaybeTextCodec {
+    decoder: Decoder,
+}
+
+impl MaybeTextCodec {
+    // The constructor takes an Option<&'static Encoding>, because an absence of an encoding indicates that we want BOM sniffing enabled
+    pub fn new(encoding: Option<&'static Encoding>) -> Self {
+        let decoder = match encoding {
+            Some(e) => e.new_decoder_with_bom_removal(),
+            None => UTF_8.new_decoder(),
+        };
+        MaybeTextCodec { decoder }
+    }
+}
+
+impl Default for MaybeTextCodec {
+    fn default() -> Self {
+        MaybeTextCodec {
+            decoder: UTF_8.new_decoder(),
+        }
+    }
+}
+
+impl futures_codec::Encoder for MaybeTextCodec {
+    type Item = StringOrBinary;
+    type Error = std::io::Error;
+
+    fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
+        match item {
+            StringOrBinary::String(s) => {
+                dst.reserve(s.len());
+                dst.put(s.as_bytes());
+                Ok(())
+            }
+            StringOrBinary::Binary(b) => {
+                dst.reserve(b.len());
+                dst.put(Bytes::from(b));
+                Ok(())
+            }
+        }
+    }
+}
+
+impl futures_codec::Decoder for MaybeTextCodec {
+    type Item = StringOrBinary;
+    type Error = ShellError;
+
+    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
+        if src.is_empty() {
+            return Ok(None);
+        }
+
+        let mut s = String::with_capacity(OUTPUT_BUFFER_SIZE);
+
+        let (res, _read, replacements) = self.decoder.decode_to_string(src, &mut s, false);
+
+        let result = if replacements {
+            // If we had to make replacements when converting to utf8, fall back to binary
+            StringOrBinary::Binary(src.to_vec())
+        } else {
+            // If original buffer size is too small, we continue to allocate new Strings and append
+            // them to the result until the input buffer is smaller than the allocated String
+            if let CoderResult::OutputFull = res {
+                let mut buffer = String::with_capacity(OUTPUT_BUFFER_SIZE);
+                loop {
+                    let (res, _read, _replacements) =
+                        self.decoder
+                            .decode_to_string(&src[s.len()..], &mut buffer, false);
+                    s.push_str(&buffer);
+
+                    if let CoderResult::InputEmpty = res {
+                        break;
+                    }
+
+                    buffer.clear();
+                }
+            }
+
+            StringOrBinary::String(s)
+        };
+
+        src.clear();
+
+        Ok(Some(result))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{MaybeTextCodec, StringOrBinary};
+    use bytes::BytesMut;
+    use futures_codec::Decoder;
+
+    // TODO: Write some more tests
+
+    #[test]
+    fn should_consume_all_bytes_from_source_when_temporary_buffer_overflows() {
+        let mut maybe_text = MaybeTextCodec::new(None);
+        let mut bytes = BytesMut::from("0123456789");
+
+        let text = maybe_text.decode(&mut bytes);
+
+        assert_eq!(
+            Ok(Some(StringOrBinary::String("0123456789".to_string()))),
+            text
+        );
+        assert!(bytes.is_empty());
+    }
+}

crates/nu-engine/src/plugin/build_plugin.rs

@@ -0,0 +1,172 @@
+use crate::plugin::run_plugin::PluginCommandBuilder;
+use log::trace;
+use nu_errors::ShellError;
+use nu_plugin::jsonrpc::JsonRpc;
+use nu_protocol::{Signature, Value};
+use std::io::{BufRead, BufReader, Write};
+use std::process::{Child, Command, Stdio};
+
+use rayon::prelude::*;
+
+pub fn build_plugin_command(
+    path: &std::path::Path,
+) -> Result<Option<PluginCommandBuilder>, ShellError> {
+    let ext = path.extension();
+    let ps1_file = match ext {
+        Some(ext) => ext == "ps1",
+        None => false,
+    };
+
+    let mut child: Child = if ps1_file {
+        Command::new("pwsh")
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .args(&[
+                "-NoLogo",
+                "-NoProfile",
+                "-ExecutionPolicy",
+                "Bypass",
+                "-File",
+                &path.to_string_lossy(),
+            ])
+            .spawn()
+            .expect("Failed to spawn PowerShell process")
+    } else {
+        Command::new(path)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .expect("Failed to spawn child process")
+    };
+
+    let stdin = child.stdin.as_mut().expect("Failed to open stdin");
+    let stdout = child.stdout.as_mut().expect("Failed to open stdout");
+
+    let mut reader = BufReader::new(stdout);
+
+    let request = JsonRpc::new("config", Vec::<Value>::new());
+    let request_raw = serde_json::to_string(&request)?;
+    trace!(target: "nu::load", "plugin infrastructure config -> path {:#?}, request {:?}", &path, &request_raw);
+    stdin.write_all(format!("{}\n", request_raw).as_bytes())?;
+    let path = dunce::canonicalize(path)?;
+
+    let mut input = String::new();
+    let result = match reader.read_line(&mut input) {
+        Ok(count) => {
+            trace!(target: "nu::load", "plugin infrastructure -> config response for {:#?}", &path);
+            trace!(target: "nu::load", "plugin infrastructure -> processing response ({} bytes)", count);
+            trace!(target: "nu::load", "plugin infrastructure -> response: {}", input);
+
+            let response = serde_json::from_str::<JsonRpc<Result<Signature, ShellError>>>(&input);
+            match response {
+                Ok(jrpc) => match jrpc.params {
+                    Ok(params) => {
+                        let fname = path.to_string_lossy();
+
+                        trace!(target: "nu::load", "plugin infrastructure -> processing {:?}", params);
+
+                        let name = params.name.clone();
+
+                        let fname = fname.to_string();
+
+                        Ok(Some(PluginCommandBuilder::new(&name, &fname, params)))
+                    }
+                    Err(e) => Err(e),
+                },
+                Err(e) => {
+                    trace!(target: "nu::load", "plugin infrastructure -> incompatible {:?}", input);
+                    Err(ShellError::untagged_runtime_error(format!(
+                        "Error: {:?}",
+                        e
+                    )))
+                }
+            }
+        }
+        Err(e) => Err(ShellError::untagged_runtime_error(format!(
+            "Error: {:?}",
+            e
+        ))),
+    };
+
+    let _ = child.wait();
+
+    result
+}
+
+pub fn scan(
+    paths: Vec<std::path::PathBuf>,
+) -> Result<Vec<crate::whole_stream_command::Command>, ShellError> {
+    let mut plugins = vec![];
+
+    let opts = glob::MatchOptions {
+        case_sensitive: false,
+        require_literal_separator: false,
+        require_literal_leading_dot: false,
+    };
+
+    for path in paths {
+        let mut pattern = path.to_path_buf();
+
+        pattern.push(std::path::Path::new("nu_plugin_[a-z0-9][a-z0-9]*"));
+
+        let plugs: Vec<_> = glob::glob_with(&pattern.to_string_lossy(), opts)?
+            .filter_map(|x| x.ok())
+            .collect();
+
+        let plugs: Vec<_> = plugs
+            .par_iter()
+            .filter_map(|path| {
+                let bin_name = {
+                    if let Some(name) = path.file_name() {
+                        name.to_str().unwrap_or("")
+                    } else {
+                        ""
+                    }
+                };
+
+                // allow plugins with extensions on all platforms
+                let is_valid_name = {
+                    bin_name
+                        .chars()
+                        .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.')
+                };
+
+                let is_executable = {
+                    #[cfg(windows)]
+                    {
+                        bin_name.ends_with(".exe") 
+                        || bin_name.ends_with(".bat")
+                        || bin_name.ends_with(".cmd")
+                        || bin_name.ends_with(".py")
+                        || bin_name.ends_with(".ps1")
+                    }
+
+                    #[cfg(not(windows))]
+                    {
+                        !bin_name.contains('.')
+                        || (bin_name.ends_with('.')
+                        || bin_name.ends_with(".py")
+                        || bin_name.ends_with(".rb")
+                        || bin_name.ends_with(".sh")
+                        || bin_name.ends_with(".bash")
+                        || bin_name.ends_with(".zsh")
+                        || bin_name.ends_with(".pl")
+                        || bin_name.ends_with(".awk")
+                        || bin_name.ends_with(".ps1"))
+                    }
+                };
+
+                if is_valid_name && is_executable {
+                    trace!(target: "nu::load", "plugin infrastructure -> Trying {:?}", path.display());
+                    build_plugin_command(&path).unwrap_or(None)
+                } else {
+                    None
+                }
+            }).map(|p| p.build())
+            .filter_map(Result::ok)
+            .collect::<Vec<crate::whole_stream_command::Command>>();
+        plugins.extend(plugs);
+    }
+
+    Ok(plugins)
+}

crates/nu-engine/src/plugin/mod.rs

@@ -0,0 +1,2 @@
+pub mod build_plugin;
+pub(crate) mod run_plugin;

crates/nu-engine/src/plugin/run_plugin.rs

@@ -0,0 +1,451 @@
+use crate::command_args::CommandArgs;
+use crate::whole_stream_command::{whole_stream_command, WholeStreamCommand};
+use async_trait::async_trait;
+use derive_new::new;
+use futures::StreamExt;
+use log::trace;
+use nu_errors::ShellError;
+use nu_plugin::jsonrpc::JsonRpc;
+use nu_protocol::{Primitive, ReturnValue, Signature, UntaggedValue, Value};
+use nu_stream::{OutputStream, ToOutputStream};
+use serde::{self, Deserialize, Serialize};
+use std::collections::VecDeque;
+use std::io::prelude::*;
+use std::io::BufReader;
+use std::io::Write;
+use std::path::Path;
+use std::process::{Child, Command, Stdio};
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "method")]
+#[allow(non_camel_case_types)]
+pub enum NuResult {
+    response {
+        params: Result<VecDeque<ReturnValue>, ShellError>,
+    },
+}
+
+enum PluginCommand {
+    Filter(PluginFilter),
+    Sink(PluginSink),
+}
+
+impl PluginCommand {
+    fn command(self) -> Result<crate::whole_stream_command::Command, ShellError> {
+        match self {
+            PluginCommand::Filter(cmd) => Ok(whole_stream_command(cmd)),
+            PluginCommand::Sink(cmd) => Ok(whole_stream_command(cmd)),
+        }
+    }
+}
+
+enum PluginMode {
+    Filter,
+    Sink,
+}
+
+pub struct PluginCommandBuilder {
+    mode: PluginMode,
+    name: String,
+    path: String,
+    config: Signature,
+}
+
+impl PluginCommandBuilder {
+    pub fn new(
+        name: impl Into<String>,
+        path: impl Into<String>,
+        config: impl Into<Signature>,
+    ) -> Self {
+        let config = config.into();
+
+        PluginCommandBuilder {
+            mode: if config.is_filter {
+                PluginMode::Filter
+            } else {
+                PluginMode::Sink
+            },
+            name: name.into(),
+            path: path.into(),
+            config,
+        }
+    }
+
+    pub fn build(&self) -> Result<crate::whole_stream_command::Command, ShellError> {
+        let mode = &self.mode;
+
+        let name = self.name.clone();
+        let path = self.path.clone();
+        let config = self.config.clone();
+
+        let cmd = match mode {
+            PluginMode::Filter => PluginCommand::Filter(PluginFilter { name, path, config }),
+            PluginMode::Sink => PluginCommand::Sink(PluginSink { name, path, config }),
+        };
+
+        cmd.command()
+    }
+}
+
+#[derive(new)]
+pub struct PluginFilter {
+    name: String,
+    path: String,
+    config: Signature,
+}
+
+#[async_trait]
+impl WholeStreamCommand for PluginFilter {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.config.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.config.usage
+    }
+
+    async fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
+        run_filter(self.path.clone(), (args)).await
+    }
+}
+
+async fn run_filter(path: String, args: CommandArgs) -> Result<OutputStream, ShellError> {
+    trace!("filter_plugin :: {}", path);
+
+    let bos = futures::stream::iter(vec![
+        UntaggedValue::Primitive(Primitive::BeginningOfStream).into_untagged_value()
+    ]);
+    let eos = futures::stream::iter(vec![
+        UntaggedValue::Primitive(Primitive::EndOfStream).into_untagged_value()
+    ]);
+
+    let args = args.evaluate_once().await?;
+
+    let real_path = Path::new(&path);
+    let ext = real_path.extension();
+    let ps1_file = match ext {
+        Some(ext) => ext == "ps1",
+        None => false,
+    };
+
+    let mut child: Child = if ps1_file {
+        Command::new("pwsh")
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .args(&[
+                "-NoLogo",
+                "-NoProfile",
+                "-ExecutionPolicy",
+                "Bypass",
+                "-File",
+                &real_path.to_string_lossy(),
+            ])
+            .spawn()
+            .expect("Failed to spawn PowerShell process")
+    } else {
+        Command::new(path)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .expect("Failed to spawn child process")
+    };
+
+    let call_info = args.call_info.clone();
+
+    trace!("filtering :: {:?}", call_info);
+
+    Ok(bos
+        .chain(args.input)
+        .chain(eos)
+        .map(move |item| {
+            match item {
+                Value {
+                    value: UntaggedValue::Primitive(Primitive::BeginningOfStream),
+                    ..
+                } => {
+                    // Beginning of the stream
+                    let stdin = child.stdin.as_mut().expect("Failed to open stdin");
+                    let stdout = child.stdout.as_mut().expect("Failed to open stdout");
+
+                    let mut reader = BufReader::new(stdout);
+
+                    let request = JsonRpc::new("begin_filter", call_info.clone());
+                    let request_raw = serde_json::to_string(&request);
+                    trace!("begin_filter:request {:?}", &request_raw);
+
+                    match request_raw {
+                        Err(_) => {
+                            return OutputStream::one(Err(ShellError::labeled_error(
+                                "Could not load json from plugin",
+                                "could not load json from plugin",
+                                &call_info.name_tag,
+                            )));
+                        }
+                        Ok(request_raw) => {
+                            match stdin.write(format!("{}\n", request_raw).as_bytes()) {
+                                Ok(_) => {}
+                                Err(err) => {
+                                    return OutputStream::one(Err(ShellError::unexpected(
+                                        format!("{}", err),
+                                    )));
+                                }
+                            }
+                        }
+                    }
+
+                    let mut input = String::new();
+                    match reader.read_line(&mut input) {
+                        Ok(_) => {
+                            let response = serde_json::from_str::<NuResult>(&input);
+                            trace!("begin_filter:response {:?}", &response);
+
+                            match response {
+                                Ok(NuResult::response { params }) => match params {
+                                    Ok(params) => futures::stream::iter(params).to_output_stream(),
+                                    Err(e) => futures::stream::iter(vec![ReturnValue::Err(e)])
+                                        .to_output_stream(),
+                                },
+
+                                Err(e) => OutputStream::one(Err(
+                                    ShellError::untagged_runtime_error(format!(
+                                        "Error while processing begin_filter response: {:?} {}",
+                                        e, input
+                                    )),
+                                )),
+                            }
+                        }
+                        Err(e) => OutputStream::one(Err(ShellError::untagged_runtime_error(
+                            format!("Error while reading begin_filter response: {:?}", e),
+                        ))),
+                    }
+                }
+                Value {
+                    value: UntaggedValue::Primitive(Primitive::EndOfStream),
+                    ..
+                } => {
+                    // post stream contents
+                    let stdin = child.stdin.as_mut().expect("Failed to open stdin");
+                    let stdout = child.stdout.as_mut().expect("Failed to open stdout");
+
+                    let mut reader = BufReader::new(stdout);
+
+                    let request: JsonRpc<std::vec::Vec<Value>> = JsonRpc::new("end_filter", vec![]);
+                    let request_raw = serde_json::to_string(&request);
+                    trace!("end_filter:request {:?}", &request_raw);
+
+                    match request_raw {
+                        Err(_) => {
+                            return OutputStream::one(Err(ShellError::labeled_error(
+                                "Could not load json from plugin",
+                                "could not load json from plugin",
+                                &call_info.name_tag,
+                            )));
+                        }
+                        Ok(request_raw) => {
+                            match stdin.write(format!("{}\n", request_raw).as_bytes()) {
+                                Ok(_) => {}
+                                Err(err) => {
+                                    return OutputStream::one(Err(ShellError::unexpected(
+                                        format!("{}", err),
+                                    )));
+                                }
+                            }
+                        }
+                    }
+
+                    let mut input = String::new();
+                    let stream = match reader.read_line(&mut input) {
+                        Ok(_) => {
+                            let response = serde_json::from_str::<NuResult>(&input);
+                            trace!("end_filter:response {:?}", &response);
+
+                            match response {
+                                Ok(NuResult::response { params }) => match params {
+                                    Ok(params) => futures::stream::iter(params).to_output_stream(),
+                                    Err(e) => futures::stream::iter(vec![ReturnValue::Err(e)])
+                                        .to_output_stream(),
+                                },
+                                Err(e) => futures::stream::iter(vec![Err(
+                                    ShellError::untagged_runtime_error(format!(
+                                        "Error while processing end_filter response: {:?} {}",
+                                        e, input
+                                    )),
+                                )])
+                                .to_output_stream(),
+                            }
+                        }
+                        Err(e) => {
+                            futures::stream::iter(vec![Err(ShellError::untagged_runtime_error(
+                                format!("Error while reading end_filter response: {:?}", e),
+                            ))])
+                            .to_output_stream()
+                        }
+                    };
+
+                    let stdin = child.stdin.as_mut().expect("Failed to open stdin");
+
+                    let request: JsonRpc<std::vec::Vec<Value>> = JsonRpc::new("quit", vec![]);
+                    let request_raw = serde_json::to_string(&request);
+                    trace!("quit:request {:?}", &request_raw);
+
+                    match request_raw {
+                        Ok(request_raw) => {
+                            let _ = stdin.write(format!("{}\n", request_raw).as_bytes());
+                            // TODO: Handle error
+                        }
+                        Err(e) => {
+                            return OutputStream::one(Err(ShellError::untagged_runtime_error(
+                                format!("Error while processing quit response: {:?}", e),
+                            )));
+                        }
+                    }
+                    let _ = child.wait();
+
+                    stream
+                }
+
+                v => {
+                    // Stream contents
+                    let stdin = child.stdin.as_mut().expect("Failed to open stdin");
+                    let stdout = child.stdout.as_mut().expect("Failed to open stdout");
+
+                    let mut reader = BufReader::new(stdout);
+
+                    let request = JsonRpc::new("filter", v);
+                    let request_raw = serde_json::to_string(&request);
+                    trace!("filter:request {:?}", &request_raw);
+
+                    match request_raw {
+                        Ok(request_raw) => {
+                            let _ = stdin.write(format!("{}\n", request_raw).as_bytes());
+                            // TODO: Handle error
+                        }
+                        Err(e) => {
+                            return OutputStream::one(Err(ShellError::untagged_runtime_error(
+                                format!("Error while processing filter response: {:?}", e),
+                            )));
+                        }
+                    }
+
+                    let mut input = String::new();
+                    match reader.read_line(&mut input) {
+                        Ok(_) => {
+                            let response = serde_json::from_str::<NuResult>(&input);
+                            trace!("filter:response {:?}", &response);
+
+                            match response {
+                                Ok(NuResult::response { params }) => match params {
+                                    Ok(params) => futures::stream::iter(params).to_output_stream(),
+                                    Err(e) => futures::stream::iter(vec![ReturnValue::Err(e)])
+                                        .to_output_stream(),
+                                },
+                                Err(e) => OutputStream::one(Err(
+                                    ShellError::untagged_runtime_error(format!(
+                                    "Error while processing filter response: {:?}\n== input ==\n{}",
+                                    e, input
+                                )),
+                                )),
+                            }
+                        }
+                        Err(e) => OutputStream::one(Err(ShellError::untagged_runtime_error(
+                            format!("Error while reading filter response: {:?}", e),
+                        ))),
+                    }
+                }
+            }
+        })
+        .flatten()
+        .to_output_stream())
+}
+
+#[derive(new)]
+pub struct PluginSink {
+    name: String,
+    path: String,
+    config: Signature,
+}
+
+#[async_trait]
+impl WholeStreamCommand for PluginSink {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.config.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.config.usage
+    }
+
+    async fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
+        run_sink(self.path.clone(), args).await
+    }
+}
+
+async fn run_sink(path: String, args: CommandArgs) -> Result<OutputStream, ShellError> {
+    let args = args.evaluate_once().await?;
+    let call_info = args.call_info.clone();
+
+    let input: Vec<Value> = args.input.collect().await;
+
+    let request = JsonRpc::new("sink", (call_info.clone(), input));
+    let request_raw = serde_json::to_string(&request);
+    if let Ok(request_raw) = request_raw {
+        if let Ok(mut tmpfile) = tempfile::NamedTempFile::new() {
+            let _ = writeln!(tmpfile, "{}", request_raw);
+            let _ = tmpfile.flush();
+
+            let real_path = Path::new(&path);
+            let ext = real_path.extension();
+            let ps1_file = match ext {
+                Some(ext) => ext == "ps1",
+                None => false,
+            };
+
+            // TODO: This sink may not work in powershell, trying to find
+            // an example of what CallInfo would look like in this temp file
+            let child = if ps1_file {
+                Command::new("pwsh")
+                    .args(&[
+                        "-NoLogo",
+                        "-NoProfile",
+                        "-ExecutionPolicy",
+                        "Bypass",
+                        "-File",
+                        &real_path.to_string_lossy(),
+                        &tmpfile
+                            .path()
+                            .to_str()
+                            .expect("Failed getting tmpfile path"),
+                    ])
+                    .spawn()
+            } else {
+                Command::new(path).arg(&tmpfile.path()).spawn()
+            };
+
+            if let Ok(mut child) = child {
+                let _ = child.wait();
+
+                Ok(OutputStream::empty())
+            } else {
+                Err(ShellError::untagged_runtime_error(
+                    "Could not create process for sink command",
+                ))
+            }
+        } else {
+            Err(ShellError::untagged_runtime_error(
+                "Could not open file to send sink command message",
+            ))
+        }
+    } else {
+        Err(ShellError::untagged_runtime_error(
+            "Could not create message to sink command",
+        ))
+    }
+}

crates/nu-engine/src/shell/help_shell.rs

@@ -0,0 +1,247 @@
+use crate::command_args::EvaluatedWholeStreamCommandArgs;
+use crate::evaluate::scope::Scope;
+use crate::maybe_text_codec::StringOrBinary;
+use crate::shell::shell_args::{CdArgs, CopyArgs, LsArgs, MkdirArgs, MvArgs, RemoveArgs};
+use crate::shell::Shell;
+use crate::whole_stream_command::Command;
+use encoding_rs::Encoding;
+use futures::stream::BoxStream;
+use nu_data::command::signature_dict;
+use nu_errors::ShellError;
+use nu_protocol::{
+    Primitive, ReturnSuccess, ShellTypeName, TaggedDictBuilder, UntaggedValue, Value,
+};
+use nu_source::Tagged;
+use nu_source::{Span, SpannedItem, Tag};
+use nu_stream::OutputStream;
+use nu_value_ext::ValueExt;
+use std::collections::VecDeque;
+use std::ffi::OsStr;
+use std::path::PathBuf;
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
+
+pub fn command_dict(command: Command, tag: impl Into<Tag>) -> Value {
+    let tag = tag.into();
+
+    let mut cmd_dict = TaggedDictBuilder::new(&tag);
+
+    cmd_dict.insert_untagged("name", UntaggedValue::string(command.name()));
+
+    cmd_dict.insert_untagged("type", UntaggedValue::string("Command"));
+
+    cmd_dict.insert_value("signature", signature_dict(command.signature(), tag));
+    cmd_dict.insert_untagged("usage", UntaggedValue::string(command.usage()));
+
+    cmd_dict.into_value()
+}
+
+#[derive(Clone, Debug)]
+pub struct HelpShell {
+    pub(crate) path: String,
+    pub(crate) value: Value,
+}
+
+impl HelpShell {
+    pub fn index(scope: &Scope) -> Result<HelpShell, ShellError> {
+        let mut cmds = TaggedDictBuilder::new(Tag::unknown());
+        let mut specs = Vec::new();
+
+        for cmd in scope.get_command_names() {
+            if let Some(cmd_value) = scope.get_command(&cmd) {
+                let mut spec = TaggedDictBuilder::new(Tag::unknown());
+                let value = command_dict(cmd_value, Tag::unknown());
+
+                spec.insert_untagged("name", cmd);
+                spec.insert_untagged(
+                    "description",
+                    value
+                        .get_data_by_key("usage".spanned_unknown())
+                        .ok_or_else(|| {
+                            ShellError::untagged_runtime_error(
+                                "Internal error: expected to find usage",
+                            )
+                        })?
+                        .as_string()?,
+                );
+                spec.insert_value("details", value);
+
+                specs.push(spec.into_value());
+            } else {
+            }
+        }
+
+        cmds.insert_untagged("help", UntaggedValue::Table(specs));
+
+        Ok(HelpShell {
+            path: "/help".to_string(),
+            value: cmds.into_value(),
+        })
+    }
+
+    pub fn for_command(cmd: Value, scope: &Scope) -> Result<HelpShell, ShellError> {
+        let mut sh = HelpShell::index(scope)?;
+
+        if let Value {
+            value: UntaggedValue::Primitive(Primitive::String(name)),
+            ..
+        } = cmd
+        {
+            sh.set_path(format!("/help/{:}/details", name));
+        }
+
+        Ok(sh)
+    }
+
+    fn commands(&self) -> VecDeque<Value> {
+        let mut cmds = VecDeque::new();
+        let full_path = PathBuf::from(&self.path);
+
+        let mut viewed = self.value.clone();
+        let sep_string = std::path::MAIN_SEPARATOR.to_string();
+        let sep = OsStr::new(&sep_string);
+
+        for p in full_path.iter() {
+            match p {
+                x if x == sep => {}
+                step => {
+                    let step: &str = &step.to_string_lossy().to_string();
+                    let value = viewed.get_data_by_key(step.spanned_unknown());
+                    if let Some(v) = value {
+                        viewed = v.clone();
+                    }
+                }
+            }
+        }
+        match viewed {
+            Value {
+                value: UntaggedValue::Table(l),
+                ..
+            } => {
+                for item in l {
+                    cmds.push_back(item.clone());
+                }
+            }
+            x => {
+                cmds.push_back(x);
+            }
+        }
+
+        cmds
+    }
+}
+
+impl Shell for HelpShell {
+    fn name(&self) -> String {
+        let anchor_name = self.value.anchor_name();
+
+        match anchor_name {
+            Some(x) => format!("{{{}}}", x),
+            None => format!("<{}>", self.value.type_name()),
+        }
+    }
+
+    fn homedir(&self) -> Option<PathBuf> {
+        #[cfg(feature = "dirs")]
+        {
+            dirs::home_dir()
+        }
+
+        #[cfg(not(feature = "dirs"))]
+        {
+            None
+        }
+    }
+
+    fn path(&self) -> String {
+        self.path.clone()
+    }
+
+    fn pwd(&self, _: EvaluatedWholeStreamCommandArgs) -> Result<OutputStream, ShellError> {
+        Ok(OutputStream::empty())
+    }
+
+    fn set_path(&mut self, path: String) {
+        let _ = std::env::set_current_dir(&path);
+        self.path = path;
+    }
+
+    fn ls(
+        &self,
+        _args: LsArgs,
+        _name: Tag,
+        _ctrl_c: Arc<AtomicBool>,
+    ) -> Result<OutputStream, ShellError> {
+        let output = self
+            .commands()
+            .into_iter()
+            .map(ReturnSuccess::value)
+            .collect::<VecDeque<_>>();
+        Ok(output.into())
+    }
+
+    fn cd(&self, args: CdArgs, _name: Tag) -> Result<OutputStream, ShellError> {
+        let path = match args.path {
+            None => "/".to_string(),
+            Some(v) => {
+                let Tagged { item: target, .. } = v;
+                let mut cwd = PathBuf::from(&self.path);
+
+                if target == PathBuf::from("..") {
+                    cwd.pop();
+                } else {
+                    match target.to_str() {
+                        Some(target) => match target.chars().next() {
+                            Some(x) if x == '/' => cwd = PathBuf::from(target),
+                            _ => cwd.push(target),
+                        },
+                        None => cwd.push(target),
+                    }
+                }
+                cwd.to_string_lossy().to_string()
+            }
+        };
+
+        let mut stream = VecDeque::new();
+        stream.push_back(ReturnSuccess::change_cwd(path));
+        Ok(stream.into())
+    }
+
+    fn cp(&self, _args: CopyArgs, _name: Tag, _path: &str) -> Result<OutputStream, ShellError> {
+        Ok(OutputStream::empty())
+    }
+
+    fn mv(&self, _args: MvArgs, _name: Tag, _path: &str) -> Result<OutputStream, ShellError> {
+        Ok(OutputStream::empty())
+    }
+
+    fn mkdir(&self, _args: MkdirArgs, _name: Tag, _path: &str) -> Result<OutputStream, ShellError> {
+        Ok(OutputStream::empty())
+    }
+
+    fn rm(&self, _args: RemoveArgs, _name: Tag, _path: &str) -> Result<OutputStream, ShellError> {
+        Ok(OutputStream::empty())
+    }
+
+    fn open(
+        &self,
+        _path: &PathBuf,
+        _name: Span,
+        _with_encoding: Option<&'static Encoding>,
+    ) -> Result<BoxStream<'static, Result<StringOrBinary, ShellError>>, ShellError> {
+        Err(ShellError::unimplemented(
+            "open on help shell is not supported",
+        ))
+    }
+
+    fn save(
+        &mut self,
+        _path: &PathBuf,
+        _contents: &[u8],
+        _name: Span,
+    ) -> Result<OutputStream, ShellError> {
+        Err(ShellError::unimplemented(
+            "save on help shell is not supported",
+        ))
+    }
+}

crates/nu-engine/src/shell/mod.rs

@@ -0,0 +1,51 @@
+use nu_stream::OutputStream;
+
+use crate::command_args::EvaluatedWholeStreamCommandArgs;
+use crate::maybe_text_codec::StringOrBinary;
+pub use crate::shell::shell_args::{CdArgs, CopyArgs, LsArgs, MkdirArgs, MvArgs, RemoveArgs};
+use encoding_rs::Encoding;
+use futures::stream::BoxStream;
+use nu_errors::ShellError;
+use nu_source::{Span, Tag};
+use std::path::PathBuf;
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
+
+pub(crate) mod help_shell;
+pub(crate) mod painter;
+pub(crate) mod palette;
+pub(crate) mod shell_args;
+pub(crate) mod shell_manager;
+pub(crate) mod value_shell;
+
+pub trait Shell: std::fmt::Debug {
+    fn name(&self) -> String;
+    fn homedir(&self) -> Option<PathBuf>;
+
+    fn ls(
+        &self,
+        args: LsArgs,
+        name: Tag,
+        ctrl_c: Arc<AtomicBool>,
+    ) -> Result<OutputStream, ShellError>;
+    fn cd(&self, args: CdArgs, name: Tag) -> Result<OutputStream, ShellError>;
+    fn cp(&self, args: CopyArgs, name: Tag, path: &str) -> Result<OutputStream, ShellError>;
+    fn mkdir(&self, args: MkdirArgs, name: Tag, path: &str) -> Result<OutputStream, ShellError>;
+    fn mv(&self, args: MvArgs, name: Tag, path: &str) -> Result<OutputStream, ShellError>;
+    fn rm(&self, args: RemoveArgs, name: Tag, path: &str) -> Result<OutputStream, ShellError>;
+    fn path(&self) -> String;
+    fn pwd(&self, args: EvaluatedWholeStreamCommandArgs) -> Result<OutputStream, ShellError>;
+    fn set_path(&mut self, path: String);
+    fn open(
+        &self,
+        path: &PathBuf,
+        name: Span,
+        with_encoding: Option<&'static Encoding>,
+    ) -> Result<BoxStream<'static, Result<StringOrBinary, ShellError>>, ShellError>;
+    fn save(
+        &mut self,
+        path: &PathBuf,
+        contents: &[u8],
+        name: Span,
+    ) -> Result<OutputStream, ShellError>;
+}

crates/nu-engine/src/shell/painter.rs

@@ -0,0 +1,85 @@
+use crate::evaluate::scope::Scope;
+use crate::shell::palette::Palette;
+use ansi_term::{Color, Style};
+use nu_parser::ParserScope;
+use nu_protocol::hir::FlatShape;
+use nu_source::Spanned;
+use std::borrow::Cow;
+
+// FIXME: find a good home, as nu-engine may be too core for styling
+pub struct Painter {
+    original: Vec<u8>,
+    styles: Vec<Style>,
+}
+
+impl Painter {
+    fn new(original: &str) -> Painter {
+        let bytes: Vec<u8> = original.bytes().collect();
+        let bytes_count = bytes.len();
+        Painter {
+            original: bytes,
+            styles: vec![Color::White.normal(); bytes_count],
+        }
+    }
+
+    pub fn paint_string<'l, P: Palette>(line: &'l str, scope: &Scope, palette: &P) -> Cow<'l, str> {
+        scope.enter_scope();
+        let (block, _) = nu_parser::parse(line, 0, scope);
+        scope.exit_scope();
+
+        let shapes = nu_parser::shapes(&block);
+        let mut painter = Painter::new(line);
+
+        for shape in shapes {
+            painter.paint_shape(&shape, palette);
+        }
+
+        Cow::Owned(painter.into_string())
+    }
+
+    fn paint_shape<P: Palette>(&mut self, shape: &Spanned<FlatShape>, palette: &P) {
+        palette
+            .styles_for_shape(shape)
+            .iter()
+            .for_each(|x| self.paint(x));
+    }
+
+    fn paint(&mut self, styled_span: &Spanned<Style>) {
+        for pos in styled_span.span.start()..styled_span.span.end() {
+            self.styles[pos] = styled_span.item;
+        }
+    }
+
+    fn into_string(self) -> String {
+        let mut idx_start = 0;
+        let mut idx_end = 1;
+
+        if self.original.is_empty() {
+            String::new()
+        } else {
+            let mut builder = String::new();
+
+            let mut current_style = self.styles[0];
+
+            while idx_end < self.styles.len() {
+                if self.styles[idx_end] != current_style {
+                    // Emit, as we changed styles
+                    let intermediate = String::from_utf8_lossy(&self.original[idx_start..idx_end]);
+
+                    builder.push_str(&format!("{}", current_style.paint(intermediate)));
+
+                    current_style = self.styles[idx_end];
+                    idx_start = idx_end;
+                    idx_end += 1;
+                } else {
+                    idx_end += 1;
+                }
+            }
+
+            let intermediate = String::from_utf8_lossy(&self.original[idx_start..idx_end]);
+            builder.push_str(&format!("{}", current_style.paint(intermediate)));
+
+            builder
+        }
+    }
+}

crates/nu-engine/src/shell/palette.rs

@@ -0,0 +1,337 @@
+use ansi_term::{Color, Style};
+use nu_protocol::hir::FlatShape;
+use nu_source::{Span, Spanned};
+use serde::{self, Deserialize, Deserializer, Serialize, Serializer};
+use std::error::Error;
+use std::ops::Deref;
+use std::str::Bytes;
+use std::{fmt, io};
+
+// FIXME: find a good home, as nu-engine may be too core for styling
+pub trait Palette {
+    fn styles_for_shape(&self, shape: &Spanned<FlatShape>) -> Vec<Spanned<Style>>;
+}
+
+pub struct DefaultPalette {}
+
+impl Palette for DefaultPalette {
+    fn styles_for_shape(&self, shape: &Spanned<FlatShape>) -> Vec<Spanned<Style>> {
+        match &shape.item {
+            FlatShape::OpenDelimiter(_) => single_style_span(Color::White.normal(), shape.span),
+            FlatShape::CloseDelimiter(_) => single_style_span(Color::White.normal(), shape.span),
+            FlatShape::ItVariable | FlatShape::Keyword => {
+                single_style_span(Color::Purple.bold(), shape.span)
+            }
+            FlatShape::Variable | FlatShape::Identifier => {
+                single_style_span(Color::Purple.normal(), shape.span)
+            }
+            FlatShape::Type => single_style_span(Color::Blue.bold(), shape.span),
+            FlatShape::Operator => single_style_span(Color::Yellow.normal(), shape.span),
+            FlatShape::DotDotLeftAngleBracket => {
+                single_style_span(Color::Yellow.bold(), shape.span)
+            }
+            FlatShape::DotDot => single_style_span(Color::Yellow.bold(), shape.span),
+            FlatShape::Dot => single_style_span(Style::new().fg(Color::White), shape.span),
+            FlatShape::InternalCommand => single_style_span(Color::Cyan.bold(), shape.span),
+            FlatShape::ExternalCommand => single_style_span(Color::Cyan.normal(), shape.span),
+            FlatShape::ExternalWord => single_style_span(Color::Green.bold(), shape.span),
+            FlatShape::BareMember => single_style_span(Color::Yellow.bold(), shape.span),
+            FlatShape::StringMember => single_style_span(Color::Yellow.bold(), shape.span),
+            FlatShape::String => single_style_span(Color::Green.normal(), shape.span),
+            FlatShape::Path => single_style_span(Color::Cyan.normal(), shape.span),
+            FlatShape::GlobPattern => single_style_span(Color::Cyan.bold(), shape.span),
+            FlatShape::Word => single_style_span(Color::Green.normal(), shape.span),
+            FlatShape::Pipe => single_style_span(Color::Purple.bold(), shape.span),
+            FlatShape::Flag => single_style_span(Color::Blue.bold(), shape.span),
+            FlatShape::ShorthandFlag => single_style_span(Color::Blue.bold(), shape.span),
+            FlatShape::Int => single_style_span(Color::Purple.bold(), shape.span),
+            FlatShape::Decimal => single_style_span(Color::Purple.bold(), shape.span),
+            FlatShape::Whitespace | FlatShape::Separator => {
+                single_style_span(Color::White.normal(), shape.span)
+            }
+            FlatShape::Comment => single_style_span(Color::Green.bold(), shape.span),
+            FlatShape::Garbage => {
+                single_style_span(Style::new().fg(Color::White).on(Color::Red), shape.span)
+            }
+            FlatShape::Size { number, unit } => vec![
+                Spanned::<Style> {
+                    span: *number,
+                    item: Color::Purple.bold(),
+                },
+                Spanned::<Style> {
+                    span: *unit,
+                    item: Color::Cyan.bold(),
+                },
+            ],
+        }
+    }
+}
+
+pub struct ThemedPalette {
+    theme: Theme,
+}
+
+impl ThemedPalette {
+    // remove this once we start actually loading themes.
+    #[allow(dead_code)]
+    pub fn new<R: io::Read>(reader: &mut R) -> Result<ThemedPalette, ThemeError> {
+        let theme = serde_json::from_reader(reader)?;
+        Ok(ThemedPalette { theme })
+    }
+}
+
+impl Palette for ThemedPalette {
+    fn styles_for_shape(&self, shape: &Spanned<FlatShape>) -> Vec<Spanned<Style>> {
+        match &shape.item {
+            FlatShape::OpenDelimiter(_) => {
+                single_style_span(self.theme.open_delimiter.normal(), shape.span)
+            }
+            FlatShape::CloseDelimiter(_) => {
+                single_style_span(self.theme.close_delimiter.normal(), shape.span)
+            }
+            FlatShape::ItVariable => single_style_span(self.theme.it_variable.bold(), shape.span),
+            FlatShape::Keyword => single_style_span(self.theme.keyword.bold(), shape.span),
+            FlatShape::Variable => single_style_span(self.theme.variable.normal(), shape.span),
+            FlatShape::Identifier => single_style_span(self.theme.identifier.normal(), shape.span),
+            FlatShape::Type => single_style_span(self.theme.r#type.bold(), shape.span),
+            FlatShape::Operator => single_style_span(self.theme.operator.normal(), shape.span),
+            FlatShape::DotDotLeftAngleBracket => {
+                single_style_span(self.theme.dot_dot.bold(), shape.span)
+            }
+            FlatShape::DotDot => single_style_span(self.theme.dot_dot.bold(), shape.span),
+            FlatShape::Dot => single_style_span(Style::new().fg(*self.theme.dot), shape.span),
+            FlatShape::InternalCommand => {
+                single_style_span(self.theme.internal_command.bold(), shape.span)
+            }
+            FlatShape::ExternalCommand => {
+                single_style_span(self.theme.external_command.normal(), shape.span)
+            }
+            FlatShape::ExternalWord => {
+                single_style_span(self.theme.external_word.bold(), shape.span)
+            }
+            FlatShape::BareMember => single_style_span(self.theme.bare_member.bold(), shape.span),
+            FlatShape::StringMember => {
+                single_style_span(self.theme.string_member.bold(), shape.span)
+            }
+            FlatShape::String => single_style_span(self.theme.string.normal(), shape.span),
+            FlatShape::Path => single_style_span(self.theme.path.normal(), shape.span),
+            FlatShape::GlobPattern => single_style_span(self.theme.glob_pattern.bold(), shape.span),
+            FlatShape::Word => single_style_span(self.theme.word.normal(), shape.span),
+            FlatShape::Pipe => single_style_span(self.theme.pipe.bold(), shape.span),
+            FlatShape::Flag => single_style_span(self.theme.flag.bold(), shape.span),
+            FlatShape::ShorthandFlag => {
+                single_style_span(self.theme.shorthand_flag.bold(), shape.span)
+            }
+            FlatShape::Int => single_style_span(self.theme.int.bold(), shape.span),
+            FlatShape::Decimal => single_style_span(self.theme.decimal.bold(), shape.span),
+            FlatShape::Whitespace => single_style_span(self.theme.whitespace.normal(), shape.span),
+            FlatShape::Separator => single_style_span(self.theme.separator.normal(), shape.span),
+            FlatShape::Comment => single_style_span(self.theme.comment.bold(), shape.span),
+            FlatShape::Garbage => single_style_span(
+                Style::new().fg(*self.theme.garbage).on(Color::Red),
+                shape.span,
+            ),
+            FlatShape::Size { number, unit } => vec![
+                Spanned::<Style> {
+                    span: *number,
+                    item: self.theme.size_number.bold(),
+                },
+                Spanned::<Style> {
+                    span: *unit,
+                    item: self.theme.size_unit.bold(),
+                },
+            ],
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ThemeError {
+    serde_err: serde_json::error::Error,
+}
+
+impl fmt::Display for ThemeError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "failure to load theme")
+    }
+}
+
+impl Error for ThemeError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        Some(&self.serde_err)
+    }
+}
+
+impl From<serde_json::error::Error> for ThemeError {
+    fn from(serde_err: serde_json::error::Error) -> Self {
+        ThemeError { serde_err }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct Theme {
+    open_delimiter: ThemeColor,
+    close_delimiter: ThemeColor,
+    r#type: ThemeColor,
+    identifier: ThemeColor,
+    it_variable: ThemeColor,
+    variable: ThemeColor,
+    operator: ThemeColor,
+    dot: ThemeColor,
+    dot_dot: ThemeColor,
+    internal_command: ThemeColor,
+    external_command: ThemeColor,
+    external_word: ThemeColor,
+    bare_member: ThemeColor,
+    string_member: ThemeColor,
+    string: ThemeColor,
+    path: ThemeColor,
+    word: ThemeColor,
+    keyword: ThemeColor,
+    pipe: ThemeColor,
+    glob_pattern: ThemeColor,
+    flag: ThemeColor,
+    shorthand_flag: ThemeColor,
+    int: ThemeColor,
+    decimal: ThemeColor,
+    garbage: ThemeColor,
+    whitespace: ThemeColor,
+    separator: ThemeColor,
+    comment: ThemeColor,
+    size_number: ThemeColor,
+    size_unit: ThemeColor,
+}
+
+#[derive(Debug)]
+struct ThemeColor(Color);
+
+impl Deref for ThemeColor {
+    type Target = Color;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl Serialize for ThemeColor {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str("TODO: IMPLEMENT SERIALIZATION")
+    }
+}
+
+impl<'de> Deserialize<'de> for ThemeColor {
+    fn deserialize<D>(deserializer: D) -> Result<ThemeColor, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        ThemeColor::from_str(&s)
+    }
+}
+
+impl ThemeColor {
+    fn from_str<E>(s: &str) -> Result<ThemeColor, E>
+    where
+        E: serde::de::Error,
+    {
+        let mut bytes = s.bytes();
+        let r = ThemeColor::xtoi(&mut bytes)?;
+        let g = ThemeColor::xtoi(&mut bytes)?;
+        let b = ThemeColor::xtoi(&mut bytes)?;
+        Ok(ThemeColor(Color::RGB(r, g, b)))
+    }
+
+    fn xtoi<E>(b: &mut Bytes) -> Result<u8, E>
+    where
+        E: serde::de::Error,
+    {
+        let upper = b
+            .next()
+            .ok_or_else(|| E::custom("color string too short"))?;
+        let lower = b
+            .next()
+            .ok_or_else(|| E::custom("color string too short"))?;
+        let mut val = ThemeColor::numerical_value(upper)?;
+        val = (val << 4) | ThemeColor::numerical_value(lower)?;
+        Ok(val)
+    }
+
+    fn numerical_value<E>(character: u8) -> Result<u8, E>
+    where
+        E: serde::de::Error,
+    {
+        match character {
+            b'0'..=b'9' => Ok(character - b'0'),
+            b'a'..=b'z' => Ok(character - (b'a' - 10)),
+            _ => Err(E::custom(format!("invalid character {}", character))),
+        }
+    }
+}
+
+fn single_style_span(style: Style, span: Span) -> Vec<Spanned<Style>> {
+    vec![Spanned::<Style> { span, item: style }]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{Palette, ThemedPalette};
+    use ansi_term::Color;
+    use nu_protocol::hir::FlatShape;
+    use nu_source::{Span, Spanned};
+    use std::io::Cursor;
+
+    #[test]
+    fn create_themed_palette() {
+        let json = r#"
+{
+    "open_delimiter": "a359cc",
+    "close_delimiter": "a359cc",
+    "type": "a359cc",
+    "identifier": "a359cc",
+    "it_variable": "a359cc",
+    "variable": "a359cc",
+    "operator": "a359cc",
+    "dot": "a359cc",
+    "dot_dot": "a359cc",
+    "internal_command": "a359cc",
+    "external_command": "a359cc",
+    "external_word": "a359cc",
+    "bare_member": "a359cc",
+    "string_member": "a359cc",
+    "string": "a359cc",
+    "path": "a359cc",
+    "word": "a359cc",
+    "keyword": "a359cc",
+    "pipe": "a359cc",
+    "glob_pattern": "a359cc",
+    "flag": "a359cc",
+    "shorthand_flag": "a359cc",
+    "int": "a359cc",
+    "decimal": "a359cc",
+    "garbage": "a359cc",
+    "whitespace": "a359cc",
+    "separator": "a359cc",
+    "comment": "a359cc",
+    "size_number": "a359cc",
+    "size_unit": "a359cc"
+}"#;
+        let mut json_reader = Cursor::new(json);
+        let themed_palette = ThemedPalette::new(&mut json_reader).unwrap();
+        let test_shape = Spanned {
+            item: FlatShape::Type,
+            span: Span::new(4, 9),
+        };
+        let styled = themed_palette.styles_for_shape(&test_shape);
+        assert_eq!(styled.len(), 1);
+        assert_eq!(
+            styled[0],
+            Spanned {
+                item: Color::RGB(163, 89, 204).bold(),
+                span: Span::new(4, 9),
+            },
+        );
+    }
+}

crates/nu-engine/src/shell/shell_args.rs

@@ -0,0 +1,50 @@
+use nu_source::Tagged;
+use serde::{self, Deserialize};
+use std::path::PathBuf;
+
+#[derive(Deserialize)]
+pub struct CdArgs {
+    pub path: Option<Tagged<PathBuf>>,
+}
+
+#[derive(Deserialize)]
+pub struct CopyArgs {
+    pub src: Tagged<PathBuf>,
+    pub dst: Tagged<PathBuf>,
+    pub recursive: Tagged<bool>,
+}
+
+#[derive(Deserialize)]
+pub struct LsArgs {
+    pub path: Option<Tagged<PathBuf>>,
+    pub all: bool,
+    pub long: bool,
+    #[serde(rename = "short-names")]
+    pub short_names: bool,
+    #[serde(rename = "du")]
+    pub du: bool,
+}
+
+#[derive(Deserialize)]
+pub struct MvArgs {
+    pub src: Tagged<PathBuf>,
+    pub dst: Tagged<PathBuf>,
+}
+
+#[derive(Deserialize)]
+pub struct MkdirArgs {
+    pub rest: Vec<Tagged<PathBuf>>,
+    #[serde(rename = "show-created-paths")]
+    pub show_created_paths: bool,
+}
+
+#[derive(Deserialize)]
+pub struct RemoveArgs {
+    pub rest: Vec<Tagged<PathBuf>>,
+    pub recursive: Tagged<bool>,
+    #[allow(unused)]
+    pub trash: Tagged<bool>,
+    #[allow(unused)]
+    pub permanent: Tagged<bool>,
+    pub force: Tagged<bool>,
+}

crates/nu-engine/src/shell/shell_manager.rs

@@ -0,0 +1,167 @@
+use crate::command_args::EvaluatedWholeStreamCommandArgs;
+use crate::maybe_text_codec::StringOrBinary;
+use crate::shell::Shell;
+use futures::Stream;
+use nu_stream::OutputStream;
+
+use crate::shell::shell_args::{CdArgs, CopyArgs, LsArgs, MkdirArgs, MvArgs, RemoveArgs};
+use encoding_rs::Encoding;
+use nu_errors::ShellError;
+use nu_source::{Span, Tag};
+use parking_lot::Mutex;
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::Arc;
+
+#[derive(Clone, Debug)]
+pub struct ShellManager {
+    pub current_shell: Arc<AtomicUsize>,
+    pub shells: Arc<Mutex<Vec<Box<dyn Shell + Send>>>>,
+}
+
+impl ShellManager {
+    pub fn insert_at_current(&self, shell: Box<dyn Shell + Send>) {
+        self.shells.lock().push(shell);
+        self.current_shell
+            .store(self.shells.lock().len() - 1, Ordering::SeqCst);
+        self.set_path(self.path());
+    }
+
+    pub fn current_shell(&self) -> usize {
+        self.current_shell.load(Ordering::SeqCst)
+    }
+
+    pub fn remove_at_current(&self) {
+        {
+            let mut shells = self.shells.lock();
+            if shells.len() > 0 {
+                if self.current_shell() == shells.len() - 1 {
+                    shells.pop();
+                    let new_len = shells.len();
+                    if new_len > 0 {
+                        self.current_shell.store(new_len - 1, Ordering::SeqCst);
+                    } else {
+                        return;
+                    }
+                } else {
+                    shells.remove(self.current_shell());
+                }
+            }
+        }
+        self.set_path(self.path())
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.shells.lock().is_empty()
+    }
+
+    pub fn path(&self) -> String {
+        self.shells.lock()[self.current_shell()].path()
+    }
+
+    pub fn pwd(&self, args: EvaluatedWholeStreamCommandArgs) -> Result<OutputStream, ShellError> {
+        let env = self.shells.lock();
+
+        env[self.current_shell()].pwd(args)
+    }
+
+    pub fn set_path(&self, path: String) {
+        self.shells.lock()[self.current_shell()].set_path(path)
+    }
+
+    pub fn open(
+        &self,
+        full_path: &PathBuf,
+        name: Span,
+        with_encoding: Option<&'static Encoding>,
+    ) -> Result<impl Stream<Item = Result<StringOrBinary, ShellError>> + Send + 'static, ShellError>
+    {
+        self.shells.lock()[self.current_shell()].open(full_path, name, with_encoding)
+    }
+
+    pub fn save(
+        &self,
+        full_path: &PathBuf,
+        save_data: &[u8],
+        name: Span,
+    ) -> Result<OutputStream, ShellError> {
+        self.shells.lock()[self.current_shell()].save(full_path, save_data, name)
+    }
+
+    pub fn next(&self) {
+        {
+            let shell_len = self.shells.lock().len();
+            if self.current_shell() == (shell_len - 1) {
+                self.current_shell.store(0, Ordering::SeqCst);
+            } else {
+                self.current_shell
+                    .store(self.current_shell() + 1, Ordering::SeqCst);
+            }
+        }
+        self.set_path(self.path())
+    }
+
+    pub fn prev(&self) {
+        {
+            let shell_len = self.shells.lock().len();
+            if self.current_shell() == 0 {
+                self.current_shell.store(shell_len - 1, Ordering::SeqCst);
+            } else {
+                self.current_shell
+                    .store(self.current_shell() - 1, Ordering::SeqCst);
+            }
+        }
+        self.set_path(self.path())
+    }
+
+    pub fn homedir(&self) -> Option<PathBuf> {
+        let env = self.shells.lock();
+
+        env[self.current_shell()].homedir()
+    }
+
+    pub fn ls(
+        &self,
+        args: LsArgs,
+        name: Tag,
+        ctrl_c: Arc<AtomicBool>,
+    ) -> Result<OutputStream, ShellError> {
+        let env = self.shells.lock();
+
+        env[self.current_shell()].ls(args, name, ctrl_c)
+    }
+
+    pub fn cd(&self, args: CdArgs, name: Tag) -> Result<OutputStream, ShellError> {
+        let env = self.shells.lock();
+
+        env[self.current_shell()].cd(args, name)
+    }
+
+    pub fn cp(&self, args: CopyArgs, name: Tag) -> Result<OutputStream, ShellError> {
+        let shells = self.shells.lock();
+
+        let path = shells[self.current_shell()].path();
+        shells[self.current_shell()].cp(args, name, &path)
+    }
+
+    pub fn rm(&self, args: RemoveArgs, name: Tag) -> Result<OutputStream, ShellError> {
+        let shells = self.shells.lock();
+
+        let path = shells[self.current_shell()].path();
+        shells[self.current_shell()].rm(args, name, &path)
+    }
+
+    pub fn mkdir(&self, args: MkdirArgs, name: Tag) -> Result<OutputStream, ShellError> {
+        let shells = self.shells.lock();
+
+        let path = shells[self.current_shell()].path();
+        shells[self.current_shell()].mkdir(args, name, &path)
+    }
+
+    pub fn mv(&self, args: MvArgs, name: Tag) -> Result<OutputStream, ShellError> {
+        let shells = self.shells.lock();
+
+        let path = shells[self.current_shell()].path();
+        shells[self.current_shell()].mv(args, name, &path)
+    }
+}

crates/nu-engine/src/shell/value_shell.rs

@@ -0,0 +1,257 @@
+use crate::command_args::EvaluatedWholeStreamCommandArgs;
+use crate::maybe_text_codec::StringOrBinary;
+use crate::shell::shell_args::{CdArgs, CopyArgs, LsArgs, MkdirArgs, MvArgs, RemoveArgs};
+use crate::shell::Shell;
+use encoding_rs::Encoding;
+use futures::stream::BoxStream;
+use nu_errors::ShellError;
+use nu_protocol::ValueStructure;
+use nu_protocol::{ReturnSuccess, ShellTypeName, UntaggedValue, Value};
+use nu_source::SpannedItem;
+use nu_source::{Span, Tag, Tagged};
+use nu_stream::OutputStream;
+use nu_value_ext::ValueExt;
+use std::collections::VecDeque;
+use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub struct ValueShell {
+    pub(crate) path: String,
+    pub(crate) last_path: String,
+    pub(crate) value: Value,
+}
+
+impl std::fmt::Debug for ValueShell {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "ValueShell @ {}", self.path)
+    }
+}
+
+impl ValueShell {
+    pub fn new(value: Value) -> ValueShell {
+        ValueShell {
+            path: "/".to_string(),
+            last_path: "/".to_string(),
+            value,
+        }
+    }
+
+    fn members_under(&self, path: &Path) -> VecDeque<Value> {
+        let mut shell_entries = VecDeque::new();
+        let full_path = path.to_path_buf();
+        let mut viewed = self.value.clone();
+        let sep_string = std::path::MAIN_SEPARATOR.to_string();
+        let sep = OsStr::new(&sep_string);
+        for p in full_path.iter() {
+            match p {
+                x if x == sep => {}
+                step => {
+                    let name: &str = &step.to_string_lossy().to_string();
+                    let value = viewed.get_data_by_key(name.spanned_unknown());
+                    if let Some(v) = value {
+                        viewed = v.clone();
+                    }
+                }
+            }
+        }
+        match viewed {
+            Value {
+                value: UntaggedValue::Table(l),
+                ..
+            } => {
+                for item in l {
+                    shell_entries.push_back(item.clone());
+                }
+            }
+            x => {
+                shell_entries.push_back(x);
+            }
+        }
+
+        shell_entries
+    }
+
+    // TODO make use of this in the new completion engine
+    #[allow(dead_code)]
+    fn members(&self) -> VecDeque<Value> {
+        self.members_under(Path::new("."))
+    }
+}
+
+impl Shell for ValueShell {
+    fn name(&self) -> String {
+        let anchor_name = self.value.anchor_name();
+
+        match anchor_name {
+            Some(x) => format!("{{{}}}", x),
+            None => format!("<{}>", self.value.type_name()),
+        }
+    }
+
+    fn homedir(&self) -> Option<PathBuf> {
+        Some(PathBuf::from("/"))
+    }
+
+    fn ls(
+        &self,
+        LsArgs { path, .. }: LsArgs,
+        name_tag: Tag,
+        _ctrl_c: Arc<AtomicBool>,
+    ) -> Result<OutputStream, ShellError> {
+        let mut full_path = PathBuf::from(self.path());
+
+        if let Some(value) = &path {
+            full_path.push(value.as_ref());
+        }
+
+        let mut value_system = ValueStructure::new();
+        value_system.walk_decorate(&self.value)?;
+
+        if !value_system.exists(&full_path) {
+            if let Some(target) = &path {
+                return Err(ShellError::labeled_error(
+                    "Can not list entries inside",
+                    "No such path exists",
+                    target.tag(),
+                ));
+            }
+
+            return Err(ShellError::labeled_error(
+                "Can not list entries inside",
+                "No such path exists",
+                name_tag,
+            ));
+        }
+
+        let output = self
+            .members_under(full_path.as_path())
+            .into_iter()
+            .map(ReturnSuccess::value)
+            .collect::<VecDeque<_>>();
+        Ok(output.into())
+    }
+
+    fn cd(&self, args: CdArgs, name: Tag) -> Result<OutputStream, ShellError> {
+        let destination = args.path;
+
+        let path = match destination {
+            None => "/".to_string(),
+            Some(ref v) => {
+                let Tagged { item: target, .. } = v;
+                let mut cwd = PathBuf::from(&self.path);
+
+                if target == &PathBuf::from("..") {
+                    cwd.pop();
+                } else if target == &PathBuf::from("-") {
+                    cwd = PathBuf::from(&self.last_path);
+                } else {
+                    match target.to_str() {
+                        Some(target) => match target.chars().next() {
+                            Some(x) if x == '/' => cwd = PathBuf::from(target),
+                            _ => cwd.push(target),
+                        },
+                        None => cwd.push(target),
+                    }
+                }
+                cwd.to_string_lossy().to_string()
+            }
+        };
+
+        let mut value_system = ValueStructure::new();
+        value_system.walk_decorate(&self.value)?;
+
+        if !value_system.exists(&PathBuf::from(&path)) {
+            if let Some(destination) = destination {
+                return Err(ShellError::labeled_error(
+                    "Can not change to path inside",
+                    "No such path exists",
+                    destination.tag(),
+                ));
+            }
+
+            return Err(ShellError::labeled_error(
+                "Can not change to path inside",
+                "No such path exists",
+                &name,
+            ));
+        }
+
+        let mut stream = VecDeque::new();
+        stream.push_back(ReturnSuccess::change_cwd(path));
+        Ok(stream.into())
+    }
+
+    fn cp(&self, _args: CopyArgs, name: Tag, _path: &str) -> Result<OutputStream, ShellError> {
+        Err(ShellError::labeled_error(
+            "cp not currently supported on values",
+            "not currently supported",
+            name,
+        ))
+    }
+
+    fn mv(&self, _args: MvArgs, name: Tag, _path: &str) -> Result<OutputStream, ShellError> {
+        Err(ShellError::labeled_error(
+            "mv not currently supported on values",
+            "not currently supported",
+            name,
+        ))
+    }
+
+    fn mkdir(&self, _args: MkdirArgs, name: Tag, _path: &str) -> Result<OutputStream, ShellError> {
+        Err(ShellError::labeled_error(
+            "mkdir not currently supported on values",
+            "not currently supported",
+            name,
+        ))
+    }
+
+    fn rm(&self, _args: RemoveArgs, name: Tag, _path: &str) -> Result<OutputStream, ShellError> {
+        Err(ShellError::labeled_error(
+            "rm not currently supported on values",
+            "not currently supported",
+            name,
+        ))
+    }
+
+    fn path(&self) -> String {
+        self.path.clone()
+    }
+
+    fn pwd(&self, args: EvaluatedWholeStreamCommandArgs) -> Result<OutputStream, ShellError> {
+        let mut stream = VecDeque::new();
+        stream.push_back(ReturnSuccess::value(
+            UntaggedValue::string(self.path()).into_value(&args.call_info.name_tag),
+        ));
+        Ok(stream.into())
+    }
+
+    fn set_path(&mut self, path: String) {
+        self.last_path = self.path.clone();
+        self.path = path;
+    }
+
+    fn open(
+        &self,
+        _path: &PathBuf,
+        _name: Span,
+        _with_encoding: Option<&'static Encoding>,
+    ) -> Result<BoxStream<'static, Result<StringOrBinary, ShellError>>, ShellError> {
+        Err(ShellError::unimplemented(
+            "open on help shell is not supported",
+        ))
+    }
+
+    fn save(
+        &mut self,
+        _path: &PathBuf,
+        _contents: &[u8],
+        _name: Span,
+    ) -> Result<OutputStream, ShellError> {
+        Err(ShellError::unimplemented(
+            "save on help shell is not supported",
+        ))
+    }
+}

crates/nu-engine/src/whole_stream_command.rs

@@ -0,0 +1,197 @@
+use crate::command_args::CommandArgs;
+use crate::documentation::get_help;
+use crate::evaluate::block::run_block;
+use crate::evaluation_context::EvaluationContext;
+use crate::example::Example;
+use async_trait::async_trait;
+use nu_errors::ShellError;
+use nu_parser::ParserScope;
+use nu_protocol::hir::Block;
+use nu_protocol::{ReturnSuccess, Signature, UntaggedValue};
+use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Tag};
+use nu_stream::{OutputStream, ToOutputStream};
+use std::sync::Arc;
+
+#[async_trait]
+pub trait WholeStreamCommand: Send + Sync {
+    fn name(&self) -> &str;
+
+    fn signature(&self) -> Signature {
+        Signature::new(self.name()).desc(self.usage()).filter()
+    }
+
+    fn usage(&self) -> &str;
+
+    async fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError>;
+
+    fn is_binary(&self) -> bool {
+        false
+    }
+
+    // Commands that are not meant to be run by users
+    fn is_internal(&self) -> bool {
+        false
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        Vec::new()
+    }
+}
+
+// Custom commands are blocks, so we can use the information in the block to also
+// implement a WholeStreamCommand
+#[allow(clippy::suspicious_else_formatting)]
+#[async_trait]
+impl WholeStreamCommand for Block {
+    fn name(&self) -> &str {
+        &self.params.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.params.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.params.usage
+    }
+
+    async fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
+        let call_info = args.call_info.clone();
+
+        let mut block = self.clone();
+        block.set_redirect(call_info.args.external_redirection);
+
+        let ctx = EvaluationContext::from_args(&args);
+        let evaluated = call_info.evaluate(&ctx).await?;
+
+        let input = args.input;
+        ctx.scope.enter_scope();
+        if let Some(args) = evaluated.args.positional {
+            // FIXME: do not do this
+            for arg in args.into_iter().zip(self.params.positional.iter()) {
+                let name = arg.1 .0.name();
+
+                if name.starts_with('$') {
+                    ctx.scope.add_var(name, arg.0);
+                } else {
+                    ctx.scope.add_var(format!("${}", name), arg.0);
+                }
+            }
+        }
+        if let Some(args) = evaluated.args.named {
+            for named in &block.params.named {
+                let name = named.0;
+                if let Some(value) = args.get(name) {
+                    if name.starts_with('$') {
+                        ctx.scope.add_var(name, value.clone());
+                    } else {
+                        ctx.scope.add_var(format!("${}", name), value.clone());
+                    }
+                } else if name.starts_with('$') {
+                    ctx.scope
+                        .add_var(name, UntaggedValue::nothing().into_untagged_value());
+                } else {
+                    ctx.scope.add_var(
+                        format!("${}", name),
+                        UntaggedValue::nothing().into_untagged_value(),
+                    );
+                }
+            }
+        } else {
+            for named in &block.params.named {
+                let name = named.0;
+                if name.starts_with('$') {
+                    ctx.scope
+                        .add_var(name, UntaggedValue::nothing().into_untagged_value());
+                } else {
+                    ctx.scope.add_var(
+                        format!("${}", name),
+                        UntaggedValue::nothing().into_untagged_value(),
+                    );
+                }
+            }
+        }
+        let result = run_block(&block, &ctx, input).await;
+        ctx.scope.exit_scope();
+        result.map(|x| x.to_output_stream())
+    }
+
+    fn is_binary(&self) -> bool {
+        false
+    }
+
+    fn is_internal(&self) -> bool {
+        false
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![]
+    }
+}
+
+#[derive(Clone)]
+pub struct Command(Arc<dyn WholeStreamCommand>);
+
+impl PrettyDebugWithSource for Command {
+    fn pretty_debug(&self, source: &str) -> DebugDocBuilder {
+        b::typed(
+            "whole stream command",
+            b::description(self.name())
+                + b::space()
+                + b::equals()
+                + b::space()
+                + self.signature().pretty_debug(source),
+        )
+    }
+}
+
+impl std::fmt::Debug for Command {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Command({})", self.name())
+    }
+}
+
+impl Command {
+    pub fn name(&self) -> &str {
+        self.0.name()
+    }
+
+    pub fn signature(&self) -> Signature {
+        self.0.signature()
+    }
+
+    pub fn usage(&self) -> &str {
+        self.0.usage()
+    }
+
+    pub fn examples(&self) -> Vec<Example> {
+        self.0.examples()
+    }
+
+    pub async fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
+        if args.call_info.switch_present("help") {
+            let cl = self.0.clone();
+            Ok(OutputStream::one(Ok(ReturnSuccess::Value(
+                UntaggedValue::string(get_help(&*cl, &args.scope)).into_value(Tag::unknown()),
+            ))))
+        } else {
+            self.0.run(args).await
+        }
+    }
+
+    pub fn is_binary(&self) -> bool {
+        self.0.is_binary()
+    }
+
+    pub fn is_internal(&self) -> bool {
+        self.0.is_internal()
+    }
+
+    pub fn stream_command(&self) -> &dyn WholeStreamCommand {
+        &*self.0
+    }
+}
+
+pub fn whole_stream_command(command: impl WholeStreamCommand + 'static) -> Command {
+    Command(Arc::new(command))
+}