engine-q merge

crates/nu-protocol/Cargo.toml

@@ -1,4 +1,5 @@
 [package]
+<<<<<<< HEAD
 authors = ["The Nu Project Contributors"]
 description = "Core values and protocols for Nushell"
 edition = "2018"
@@ -36,3 +37,31 @@ features = ["docs", "zip_with", "csv-file", "temporal", "performant", "pretty_fm
 dataframe = ["polars"]
 
 [build-dependencies]
+=======
+name = "nu-protocol"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+thiserror = "1.0.29"
+miette = "3.0.0"
+serde = {version = "1.0.130", features = ["derive"]}
+chrono = { version="0.4.19", features=["serde"] }
+indexmap = { version="1.7", features=["serde-1"] }
+chrono-humanize = "0.2.1"
+byte-unit = "4.0.9"
+im = "15.0.0"
+serde_json = { version = "1.0", optional = true }
+nu-json = { path = "../nu-json" }
+typetag = "0.1.8"
+num-format = "0.4.0"
+sys-locale = "0.1.0"
+
+[features]
+plugin = ["serde_json"]
+
+[dev-dependencies]
+serde_json = "1.0"
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce

crates/nu-protocol/README.md

@@ -0,0 +1,3 @@
+# nu-protocol
+
+The nu-protocol crate holds the definitions of structs/traits that are used throughout Nushell. This gives us one way to expose them to many other crates, as well as make these definitions available to each other, without causing mutually recursive dependencies.

crates/nu-protocol/src/ast/block.rs

@@ -0,0 +1,71 @@
+use std::ops::{Index, IndexMut};
+
+use crate::{Signature, Span, VarId};
+
+use super::Statement;
+
+#[derive(Debug, Clone)]
+pub struct Block {
+    pub signature: Box<Signature>,
+    pub stmts: Vec<Statement>,
+    pub captures: Vec<VarId>,
+    pub redirect_env: bool,
+    pub span: Option<Span>, // None option encodes no span to avoid using test_span()
+}
+
+impl Block {
+    pub fn len(&self) -> usize {
+        self.stmts.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.stmts.is_empty()
+    }
+}
+
+impl Index<usize> for Block {
+    type Output = Statement;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        &self.stmts[index]
+    }
+}
+
+impl IndexMut<usize> for Block {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        &mut self.stmts[index]
+    }
+}
+
+impl Default for Block {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Block {
+    pub fn new() -> Self {
+        Self {
+            signature: Box::new(Signature::new("")),
+            stmts: vec![],
+            captures: vec![],
+            redirect_env: false,
+            span: None,
+        }
+    }
+}
+
+impl<T> From<T> for Block
+where
+    T: Iterator<Item = Statement>,
+{
+    fn from(stmts: T) -> Self {
+        Self {
+            signature: Box::new(Signature::new("")),
+            stmts: stmts.collect(),
+            captures: vec![],
+            redirect_env: false,
+            span: None,
+        }
+    }
+}

crates/nu-protocol/src/ast/call.rs

@@ -0,0 +1,56 @@
+use super::Expression;
+use crate::{DeclId, Span, Spanned};
+
+#[derive(Debug, Clone)]
+pub struct Call {
+    /// identifier of the declaration to call
+    pub decl_id: DeclId,
+    pub head: Span,
+    pub positional: Vec<Expression>,
+    pub named: Vec<(Spanned<String>, Option<Expression>)>,
+}
+
+impl Call {
+    pub fn new(head: Span) -> Call {
+        Self {
+            decl_id: 0,
+            head,
+            positional: vec![],
+            named: vec![],
+        }
+    }
+
+    pub fn has_flag(&self, flag_name: &str) -> bool {
+        for name in &self.named {
+            if flag_name == name.0.item {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    pub fn get_flag_expr(&self, flag_name: &str) -> Option<Expression> {
+        for name in &self.named {
+            if flag_name == name.0.item {
+                return name.1.clone();
+            }
+        }
+
+        None
+    }
+
+    pub fn get_named_arg(&self, flag_name: &str) -> Option<Spanned<String>> {
+        for name in &self.named {
+            if flag_name == name.0.item {
+                return Some(name.0.clone());
+            }
+        }
+
+        None
+    }
+
+    pub fn nth(&self, pos: usize) -> Option<Expression> {
+        self.positional.get(pos).cloned()
+    }
+}

crates/nu-protocol/src/ast/cell_path.rs

@@ -0,0 +1,48 @@
+use super::Expression;
+use crate::Span;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum PathMember {
+    String { val: String, span: Span },
+    Int { val: usize, span: Span },
+}
+
+impl PartialEq for PathMember {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Self::String { val: l_val, .. }, Self::String { val: r_val, .. }) => l_val == r_val,
+            (Self::Int { val: l_val, .. }, Self::Int { val: r_val, .. }) => l_val == r_val,
+            _ => false,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CellPath {
+    pub members: Vec<PathMember>,
+}
+
+impl CellPath {
+    pub fn into_string(&self) -> String {
+        let mut output = String::new();
+
+        for (idx, elem) in self.members.iter().enumerate() {
+            if idx > 0 {
+                output.push('.');
+            }
+            match elem {
+                PathMember::Int { val, .. } => output.push_str(&format!("{}", val)),
+                PathMember::String { val, .. } => output.push_str(val),
+            }
+        }
+
+        output
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct FullCellPath {
+    pub head: Expression,
+    pub tail: Vec<PathMember>,
+}

crates/nu-protocol/src/ast/expr.rs

@@ -0,0 +1,39 @@
+use super::{Call, CellPath, Expression, FullCellPath, Operator, RangeOperator};
+use crate::{ast::ImportPattern, BlockId, Signature, Span, Spanned, Unit, VarId};
+
+#[derive(Debug, Clone)]
+pub enum Expr {
+    Bool(bool),
+    Int(i64),
+    Float(f64),
+    Range(
+        Option<Box<Expression>>, // from
+        Option<Box<Expression>>, // next value after "from"
+        Option<Box<Expression>>, // to
+        RangeOperator,
+    ),
+    Var(VarId),
+    VarDecl(VarId),
+    Call(Box<Call>),
+    ExternalCall(Box<Expression>, Vec<Expression>),
+    Operator(Operator),
+    RowCondition(BlockId),
+    BinaryOp(Box<Expression>, Box<Expression>, Box<Expression>), //lhs, op, rhs
+    Subexpression(BlockId),
+    Block(BlockId),
+    List(Vec<Expression>),
+    Table(Vec<Expression>, Vec<Vec<Expression>>),
+    Record(Vec<(Expression, Expression)>),
+    Keyword(Vec<u8>, Span, Box<Expression>),
+    ValueWithUnit(Box<Expression>, Spanned<Unit>),
+    Filepath(String),
+    GlobPattern(String),
+    String(String),
+    CellPath(CellPath),
+    FullCellPath(Box<FullCellPath>),
+    ImportPattern(ImportPattern),
+    Signature(Box<Signature>),
+    StringInterpolation(Vec<Expression>),
+    Nothing,
+    Garbage,
+}

crates/nu-protocol/src/ast/expression.rs

@@ -0,0 +1,405 @@
+use super::{Expr, Operator, Statement};
+use crate::ast::ImportPattern;
+use crate::{engine::StateWorkingSet, BlockId, Signature, Span, Type, VarId, IN_VARIABLE_ID};
+
+#[derive(Debug, Clone)]
+pub struct Expression {
+    pub expr: Expr,
+    pub span: Span,
+    pub ty: Type,
+    pub custom_completion: Option<String>,
+}
+
+impl Expression {
+    pub fn garbage(span: Span) -> Expression {
+        Expression {
+            expr: Expr::Garbage,
+            span,
+            ty: Type::Unknown,
+            custom_completion: None,
+        }
+    }
+
+    pub fn precedence(&self) -> usize {
+        match &self.expr {
+            Expr::Operator(operator) => {
+                // Higher precedence binds tighter
+
+                match operator {
+                    Operator::Pow => 100,
+                    Operator::Multiply | Operator::Divide | Operator::Modulo => 95,
+                    Operator::Plus | Operator::Minus => 90,
+                    Operator::NotContains
+                    | Operator::Contains
+                    | Operator::LessThan
+                    | Operator::LessThanOrEqual
+                    | Operator::GreaterThan
+                    | Operator::GreaterThanOrEqual
+                    | Operator::Equal
+                    | Operator::NotEqual
+                    | Operator::In
+                    | Operator::NotIn => 80,
+                    Operator::And => 50,
+                    Operator::Or => 40,
+                }
+            }
+            _ => 0,
+        }
+    }
+
+    pub fn as_block(&self) -> Option<BlockId> {
+        match self.expr {
+            Expr::Block(block_id) => Some(block_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_row_condition_block(&self) -> Option<BlockId> {
+        match self.expr {
+            Expr::RowCondition(block_id) => Some(block_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_signature(&self) -> Option<Box<Signature>> {
+        match &self.expr {
+            Expr::Signature(sig) => Some(sig.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_list(&self) -> Option<Vec<Expression>> {
+        match &self.expr {
+            Expr::List(list) => Some(list.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_keyword(&self) -> Option<&Expression> {
+        match &self.expr {
+            Expr::Keyword(_, _, expr) => Some(expr),
+            _ => None,
+        }
+    }
+
+    pub fn as_var(&self) -> Option<VarId> {
+        match self.expr {
+            Expr::Var(var_id) => Some(var_id),
+            Expr::VarDecl(var_id) => Some(var_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_string(&self) -> Option<String> {
+        match &self.expr {
+            Expr::String(string) => Some(string.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_import_pattern(&self) -> Option<ImportPattern> {
+        match &self.expr {
+            Expr::ImportPattern(pattern) => Some(pattern.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn has_in_variable(&self, working_set: &StateWorkingSet) -> bool {
+        match &self.expr {
+            Expr::BinaryOp(left, _, right) => {
+                left.has_in_variable(working_set) || right.has_in_variable(working_set)
+            }
+            Expr::Block(block_id) => {
+                let block = working_set.get_block(*block_id);
+
+                if block.captures.contains(&IN_VARIABLE_ID) {
+                    return true;
+                }
+
+                if let Some(Statement::Pipeline(pipeline)) = block.stmts.get(0) {
+                    match pipeline.expressions.get(0) {
+                        Some(expr) => expr.has_in_variable(working_set),
+                        None => false,
+                    }
+                } else {
+                    false
+                }
+            }
+            Expr::Bool(_) => false,
+            Expr::Call(call) => {
+                for positional in &call.positional {
+                    if positional.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                for named in &call.named {
+                    if let Some(expr) = &named.1 {
+                        if expr.has_in_variable(working_set) {
+                            return true;
+                        }
+                    }
+                }
+                false
+            }
+            Expr::CellPath(_) => false,
+            Expr::ExternalCall(head, args) => {
+                if head.has_in_variable(working_set) {
+                    return true;
+                }
+                for arg in args {
+                    if arg.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Expr::ImportPattern(_) => false,
+            Expr::Filepath(_) => false,
+            Expr::Float(_) => false,
+            Expr::FullCellPath(full_cell_path) => {
+                if full_cell_path.head.has_in_variable(working_set) {
+                    return true;
+                }
+                false
+            }
+            Expr::Garbage => false,
+            Expr::Nothing => false,
+            Expr::GlobPattern(_) => false,
+            Expr::Int(_) => false,
+            Expr::Keyword(_, _, expr) => expr.has_in_variable(working_set),
+            Expr::List(list) => {
+                for l in list {
+                    if l.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Expr::StringInterpolation(items) => {
+                for i in items {
+                    if i.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Expr::Operator(_) => false,
+            Expr::Range(left, middle, right, ..) => {
+                if let Some(left) = &left {
+                    if left.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                if let Some(middle) = &middle {
+                    if middle.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                if let Some(right) = &right {
+                    if right.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Expr::Record(fields) => {
+                for (field_name, field_value) in fields {
+                    if field_name.has_in_variable(working_set) {
+                        return true;
+                    }
+                    if field_value.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Expr::Signature(_) => false,
+            Expr::String(_) => false,
+            Expr::RowCondition(block_id) | Expr::Subexpression(block_id) => {
+                let block = working_set.get_block(*block_id);
+
+                if let Some(Statement::Pipeline(pipeline)) = block.stmts.get(0) {
+                    if let Some(expr) = pipeline.expressions.get(0) {
+                        expr.has_in_variable(working_set)
+                    } else {
+                        false
+                    }
+                } else {
+                    false
+                }
+            }
+            Expr::Table(headers, cells) => {
+                for header in headers {
+                    if header.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+
+                for row in cells {
+                    for cell in row.iter() {
+                        if cell.has_in_variable(working_set) {
+                            return true;
+                        }
+                    }
+                }
+
+                false
+            }
+
+            Expr::ValueWithUnit(expr, _) => expr.has_in_variable(working_set),
+            Expr::Var(var_id) => *var_id == IN_VARIABLE_ID,
+            Expr::VarDecl(_) => false,
+        }
+    }
+
+    pub fn replace_in_variable(&mut self, working_set: &mut StateWorkingSet, new_var_id: VarId) {
+        match &mut self.expr {
+            Expr::BinaryOp(left, _, right) => {
+                left.replace_in_variable(working_set, new_var_id);
+                right.replace_in_variable(working_set, new_var_id);
+            }
+            Expr::Block(block_id) => {
+                let block = working_set.get_block(*block_id);
+
+                let new_expr = if let Some(Statement::Pipeline(pipeline)) = block.stmts.get(0) {
+                    if let Some(expr) = pipeline.expressions.get(0) {
+                        let mut new_expr = expr.clone();
+                        new_expr.replace_in_variable(working_set, new_var_id);
+                        Some(new_expr)
+                    } else {
+                        None
+                    }
+                } else {
+                    None
+                };
+
+                let block = working_set.get_block_mut(*block_id);
+
+                if let Some(new_expr) = new_expr {
+                    if let Some(Statement::Pipeline(pipeline)) = block.stmts.get_mut(0) {
+                        if let Some(expr) = pipeline.expressions.get_mut(0) {
+                            *expr = new_expr
+                        }
+                    }
+                }
+
+                block.captures = block
+                    .captures
+                    .iter()
+                    .map(|x| if *x != IN_VARIABLE_ID { *x } else { new_var_id })
+                    .collect();
+            }
+            Expr::Bool(_) => {}
+            Expr::Call(call) => {
+                for positional in &mut call.positional {
+                    positional.replace_in_variable(working_set, new_var_id);
+                }
+                for named in &mut call.named {
+                    if let Some(expr) = &mut named.1 {
+                        expr.replace_in_variable(working_set, new_var_id)
+                    }
+                }
+            }
+            Expr::CellPath(_) => {}
+            Expr::ExternalCall(head, args) => {
+                head.replace_in_variable(working_set, new_var_id);
+                for arg in args {
+                    arg.replace_in_variable(working_set, new_var_id)
+                }
+            }
+            Expr::Filepath(_) => {}
+            Expr::Float(_) => {}
+            Expr::FullCellPath(full_cell_path) => {
+                full_cell_path
+                    .head
+                    .replace_in_variable(working_set, new_var_id);
+            }
+            Expr::ImportPattern(_) => {}
+            Expr::Garbage => {}
+            Expr::Nothing => {}
+            Expr::GlobPattern(_) => {}
+            Expr::Int(_) => {}
+            Expr::Keyword(_, _, expr) => expr.replace_in_variable(working_set, new_var_id),
+            Expr::List(list) => {
+                for l in list {
+                    l.replace_in_variable(working_set, new_var_id)
+                }
+            }
+            Expr::Operator(_) => {}
+            Expr::Range(left, middle, right, ..) => {
+                if let Some(left) = left {
+                    left.replace_in_variable(working_set, new_var_id)
+                }
+                if let Some(middle) = middle {
+                    middle.replace_in_variable(working_set, new_var_id)
+                }
+                if let Some(right) = right {
+                    right.replace_in_variable(working_set, new_var_id)
+                }
+            }
+            Expr::Record(fields) => {
+                for (field_name, field_value) in fields {
+                    field_name.replace_in_variable(working_set, new_var_id);
+                    field_value.replace_in_variable(working_set, new_var_id);
+                }
+            }
+            Expr::Signature(_) => {}
+            Expr::String(_) => {}
+            Expr::StringInterpolation(items) => {
+                for i in items {
+                    i.replace_in_variable(working_set, new_var_id)
+                }
+            }
+            Expr::RowCondition(block_id) | Expr::Subexpression(block_id) => {
+                let block = working_set.get_block(*block_id);
+
+                let new_expr = if let Some(Statement::Pipeline(pipeline)) = block.stmts.get(0) {
+                    if let Some(expr) = pipeline.expressions.get(0) {
+                        let mut new_expr = expr.clone();
+                        new_expr.replace_in_variable(working_set, new_var_id);
+                        Some(new_expr)
+                    } else {
+                        None
+                    }
+                } else {
+                    None
+                };
+
+                let block = working_set.get_block_mut(*block_id);
+
+                if let Some(new_expr) = new_expr {
+                    if let Some(Statement::Pipeline(pipeline)) = block.stmts.get_mut(0) {
+                        if let Some(expr) = pipeline.expressions.get_mut(0) {
+                            *expr = new_expr
+                        }
+                    }
+                }
+
+                block.captures = block
+                    .captures
+                    .iter()
+                    .map(|x| if *x != IN_VARIABLE_ID { *x } else { new_var_id })
+                    .collect();
+            }
+            Expr::Table(headers, cells) => {
+                for header in headers {
+                    header.replace_in_variable(working_set, new_var_id)
+                }
+
+                for row in cells {
+                    for cell in row.iter_mut() {
+                        cell.replace_in_variable(working_set, new_var_id)
+                    }
+                }
+            }
+
+            Expr::ValueWithUnit(expr, _) => expr.replace_in_variable(working_set, new_var_id),
+            Expr::Var(x) => {
+                if *x == IN_VARIABLE_ID {
+                    *x = new_var_id
+                }
+            }
+            Expr::VarDecl(_) => {}
+        }
+    }
+}

crates/nu-protocol/src/ast/import_pattern.rs

@@ -0,0 +1,69 @@
+use crate::{span, Span};
+use std::collections::HashSet;
+
+#[derive(Debug, Clone)]
+pub enum ImportPatternMember {
+    Glob { span: Span },
+    Name { name: Vec<u8>, span: Span },
+    List { names: Vec<(Vec<u8>, Span)> },
+}
+
+#[derive(Debug, Clone)]
+pub struct ImportPatternHead {
+    pub name: Vec<u8>,
+    pub span: Span,
+}
+
+#[derive(Debug, Clone)]
+pub struct ImportPattern {
+    pub head: ImportPatternHead,
+    pub members: Vec<ImportPatternMember>,
+    // communicate to eval which decls/aliases were hidden during `parse_hide()` so it does not
+    // interpret these as env var names:
+    pub hidden: HashSet<Vec<u8>>,
+}
+
+impl ImportPattern {
+    pub fn new() -> Self {
+        ImportPattern {
+            head: ImportPatternHead {
+                name: vec![],
+                span: Span { start: 0, end: 0 },
+            },
+            members: vec![],
+            hidden: HashSet::new(),
+        }
+    }
+
+    pub fn span(&self) -> Span {
+        let mut spans = vec![self.head.span];
+
+        for member in &self.members {
+            match member {
+                ImportPatternMember::Glob { span } => spans.push(*span),
+                ImportPatternMember::Name { name: _, span } => spans.push(*span),
+                ImportPatternMember::List { names } => {
+                    for (_, span) in names {
+                        spans.push(*span);
+                    }
+                }
+            }
+        }
+
+        span(&spans)
+    }
+
+    pub fn with_hidden(self, hidden: HashSet<Vec<u8>>) -> Self {
+        ImportPattern {
+            head: self.head,
+            members: self.members,
+            hidden,
+        }
+    }
+}
+
+impl Default for ImportPattern {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/nu-protocol/src/ast/mod.rs

@@ -0,0 +1,19 @@
+mod block;
+mod call;
+mod cell_path;
+mod expr;
+mod expression;
+mod import_pattern;
+mod operator;
+mod pipeline;
+mod statement;
+
+pub use block::*;
+pub use call::*;
+pub use cell_path::*;
+pub use expr::*;
+pub use expression::*;
+pub use import_pattern::*;
+pub use operator::*;
+pub use pipeline::*;
+pub use statement::*;

crates/nu-protocol/src/ast/operator.rs

@@ -0,0 +1,73 @@
+use crate::Span;
+
+use serde::{Deserialize, Serialize};
+use std::fmt::Display;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Operator {
+    Equal,
+    NotEqual,
+    LessThan,
+    GreaterThan,
+    LessThanOrEqual,
+    GreaterThanOrEqual,
+    Contains,
+    NotContains,
+    Plus,
+    Minus,
+    Multiply,
+    Divide,
+    In,
+    NotIn,
+    Modulo,
+    And,
+    Or,
+    Pow,
+}
+
+impl Display for Operator {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Operator::Equal => write!(f, "=="),
+            Operator::NotEqual => write!(f, "!="),
+            Operator::LessThan => write!(f, "<"),
+            Operator::GreaterThan => write!(f, ">"),
+            Operator::Contains => write!(f, "=~"),
+            Operator::NotContains => write!(f, "!~"),
+            Operator::Plus => write!(f, "+"),
+            Operator::Minus => write!(f, "-"),
+            Operator::Multiply => write!(f, "*"),
+            Operator::Divide => write!(f, "/"),
+            Operator::In => write!(f, "in"),
+            Operator::NotIn => write!(f, "not-in"),
+            Operator::Modulo => write!(f, "mod"),
+            Operator::And => write!(f, "&&"),
+            Operator::Or => write!(f, "||"),
+            Operator::Pow => write!(f, "**"),
+            Operator::LessThanOrEqual => write!(f, "<="),
+            Operator::GreaterThanOrEqual => write!(f, ">="),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
+pub enum RangeInclusion {
+    Inclusive,
+    RightExclusive,
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct RangeOperator {
+    pub inclusion: RangeInclusion,
+    pub span: Span,
+    pub next_op_span: Span,
+}
+
+impl Display for RangeOperator {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.inclusion {
+            RangeInclusion::Inclusive => write!(f, ".."),
+            RangeInclusion::RightExclusive => write!(f, "..<"),
+        }
+    }
+}

crates/nu-protocol/src/ast/pipeline.rs

@@ -0,0 +1,24 @@
+use crate::ast::Expression;
+
+#[derive(Debug, Clone)]
+pub struct Pipeline {
+    pub expressions: Vec<Expression>,
+}
+
+impl Default for Pipeline {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Pipeline {
+    pub fn new() -> Self {
+        Self {
+            expressions: vec![],
+        }
+    }
+
+    pub fn from_vec(expressions: Vec<Expression>) -> Pipeline {
+        Self { expressions }
+    }
+}

crates/nu-protocol/src/ast/statement.rs

@@ -0,0 +1,8 @@
+use super::Pipeline;
+use crate::DeclId;
+
+#[derive(Debug, Clone)]
+pub enum Statement {
+    Declaration(DeclId),
+    Pipeline(Pipeline),
+}

crates/nu-protocol/src/call_info.rs

@@ -1,121 +0,0 @@
-use crate::value::Value;
-use derive_new::new;
-use indexmap::IndexMap;
-use nu_errors::ShellError;
-use nu_source::Tag;
-use serde::{Deserialize, Serialize};
-
-/// Associated information for the call of a command, including the args passed to the command and a tag that spans the name of the command being called
-#[derive(Deserialize, Serialize, Debug, Clone)]
-pub struct CallInfo {
-    /// The arguments associated with this call
-    pub args: EvaluatedArgs,
-    /// The tag (underline-able position) of the name of the call itself
-    pub name_tag: Tag,
-}
-
-/// The set of positional and named arguments, after their values have been evaluated.
-///
-/// * Positional arguments are those who are given as values, without any associated flag. For example, in `foo arg1 arg2`, both `arg1` and `arg2` are positional arguments.
-/// * Named arguments are those associated with a flag. For example, `foo --given bar` the named argument would be name `given` and the value `bar`.
-#[derive(Debug, Default, new, Serialize, Deserialize, Clone)]
-pub struct EvaluatedArgs {
-    pub positional: Option<Vec<Value>>,
-    pub named: Option<IndexMap<String, Value>>,
-}
-
-impl EvaluatedArgs {
-    /// Retrieve a subset of positional arguments starting at a given position
-    pub fn slice_from(&self, from: usize) -> Vec<Value> {
-        let positional = &self.positional;
-
-        match positional {
-            None => vec![],
-            Some(list) => list[from..].to_vec(),
-        }
-    }
-
-    /// Get the nth positional argument, if possible
-    pub fn nth(&self, pos: usize) -> Option<&Value> {
-        match &self.positional {
-            None => None,
-            Some(array) => array.get(pos),
-        }
-    }
-
-    /// Get the nth positional argument, error if not possible
-    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
-        match &self.positional {
-            None => Err(ShellError::unimplemented("Better error: expect_nth")),
-            Some(array) => match array.get(pos) {
-                None => Err(ShellError::unimplemented("Better error: expect_nth")),
-                Some(item) => Ok(item),
-            },
-        }
-    }
-
-    /// Get the number of positional arguments available
-    pub fn len(&self) -> usize {
-        match &self.positional {
-            None => 0,
-            Some(array) => array.len(),
-        }
-    }
-
-    /// Return if there are no positional arguments
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    /// Return true if the set of named arguments contains the name provided
-    pub fn has(&self, name: &str) -> bool {
-        matches!(&self.named, Some(named) if named.contains_key(name))
-    }
-
-    /// Gets the corresponding Value for the named argument given, if possible
-    pub fn get(&self, name: &str) -> Option<&Value> {
-        match &self.named {
-            None => None,
-            Some(named) => named.get(name),
-        }
-    }
-
-    /// Gets the corresponding Value for the named argument given, error if not possible
-    pub fn expect_get(&self, name: &str) -> Result<&Value, ShellError> {
-        match &self.named {
-            None => Err(ShellError::unimplemented("Better error: expect_get")),
-            Some(named) => named
-                .get(name)
-                .ok_or_else(|| ShellError::unimplemented("Better error: expect_get")),
-        }
-    }
-
-    /// Iterates over the positional arguments
-    pub fn positional_iter(&self) -> PositionalIter<'_> {
-        match &self.positional {
-            None => PositionalIter::Empty,
-            Some(v) => {
-                let iter = v.iter();
-                PositionalIter::Array(iter)
-            }
-        }
-    }
-}
-
-/// An iterator to help iterate over positional arguments
-pub enum PositionalIter<'a> {
-    Empty,
-    Array(std::slice::Iter<'a, Value>),
-}
-
-impl<'a> Iterator for PositionalIter<'a> {
-    type Item = &'a Value;
-
-    /// The required `next` function to implement the Iterator trait
-    fn next(&mut self) -> Option<Self::Item> {
-        match self {
-            PositionalIter::Empty => None,
-            PositionalIter::Array(iter) => iter.next(),
-        }
-    }
-}

crates/nu-protocol/src/config.rs

@@ -0,0 +1,372 @@
+use crate::{BlockId, ShellError, Span, Value};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+const ANIMATE_PROMPT_DEFAULT: bool = true;
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct EnvConversion {
+    pub from_string: Option<(BlockId, Span)>,
+    pub to_string: Option<(BlockId, Span)>,
+}
+
+impl EnvConversion {
+    pub fn from_record(value: &Value) -> Result<Self, ShellError> {
+        let record = value.as_record()?;
+
+        let mut conv_map = HashMap::new();
+
+        for (k, v) in record.0.iter().zip(record.1) {
+            if (k == "from_string") || (k == "to_string") {
+                conv_map.insert(k.as_str(), (v.as_block()?, v.span()?));
+            } else {
+                return Err(ShellError::UnsupportedConfigValue(
+                    "'from_string' and 'to_string' fields".into(),
+                    k.into(),
+                    value.span()?,
+                ));
+            }
+        }
+
+        let from_string = conv_map.get("from_string").cloned();
+        let to_string = conv_map.get("to_string").cloned();
+
+        Ok(EnvConversion {
+            from_string,
+            to_string,
+        })
+    }
+}
+
+/// Definition of a parsed keybinding from the config object
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ParsedKeybinding {
+    pub modifier: Value,
+    pub keycode: Value,
+    pub event: Value,
+    pub mode: Value,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct Config {
+    pub filesize_metric: bool,
+    pub table_mode: String,
+    pub use_ls_colors: bool,
+    pub color_config: HashMap<String, Value>,
+    pub use_grid_icons: bool,
+    pub footer_mode: FooterMode,
+    pub animate_prompt: bool,
+    pub float_precision: i64,
+    pub filesize_format: String,
+    pub use_ansi_coloring: bool,
+    pub quick_completions: bool,
+    pub env_conversions: HashMap<String, EnvConversion>,
+    pub edit_mode: String,
+    pub max_history_size: i64,
+    pub log_level: String,
+    pub menu_config: HashMap<String, Value>,
+    pub keybindings: Vec<ParsedKeybinding>,
+    pub history_config: HashMap<String, Value>,
+}
+
+impl Default for Config {
+    fn default() -> Config {
+        Config {
+            filesize_metric: false,
+            table_mode: "rounded".into(),
+            use_ls_colors: true,
+            color_config: HashMap::new(),
+            use_grid_icons: false,
+            footer_mode: FooterMode::Never,
+            animate_prompt: ANIMATE_PROMPT_DEFAULT,
+            float_precision: 4,
+            filesize_format: "auto".into(),
+            use_ansi_coloring: true,
+            quick_completions: false,
+            env_conversions: HashMap::new(), // TODO: Add default conversoins
+            edit_mode: "emacs".into(),
+            max_history_size: 1000,
+            log_level: String::new(),
+            menu_config: HashMap::new(),
+            keybindings: Vec::new(),
+            history_config: HashMap::new(),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub enum FooterMode {
+    /// Never show the footer
+    Never,
+    /// Always show the footer
+    Always,
+    /// Only show the footer if there are more than RowCount rows
+    RowCount(u64),
+    /// Calculate the screen height, calculate row count, if display will be bigger than screen, add the footer
+    Auto,
+}
+
+impl Value {
+    pub fn into_config(self) -> Result<Config, ShellError> {
+        let v = self.as_record();
+
+        let mut config = Config::default();
+
+        if let Ok(v) = v {
+            for (key, value) in v.0.iter().zip(v.1) {
+                match key.as_str() {
+                    "filesize_metric" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.filesize_metric = b;
+                        } else {
+                            eprintln!("$config.filesize_metric is not a bool")
+                        }
+                    }
+                    "table_mode" => {
+                        if let Ok(v) = value.as_string() {
+                            config.table_mode = v;
+                        } else {
+                            eprintln!("$config.table_mode is not a string")
+                        }
+                    }
+                    "use_ls_colors" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.use_ls_colors = b;
+                        } else {
+                            eprintln!("$config.use_ls_colors is not a bool")
+                        }
+                    }
+                    "color_config" => {
+                        if let Ok(map) = create_map(value, &config) {
+                            config.color_config = map;
+                        } else {
+                            eprintln!("$config.color_config is not a record")
+                        }
+                    }
+                    "use_grid_icons" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.use_grid_icons = b;
+                        } else {
+                            eprintln!("$config.use_grid_icons is not a bool")
+                        }
+                    }
+                    "footer_mode" => {
+                        if let Ok(b) = value.as_string() {
+                            let val_str = b.to_lowercase();
+                            config.footer_mode = match val_str.as_ref() {
+                                "auto" => FooterMode::Auto,
+                                "never" => FooterMode::Never,
+                                "always" => FooterMode::Always,
+                                _ => match &val_str.parse::<u64>() {
+                                    Ok(number) => FooterMode::RowCount(*number),
+                                    _ => FooterMode::Never,
+                                },
+                            };
+                        } else {
+                            eprintln!("$config.footer_mode is not a string")
+                        }
+                    }
+                    "animate_prompt" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.animate_prompt = b;
+                        } else {
+                            eprintln!("$config.animate_prompt is not a bool")
+                        }
+                    }
+                    "float_precision" => {
+                        if let Ok(i) = value.as_integer() {
+                            config.float_precision = i;
+                        } else {
+                            eprintln!("$config.float_precision is not an integer")
+                        }
+                    }
+                    "use_ansi_coloring" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.use_ansi_coloring = b;
+                        } else {
+                            eprintln!("$config.use_ansi_coloring is not a bool")
+                        }
+                    }
+                    "quick_completions" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.quick_completions = b;
+                        } else {
+                            eprintln!("$config.quick_completions is not a bool")
+                        }
+                    }
+                    "filesize_format" => {
+                        if let Ok(v) = value.as_string() {
+                            config.filesize_format = v.to_lowercase();
+                        } else {
+                            eprintln!("$config.filesize_format is not a string")
+                        }
+                    }
+                    "env_conversions" => {
+                        if let Ok((env_vars, conversions)) = value.as_record() {
+                            let mut env_conversions = HashMap::new();
+
+                            for (env_var, record) in env_vars.iter().zip(conversions) {
+                                // println!("{}: {:?}", env_var, record);
+                                if let Ok(conversion) = EnvConversion::from_record(record) {
+                                    env_conversions.insert(env_var.into(), conversion);
+                                } else {
+                                    eprintln!("$config.env_conversions has incorrect conversion")
+                                }
+                            }
+
+                            config.env_conversions = env_conversions;
+                        } else {
+                            eprintln!("$config.env_conversions is not a record")
+                        }
+                    }
+                    "edit_mode" => {
+                        if let Ok(v) = value.as_string() {
+                            config.edit_mode = v.to_lowercase();
+                        } else {
+                            eprintln!("$config.edit_mode is not a string")
+                        }
+                    }
+                    "max_history_size" => {
+                        if let Ok(i) = value.as_i64() {
+                            config.max_history_size = i;
+                        } else {
+                            eprintln!("$config.max_history_size is not an integer")
+                        }
+                    }
+                    "log_level" => {
+                        if let Ok(v) = value.as_string() {
+                            config.log_level = v.to_lowercase();
+                        } else {
+                            eprintln!("$config.log_level is not a string")
+                        }
+                    }
+                    "menu_config" => {
+                        if let Ok(map) = create_map(value, &config) {
+                            config.menu_config = map;
+                        } else {
+                            eprintln!("$config.menu_config is not a record")
+                        }
+                    }
+                    "keybindings" => {
+                        if let Ok(keybindings) = create_keybindings(value, &config) {
+                            config.keybindings = keybindings;
+                        } else {
+                            eprintln!("$config.keybindings is not a valid keybindings list")
+                        }
+                    }
+                    "history_config" => {
+                        if let Ok(map) = create_map(value, &config) {
+                            config.history_config = map;
+                        } else {
+                            eprintln!("$config.history_config is not a record")
+                        }
+                    }
+                    x => {
+                        eprintln!("$config.{} is an unknown config setting", x)
+                    }
+                }
+            }
+        } else {
+            eprintln!("$config is not a record");
+        }
+
+        Ok(config)
+    }
+}
+
+fn create_map(value: &Value, config: &Config) -> Result<HashMap<String, Value>, ShellError> {
+    let (cols, inner_vals) = value.as_record()?;
+    let mut hm: HashMap<String, Value> = HashMap::new();
+
+    for (k, v) in cols.iter().zip(inner_vals) {
+        match &v {
+            Value::Record {
+                cols: inner_cols,
+                vals: inner_vals,
+                span,
+            } => {
+                let val = color_value_string(span, inner_cols, inner_vals, config);
+                hm.insert(k.to_string(), val);
+            }
+            _ => {
+                hm.insert(k.to_string(), v.clone());
+            }
+        }
+    }
+
+    Ok(hm)
+}
+
+fn color_value_string(
+    span: &Span,
+    inner_cols: &[String],
+    inner_vals: &[Value],
+    config: &Config,
+) -> Value {
+    // make a string from our config.color_config section that
+    // looks like this: { fg: "#rrggbb" bg: "#rrggbb" attr: "abc", }
+    // the real key here was to have quotes around the values but not
+    // require them around the keys.
+
+    // maybe there's a better way to generate this but i'm not sure
+    // what it is.
+    let val: String = inner_cols
+        .iter()
+        .zip(inner_vals)
+        .map(|(x, y)| format!("{}: \"{}\" ", x, y.into_string(", ", config)))
+        .collect();
+
+    // now insert the braces at the front and the back to fake the json string
+    Value::String {
+        val: format!("{{{}}}", val),
+        span: *span,
+    }
+}
+
+// Parses the config object to extract the strings that will compose a keybinding for reedline
+fn create_keybindings(value: &Value, config: &Config) -> Result<Vec<ParsedKeybinding>, ShellError> {
+    match value {
+        Value::Record { cols, vals, span } => {
+            // Finding the modifier value in the record
+            let modifier = extract_value("modifier", cols, vals, span)?;
+            let keycode = extract_value("keycode", cols, vals, span)?;
+            let mode = extract_value("mode", cols, vals, span)?;
+            let event = extract_value("event", cols, vals, span)?;
+
+            let keybinding = ParsedKeybinding {
+                modifier: modifier.clone(),
+                keycode: keycode.clone(),
+                mode: mode.clone(),
+                event: event.clone(),
+            };
+
+            Ok(vec![keybinding])
+        }
+        Value::List { vals, .. } => {
+            let res = vals
+                .iter()
+                .map(|inner_value| create_keybindings(inner_value, config))
+                .collect::<Result<Vec<Vec<ParsedKeybinding>>, ShellError>>();
+
+            let res = res?
+                .into_iter()
+                .flatten()
+                .collect::<Vec<ParsedKeybinding>>();
+
+            Ok(res)
+        }
+        _ => Ok(Vec::new()),
+    }
+}
+
+pub fn extract_value<'record>(
+    name: &str,
+    cols: &'record [String],
+    vals: &'record [Value],
+    span: &Span,
+) -> Result<&'record Value, ShellError> {
+    cols.iter()
+        .position(|col| col.as_str() == name)
+        .and_then(|index| vals.get(index))
+        .ok_or_else(|| ShellError::MissingConfigValue(name.to_string(), *span))
+}

crates/nu-protocol/src/config_path.rs

@@ -1,19 +0,0 @@
-use serde::{Deserialize, Serialize};
-use std::path::PathBuf;
-
-/// Specifies a path to a configuration file and its type
-#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
-pub enum ConfigPath {
-    /// Path to the global configuration file
-    Global(PathBuf),
-    /// Path to a local configuration file
-    Local(PathBuf),
-}
-
-impl ConfigPath {
-    pub fn get_path(&self) -> &PathBuf {
-        match self {
-            ConfigPath::Global(p) | ConfigPath::Local(p) => p,
-        }
-    }
-}

crates/nu-protocol/src/dataframe/compute_between.rs

@@ -1,880 +0,0 @@
-use bigdecimal::BigDecimal;
-use nu_errors::ShellError;
-use nu_source::Span;
-use num_traits::ToPrimitive;
-
-use super::{Axis, NuDataFrame};
-use crate::hir::Operator;
-use crate::{Primitive, ShellTypeName, UntaggedValue, Value};
-
-use polars::prelude::{
-    BooleanType, ChunkCompare, ChunkedArray, DataType, Float64Type, Int64Type, IntoSeries,
-    NumOpsDispatchChecked, PolarsError, Series,
-};
-use std::ops::{Add, BitAnd, BitOr, Div, Mul, Sub};
-
-pub fn compute_between_dataframes(
-    operator: Operator,
-    left: &Value,
-    right: &Value,
-) -> Result<UntaggedValue, (&'static str, &'static str)> {
-    if let (UntaggedValue::DataFrame(lhs), UntaggedValue::DataFrame(rhs)) =
-        (&left.value, &right.value)
-    {
-        let operation_span = right.tag.span.merge(left.tag.span);
-        match (lhs.is_series(), rhs.is_series()) {
-            (true, true) => {
-                let lhs = &lhs
-                    .as_series(&left.tag.span)
-                    .expect("Already checked that is a series");
-                let rhs = &rhs
-                    .as_series(&right.tag.span)
-                    .expect("Already checked that is a series");
-
-                if lhs.dtype() != rhs.dtype() {
-                    return Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Mixed datatypes",
-                            "this datatype does not match the right hand side datatype",
-                            &left.tag.span,
-                            format!(
-                                "Perhaps you want to change this datatype to '{}'",
-                                lhs.as_ref().dtype()
-                            ),
-                            &right.tag.span,
-                        ),
-                    ));
-                }
-
-                if lhs.len() != rhs.len() {
-                    return Ok(UntaggedValue::Error(ShellError::labeled_error(
-                        "Different length",
-                        "this column length does not match the right hand column length",
-                        &left.tag.span,
-                    )));
-                }
-
-                compute_between_series(operator, lhs, rhs, &operation_span)
-            }
-            _ => {
-                if lhs.as_ref().height() != rhs.as_ref().height() {
-                    return Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Mixed datatypes",
-                            "this datatype size does not match the right hand side datatype",
-                            &left.tag.span,
-                            "Perhaps you want to select another dataframe with same number of rows",
-                            &right.tag.span,
-                        ),
-                    ));
-                }
-
-                between_dataframes(operator, lhs, rhs, &operation_span)
-            }
-        }
-    } else {
-        Err((left.type_name(), right.type_name()))
-    }
-}
-
-pub fn between_dataframes(
-    operator: Operator,
-    lhs: &NuDataFrame,
-    rhs: &NuDataFrame,
-    operation_span: &Span,
-) -> Result<UntaggedValue, (&'static str, &'static str)> {
-    match operator {
-        Operator::Plus => match lhs.append_df(rhs, Axis::Row, operation_span) {
-            Ok(df) => Ok(df.into_untagged()),
-            Err(e) => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                "Appending error",
-                e.to_string(),
-                operation_span,
-            ))),
-        },
-        _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-            "Incorrect datatype",
-            "unable to use this datatype for this operation",
-            operation_span,
-        ))),
-    }
-}
-
-pub fn compute_between_series(
-    operator: Operator,
-    lhs: &Series,
-    rhs: &Series,
-    operation_span: &Span,
-) -> Result<UntaggedValue, (&'static str, &'static str)> {
-    match operator {
-        Operator::Plus => {
-            let mut res = lhs + rhs;
-            let name = format!("sum_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::Minus => {
-            let mut res = lhs - rhs;
-            let name = format!("sub_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::Multiply => {
-            let mut res = lhs * rhs;
-            let name = format!("mul_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::Divide => {
-            let res = lhs.checked_div(rhs);
-            match res {
-                Ok(mut res) => {
-                    let name = format!("div_{}_{}", lhs.name(), rhs.name());
-                    res.rename(&name);
-                    Ok(NuDataFrame::series_to_untagged(res, operation_span))
-                }
-                Err(e) => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                    "Division error",
-                    e.to_string(),
-                    operation_span,
-                ))),
-            }
-        }
-        Operator::Equal => {
-            let mut res = Series::eq(lhs, rhs).into_series();
-            let name = format!("eq_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::NotEqual => {
-            let mut res = Series::neq(lhs, rhs).into_series();
-            let name = format!("neq_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::LessThan => {
-            let mut res = Series::lt(lhs, rhs).into_series();
-            let name = format!("lt_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::LessThanOrEqual => {
-            let mut res = Series::lt_eq(lhs, rhs).into_series();
-            let name = format!("lte_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::GreaterThan => {
-            let mut res = Series::gt(lhs, rhs).into_series();
-            let name = format!("gt_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::GreaterThanOrEqual => {
-            let mut res = Series::gt_eq(lhs, rhs).into_series();
-            let name = format!("gte_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::And => match lhs.dtype() {
-            DataType::Boolean => {
-                let lhs_cast = lhs.bool();
-                let rhs_cast = rhs.bool();
-
-                match (lhs_cast, rhs_cast) {
-                    (Ok(l), Ok(r)) => {
-                        let mut res = l.bitand(r).into_series();
-                        let name = format!("and_{}_{}", lhs.name(), rhs.name());
-                        res.rename(&name);
-                        Ok(NuDataFrame::series_to_untagged(res, operation_span))
-                    }
-                    _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                        "Casting error",
-                        "unable to cast to boolean",
-                        operation_span,
-                    ))),
-                }
-            }
-            _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                "Incorrect datatype",
-                "And operation can only be done with boolean values",
-                operation_span,
-            ))),
-        },
-        Operator::Or => match lhs.dtype() {
-            DataType::Boolean => {
-                let lhs_cast = lhs.bool();
-                let rhs_cast = rhs.bool();
-
-                match (lhs_cast, rhs_cast) {
-                    (Ok(l), Ok(r)) => {
-                        let mut res = l.bitor(r).into_series();
-                        let name = format!("or_{}_{}", lhs.name(), rhs.name());
-                        res.rename(&name);
-                        Ok(NuDataFrame::series_to_untagged(res, operation_span))
-                    }
-                    _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                        "Casting error",
-                        "unable to cast to boolean",
-                        operation_span,
-                    ))),
-                }
-            }
-            _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                "Incorrect datatype",
-                "And operation can only be done with boolean values",
-                operation_span,
-            ))),
-        },
-        _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-            "Incorrect datatype",
-            "unable to use this datatype for this operation",
-            operation_span,
-        ))),
-    }
-}
-
-pub fn compute_series_single_value(
-    operator: Operator,
-    left: &Value,
-    right: &Value,
-) -> Result<UntaggedValue, (&'static str, &'static str)> {
-    if let (UntaggedValue::DataFrame(lhs), UntaggedValue::Primitive(_)) =
-        (&left.value, &right.value)
-    {
-        let lhs = match lhs.as_series(&left.tag.span) {
-            Ok(series) => series,
-            Err(e) => return Ok(UntaggedValue::Error(e)),
-        };
-
-        match operator {
-            Operator::Plus => match &right.value {
-                UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Int64Type>>::add,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    &val.to_i64()
-                        .expect("Internal error: protocol did not use compatible decimal"),
-                    <ChunkedArray<Int64Type>>::add,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(compute_series_decimal(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Float64Type>>::add,
-                    &left.tag.span,
-                )),
-                _ => Ok(UntaggedValue::Error(
-                    ShellError::labeled_error_with_secondary(
-                        "Operation unavailable",
-                        "unable to sum this value to the series",
-                        &right.tag.span,
-                        "Only int, bigInt or decimal values are allowed",
-                        &right.tag.span,
-                    ),
-                )),
-            },
-            Operator::Minus => match &right.value {
-                UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Int64Type>>::sub,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    &val.to_i64()
-                        .expect("Internal error: protocol did not use compatible decimal"),
-                    <ChunkedArray<Int64Type>>::sub,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(compute_series_decimal(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Float64Type>>::sub,
-                    &left.tag.span,
-                )),
-                _ => Ok(UntaggedValue::Error(
-                    ShellError::labeled_error_with_secondary(
-                        "Operation unavailable",
-                        "unable to subtract this value to the series",
-                        &right.tag.span,
-                        "Only int, bigInt or decimal values are allowed",
-                        &right.tag.span,
-                    ),
-                )),
-            },
-            Operator::Multiply => match &right.value {
-                UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Int64Type>>::mul,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    &val.to_i64()
-                        .expect("Internal error: protocol did not use compatible decimal"),
-                    <ChunkedArray<Int64Type>>::mul,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(compute_series_decimal(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Float64Type>>::mul,
-                    &left.tag.span,
-                )),
-                _ => Ok(UntaggedValue::Error(
-                    ShellError::labeled_error_with_secondary(
-                        "Operation unavailable",
-                        "unable to multiply this value to the series",
-                        &right.tag.span,
-                        "Only int, bigInt or decimal values are allowed",
-                        &right.tag.span,
-                    ),
-                )),
-            },
-            Operator::Divide => match &right.value {
-                UntaggedValue::Primitive(Primitive::Int(val)) => {
-                    if *val == 0 {
-                        Ok(UntaggedValue::Error(ShellError::labeled_error(
-                            "Division by zero",
-                            "Zero value found",
-                            &right.tag.span,
-                        )))
-                    } else {
-                        Ok(compute_series_i64(
-                            &lhs,
-                            val,
-                            <ChunkedArray<Int64Type>>::div,
-                            &left.tag.span,
-                        ))
-                    }
-                }
-                UntaggedValue::Primitive(Primitive::BigInt(val)) => {
-                    if val.eq(&0.into()) {
-                        Ok(UntaggedValue::Error(ShellError::labeled_error(
-                            "Division by zero",
-                            "Zero value found",
-                            &right.tag.span,
-                        )))
-                    } else {
-                        Ok(compute_series_i64(
-                            &lhs,
-                            &val.to_i64()
-                                .expect("Internal error: protocol did not use compatible decimal"),
-                            <ChunkedArray<Int64Type>>::div,
-                            &left.tag.span,
-                        ))
-                    }
-                }
-                UntaggedValue::Primitive(Primitive::Decimal(val)) => {
-                    if val.eq(&0.into()) {
-                        Ok(UntaggedValue::Error(ShellError::labeled_error(
-                            "Division by zero",
-                            "Zero value found",
-                            &right.tag.span,
-                        )))
-                    } else {
-                        Ok(compute_series_decimal(
-                            &lhs,
-                            val,
-                            <ChunkedArray<Float64Type>>::div,
-                            &left.tag.span,
-                        ))
-                    }
-                }
-                _ => Ok(UntaggedValue::Error(
-                    ShellError::labeled_error_with_secondary(
-                        "Operation unavailable",
-                        "unable to divide this value to the series",
-                        &right.tag.span,
-                        "Only primary values are allowed",
-                        &right.tag.span,
-                    ),
-                )),
-            },
-            Operator::Equal => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::eq, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::NotEqual => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::neq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::neq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::neq, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::LessThan => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::lt,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::lt,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::lt, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::LessThanOrEqual => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::lt_eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::lt_eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::lt_eq, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::GreaterThan => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::gt,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::gt,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::gt, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::GreaterThanOrEqual => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::gt_eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::gt_eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::gt_eq, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::Contains => match &right.value {
-                UntaggedValue::Primitive(Primitive::String(val)) => {
-                    Ok(contains_series_pat(&lhs, val, &left.tag.span))
-                }
-                _ => Ok(UntaggedValue::Error(
-                    ShellError::labeled_error_with_secondary(
-                        "Operation unavailable",
-                        "unable to perform this value to the series",
-                        &right.tag.span,
-                        "Only primary values are allowed",
-                        &right.tag.span,
-                    ),
-                )),
-            },
-            _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                "Incorrect datatype",
-                "unable to use this value for this operation",
-                &left.tag.span,
-            ))),
-        }
-    } else {
-        Err((left.type_name(), right.type_name()))
-    }
-}
-
-fn compute_series_i64<F>(series: &Series, val: &i64, f: F, span: &Span) -> UntaggedValue
-where
-    F: Fn(ChunkedArray<Int64Type>, i64) -> ChunkedArray<Int64Type>,
-{
-    match series.dtype() {
-        DataType::UInt32 | DataType::Int32 | DataType::UInt64 => {
-            let to_i64 = series.cast(&DataType::Int64);
-
-            match to_i64 {
-                Ok(series) => {
-                    let casted = series.i64();
-                    compute_casted_i64(casted, *val, f, span)
-                }
-                Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                    "Casting error",
-                    e.to_string(),
-                    span,
-                )),
-            }
-        }
-        DataType::Int64 => {
-            let casted = series.i64();
-            compute_casted_i64(casted, *val, f, span)
-        }
-        _ => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            format!(
-                "Series of type {} can not be used for operations with an i64 value",
-                series.dtype()
-            ),
-            span,
-        )),
-    }
-}
-
-fn compute_casted_i64<F>(
-    casted: Result<&ChunkedArray<Int64Type>, PolarsError>,
-    val: i64,
-    f: F,
-    span: &Span,
-) -> UntaggedValue
-where
-    F: Fn(ChunkedArray<Int64Type>, i64) -> ChunkedArray<Int64Type>,
-{
-    match casted {
-        Ok(casted) => {
-            let res = f(casted.clone(), val);
-            let res = res.into_series();
-            NuDataFrame::series_to_untagged(res, span)
-        }
-        Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            e.to_string(),
-            span,
-        )),
-    }
-}
-
-fn compute_series_decimal<F>(series: &Series, val: &BigDecimal, f: F, span: &Span) -> UntaggedValue
-where
-    F: Fn(ChunkedArray<Float64Type>, f64) -> ChunkedArray<Float64Type>,
-{
-    match series.dtype() {
-        DataType::Float32 => {
-            let to_f64 = series.cast(&DataType::Float64);
-
-            match to_f64 {
-                Ok(series) => {
-                    let casted = series.f64();
-                    compute_casted_f64(
-                        casted,
-                        val.to_f64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        f,
-                        span,
-                    )
-                }
-                Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                    "Casting error",
-                    e.to_string(),
-                    span,
-                )),
-            }
-        }
-        DataType::Float64 => {
-            let casted = series.f64();
-            compute_casted_f64(
-                casted,
-                val.to_f64()
-                    .expect("Internal error: protocol did not use compatible decimal"),
-                f,
-                span,
-            )
-        }
-        _ => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            format!(
-                "Series of type {} can not be used for operations with a decimal value",
-                series.dtype()
-            ),
-            span,
-        )),
-    }
-}
-
-fn compute_casted_f64<F>(
-    casted: Result<&ChunkedArray<Float64Type>, PolarsError>,
-    val: f64,
-    f: F,
-    span: &Span,
-) -> UntaggedValue
-where
-    F: Fn(ChunkedArray<Float64Type>, f64) -> ChunkedArray<Float64Type>,
-{
-    match casted {
-        Ok(casted) => {
-            let res = f(casted.clone(), val);
-            let res = res.into_series();
-            NuDataFrame::series_to_untagged(res, span)
-        }
-        Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            e.to_string(),
-            span,
-        )),
-    }
-}
-
-fn compare_series_i64<F>(series: &Series, val: &i64, f: F, span: &Span) -> UntaggedValue
-where
-    F: Fn(&ChunkedArray<Int64Type>, i64) -> ChunkedArray<BooleanType>,
-{
-    match series.dtype() {
-        DataType::UInt32 | DataType::Int32 | DataType::UInt64 => {
-            let to_i64 = series.cast(&DataType::Int64);
-
-            match to_i64 {
-                Ok(series) => {
-                    let casted = series.i64();
-                    compare_casted_i64(casted, *val, f, span)
-                }
-                Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                    "Casting error",
-                    e.to_string(),
-                    span,
-                )),
-            }
-        }
-        DataType::Int64 => {
-            let casted = series.i64();
-            compare_casted_i64(casted, *val, f, span)
-        }
-        _ => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            format!(
-                "Series of type {} can not be used for operations with an i64 value",
-                series.dtype()
-            ),
-            span,
-        )),
-    }
-}
-
-fn compare_casted_i64<F>(
-    casted: Result<&ChunkedArray<Int64Type>, PolarsError>,
-    val: i64,
-    f: F,
-    span: &Span,
-) -> UntaggedValue
-where
-    F: Fn(&ChunkedArray<Int64Type>, i64) -> ChunkedArray<BooleanType>,
-{
-    match casted {
-        Ok(casted) => {
-            let res = f(casted, val);
-            let res = res.into_series();
-            NuDataFrame::series_to_untagged(res, span)
-        }
-        Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            e.to_string(),
-            span,
-        )),
-    }
-}
-
-fn compare_series_decimal<F>(series: &Series, val: &BigDecimal, f: F, span: &Span) -> UntaggedValue
-where
-    F: Fn(&ChunkedArray<Float64Type>, f64) -> ChunkedArray<BooleanType>,
-{
-    match series.dtype() {
-        DataType::Float32 => {
-            let to_f64 = series.cast(&DataType::Float64);
-
-            match to_f64 {
-                Ok(series) => {
-                    let casted = series.f64();
-                    compare_casted_f64(
-                        casted,
-                        val.to_f64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        f,
-                        span,
-                    )
-                }
-                Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                    "Casting error",
-                    e.to_string(),
-                    span,
-                )),
-            }
-        }
-        DataType::Float64 => {
-            let casted = series.f64();
-            compare_casted_f64(
-                casted,
-                val.to_f64()
-                    .expect("Internal error: protocol did not use compatible decimal"),
-                f,
-                span,
-            )
-        }
-        _ => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            format!(
-                "Series of type {} can not be used for operations with a decimal value",
-                series.dtype()
-            ),
-            span,
-        )),
-    }
-}
-
-fn compare_casted_f64<F>(
-    casted: Result<&ChunkedArray<Float64Type>, PolarsError>,
-    val: f64,
-    f: F,
-    span: &Span,
-) -> UntaggedValue
-where
-    F: Fn(&ChunkedArray<Float64Type>, f64) -> ChunkedArray<BooleanType>,
-{
-    match casted {
-        Ok(casted) => {
-            let res = f(casted, val);
-            let res = res.into_series();
-            NuDataFrame::series_to_untagged(res, span)
-        }
-        Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            e.to_string(),
-            span,
-        )),
-    }
-}
-
-fn contains_series_pat(series: &Series, pat: &str, span: &Span) -> UntaggedValue {
-    let casted = series.utf8();
-    match casted {
-        Ok(casted) => {
-            let res = casted.contains(pat);
-
-            match res {
-                Ok(res) => {
-                    let res = res.into_series();
-                    NuDataFrame::series_to_untagged(res, span)
-                }
-                Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                    "Search error",
-                    e.to_string(),
-                    span,
-                )),
-            }
-        }
-        Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            e.to_string(),
-            span,
-        )),
-    }
-}

crates/nu-protocol/src/dataframe/conversion.rs

@@ -1,661 +0,0 @@
-use indexmap::map::{Entry, IndexMap};
-use polars::chunked_array::object::builder::ObjectChunkedBuilder;
-use polars::chunked_array::ChunkedArray;
-
-use bigdecimal::{FromPrimitive, ToPrimitive};
-use chrono::{DateTime, FixedOffset, NaiveDateTime};
-use nu_errors::ShellError;
-use nu_source::{Span, Tag};
-use num_bigint::BigInt;
-use polars::prelude::{
-    DataFrame, DataType, DatetimeChunked, Int64Type, IntoSeries, NamedFrom, NewChunkedArray,
-    ObjectType, PolarsNumericType, Series,
-};
-use std::ops::{Deref, DerefMut};
-
-use super::NuDataFrame;
-use crate::{Dictionary, Primitive, UntaggedValue, Value};
-
-const SECS_PER_DAY: i64 = 86_400;
-
-#[derive(Debug)]
-pub struct Column {
-    name: String,
-    values: Vec<Value>,
-}
-
-impl Column {
-    pub fn new(name: String, values: Vec<Value>) -> Self {
-        Self { name, values }
-    }
-
-    pub fn new_empty(name: String) -> Self {
-        Self {
-            name,
-            values: Vec::new(),
-        }
-    }
-
-    pub fn name(&self) -> &str {
-        self.name.as_str()
-    }
-
-    pub fn iter(&self) -> impl Iterator<Item = &Value> {
-        self.values.iter()
-    }
-}
-
-impl IntoIterator for Column {
-    type Item = Value;
-    type IntoIter = std::vec::IntoIter<Self::Item>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        self.values.into_iter()
-    }
-}
-
-impl Deref for Column {
-    type Target = Vec<Value>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.values
-    }
-}
-
-impl DerefMut for Column {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.values
-    }
-}
-
-#[derive(Debug)]
-pub enum InputType {
-    Integer,
-    Decimal,
-    String,
-    Boolean,
-    Object,
-    Date,
-    Duration,
-}
-
-#[derive(Debug)]
-pub struct TypedColumn {
-    column: Column,
-    column_type: Option<InputType>,
-}
-
-impl TypedColumn {
-    fn new_empty(name: String) -> Self {
-        Self {
-            column: Column::new_empty(name),
-            column_type: None,
-        }
-    }
-}
-
-impl Deref for TypedColumn {
-    type Target = Column;
-
-    fn deref(&self) -> &Self::Target {
-        &self.column
-    }
-}
-
-impl DerefMut for TypedColumn {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.column
-    }
-}
-
-pub type ColumnMap = IndexMap<String, TypedColumn>;
-
-pub fn create_column(
-    series: &Series,
-    from_row: usize,
-    to_row: usize,
-) -> Result<Column, ShellError> {
-    let size = to_row - from_row;
-    match series.dtype() {
-        DataType::Null => {
-            let values = std::iter::repeat(Value {
-                value: UntaggedValue::Primitive(Primitive::Nothing),
-                tag: Tag::default(),
-            })
-            .take(size)
-            .collect::<Vec<Value>>();
-
-            Ok(Column::new(series.name().into(), values))
-        }
-        DataType::UInt8 => {
-            let casted = series.u8().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::UInt16 => {
-            let casted = series.u16().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::UInt32 => {
-            let casted = series.u32().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::UInt64 => {
-            let casted = series.u64().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Int8 => {
-            let casted = series.i8().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Int16 => {
-            let casted = series.i16().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Int32 => {
-            let casted = series.i32().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Int64 => {
-            let casted = series.i64().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Float32 => {
-            let casted = series.f32().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Float64 => {
-            let casted = series.f64().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Boolean => {
-            let casted = series.bool().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-
-            let values = casted
-                .into_iter()
-                .skip(from_row)
-                .take(size)
-                .map(|v| match v {
-                    Some(a) => Value {
-                        value: UntaggedValue::Primitive((a).into()),
-                        tag: Tag::default(),
-                    },
-                    None => Value {
-                        value: UntaggedValue::Primitive(Primitive::Nothing),
-                        tag: Tag::default(),
-                    },
-                })
-                .collect::<Vec<Value>>();
-
-            Ok(Column::new(casted.name().into(), values))
-        }
-        DataType::Utf8 => {
-            let casted = series.utf8().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-
-            let values = casted
-                .into_iter()
-                .skip(from_row)
-                .take(size)
-                .map(|v| match v {
-                    Some(a) => Value {
-                        value: UntaggedValue::Primitive((a).into()),
-                        tag: Tag::default(),
-                    },
-                    None => Value {
-                        value: UntaggedValue::Primitive(Primitive::Nothing),
-                        tag: Tag::default(),
-                    },
-                })
-                .collect::<Vec<Value>>();
-
-            Ok(Column::new(casted.name().into(), values))
-        }
-        DataType::Object(_) => {
-            let casted = series
-                .as_any()
-                .downcast_ref::<ChunkedArray<ObjectType<Value>>>();
-
-            match casted {
-                None => Err(ShellError::labeled_error(
-                    "Format not supported",
-                    "Value not supported for conversion",
-                    Tag::unknown(),
-                )),
-                Some(ca) => {
-                    let values = ca
-                        .into_iter()
-                        .skip(from_row)
-                        .take(size)
-                        .map(|v| match v {
-                            Some(a) => a.clone(),
-                            None => Value {
-                                value: UntaggedValue::Primitive(Primitive::Nothing),
-                                tag: Tag::default(),
-                            },
-                        })
-                        .collect::<Vec<Value>>();
-
-                    Ok(Column::new(ca.name().into(), values))
-                }
-            }
-        }
-        DataType::Date => {
-            let casted = series.date().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-
-            let values = casted
-                .into_iter()
-                .skip(from_row)
-                .take(size)
-                .map(|v| match v {
-                    Some(a) => {
-                        // elapsed time in day since 1970-01-01
-                        let seconds = a as i64 * SECS_PER_DAY;
-                        let naive_datetime = NaiveDateTime::from_timestamp(seconds, 0);
-
-                        // Zero length offset
-                        let offset = FixedOffset::east(0);
-                        let datetime = DateTime::<FixedOffset>::from_utc(naive_datetime, offset);
-
-                        Value {
-                            value: UntaggedValue::Primitive(Primitive::Date(datetime)),
-                            tag: Tag::default(),
-                        }
-                    }
-                    None => Value {
-                        value: UntaggedValue::Primitive(Primitive::Nothing),
-                        tag: Tag::default(),
-                    },
-                })
-                .collect::<Vec<Value>>();
-
-            Ok(Column::new(casted.name().into(), values))
-        }
-        DataType::Datetime => {
-            let casted = series.datetime().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-
-            let values = casted
-                .into_iter()
-                .skip(from_row)
-                .take(size)
-                .map(|v| match v {
-                    Some(a) => {
-                        // elapsed time in milliseconds since 1970-01-01
-                        let seconds = a / 1000;
-                        let naive_datetime = NaiveDateTime::from_timestamp(seconds, 0);
-
-                        // Zero length offset
-                        let offset = FixedOffset::east(0);
-                        let datetime = DateTime::<FixedOffset>::from_utc(naive_datetime, offset);
-
-                        Value {
-                            value: UntaggedValue::Primitive(Primitive::Date(datetime)),
-                            tag: Tag::default(),
-                        }
-                    }
-                    None => Value {
-                        value: UntaggedValue::Primitive(Primitive::Nothing),
-                        tag: Tag::default(),
-                    },
-                })
-                .collect::<Vec<Value>>();
-
-            Ok(Column::new(casted.name().into(), values))
-        }
-        DataType::Time => {
-            let casted = series.time().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-
-            let values = casted
-                .into_iter()
-                .skip(from_row)
-                .take(size)
-                .map(|v| match v {
-                    Some(nanoseconds) => {
-                        let untagged = if let Some(bigint) = BigInt::from_i64(nanoseconds) {
-                            UntaggedValue::Primitive(Primitive::Duration(bigint))
-                        } else {
-                            unreachable!("Internal error: protocol did not use compatible decimal")
-                        };
-
-                        Value {
-                            value: untagged,
-                            tag: Tag::default(),
-                        }
-                    }
-                    None => Value {
-                        value: UntaggedValue::Primitive(Primitive::Nothing),
-                        tag: Tag::default(),
-                    },
-                })
-                .collect::<Vec<Value>>();
-
-            Ok(Column::new(casted.name().into(), values))
-        }
-        e => Err(ShellError::labeled_error(
-            "Format not supported",
-            format!("Value not supported for conversion: {}", e),
-            Tag::unknown(),
-        )),
-    }
-}
-
-fn column_from_casted<T>(casted: &ChunkedArray<T>, from_row: usize, size: usize) -> Column
-where
-    T: PolarsNumericType,
-    T::Native: Into<Primitive>,
-{
-    let values = casted
-        .into_iter()
-        .skip(from_row)
-        .take(size)
-        .map(|v| match v {
-            Some(a) => Value {
-                value: UntaggedValue::Primitive((a).into()),
-                tag: Tag::default(),
-            },
-            None => Value {
-                value: UntaggedValue::Primitive(Primitive::Nothing),
-                tag: Tag::default(),
-            },
-        })
-        .collect::<Vec<Value>>();
-
-    Column::new(casted.name().into(), values)
-}
-
-// Adds a separator to the vector of values using the column names from the
-// dataframe to create the Values Row
-pub fn add_separator(values: &mut Vec<Value>, df: &DataFrame) {
-    let column_names = df.get_column_names();
-
-    let mut dictionary = Dictionary::default();
-    for name in column_names {
-        let indicator = Value {
-            value: UntaggedValue::Primitive(Primitive::String("...".to_string())),
-            tag: Tag::unknown(),
-        };
-
-        dictionary.insert(name.to_string(), indicator);
-    }
-
-    let extra_column = Value {
-        value: UntaggedValue::Row(dictionary),
-        tag: Tag::unknown(),
-    };
-
-    values.push(extra_column);
-}
-
-// Inserting the values found in a UntaggedValue::Row
-// All the entries for the dictionary are checked in order to check if
-// the column values have the same type value.
-pub fn insert_row(column_values: &mut ColumnMap, dictionary: Dictionary) -> Result<(), ShellError> {
-    for (key, value) in dictionary.entries {
-        insert_value(value, key, column_values)?;
-    }
-
-    Ok(())
-}
-
-// Inserting the values found in a UntaggedValue::Table
-// All the entries for the table are checked in order to check if
-// the column values have the same type value.
-// The names for the columns are the enumerated numbers from the values
-pub fn insert_table(column_values: &mut ColumnMap, table: Vec<Value>) -> Result<(), ShellError> {
-    for (index, value) in table.into_iter().enumerate() {
-        let key = index.to_string();
-        insert_value(value, key, column_values)?;
-    }
-
-    Ok(())
-}
-
-pub fn insert_value(
-    value: Value,
-    key: String,
-    column_values: &mut ColumnMap,
-) -> Result<(), ShellError> {
-    let col_val = match column_values.entry(key.clone()) {
-        Entry::Vacant(entry) => entry.insert(TypedColumn::new_empty(key)),
-        Entry::Occupied(entry) => entry.into_mut(),
-    };
-
-    // Checking that the type for the value is the same
-    // for the previous value in the column
-    if col_val.values.is_empty() {
-        match &value.value {
-            UntaggedValue::Primitive(Primitive::Int(_)) => {
-                col_val.column_type = Some(InputType::Integer);
-            }
-            UntaggedValue::Primitive(Primitive::Decimal(_)) => {
-                col_val.column_type = Some(InputType::Decimal);
-            }
-            UntaggedValue::Primitive(Primitive::String(_)) => {
-                col_val.column_type = Some(InputType::String);
-            }
-            UntaggedValue::Primitive(Primitive::Boolean(_)) => {
-                col_val.column_type = Some(InputType::Boolean);
-            }
-            UntaggedValue::Primitive(Primitive::Date(_)) => {
-                col_val.column_type = Some(InputType::Date);
-            }
-            UntaggedValue::Primitive(Primitive::Duration(_)) => {
-                col_val.column_type = Some(InputType::Duration);
-            }
-            _ => col_val.column_type = Some(InputType::Object),
-        }
-        col_val.values.push(value);
-    } else {
-        let prev_value = &col_val.values[col_val.values.len() - 1];
-
-        match (&prev_value.value, &value.value) {
-            (
-                UntaggedValue::Primitive(Primitive::Int(_)),
-                UntaggedValue::Primitive(Primitive::Int(_)),
-            )
-            | (
-                UntaggedValue::Primitive(Primitive::Decimal(_)),
-                UntaggedValue::Primitive(Primitive::Decimal(_)),
-            )
-            | (
-                UntaggedValue::Primitive(Primitive::String(_)),
-                UntaggedValue::Primitive(Primitive::String(_)),
-            )
-            | (
-                UntaggedValue::Primitive(Primitive::Boolean(_)),
-                UntaggedValue::Primitive(Primitive::Boolean(_)),
-            )
-            | (
-                UntaggedValue::Primitive(Primitive::Date(_)),
-                UntaggedValue::Primitive(Primitive::Date(_)),
-            )
-            | (
-                UntaggedValue::Primitive(Primitive::Duration(_)),
-                UntaggedValue::Primitive(Primitive::Duration(_)),
-            ) => col_val.values.push(value),
-            _ => {
-                col_val.column_type = Some(InputType::Object);
-                col_val.values.push(value);
-            }
-        }
-    }
-
-    Ok(())
-}
-
-// The ColumnMap has the parsed data from the StreamInput
-// This data can be used to create a Series object that can initialize
-// the dataframe based on the type of data that is found
-pub fn from_parsed_columns(
-    column_values: ColumnMap,
-    span: &Span,
-) -> Result<NuDataFrame, ShellError> {
-    let mut df_series: Vec<Series> = Vec::new();
-    for (name, column) in column_values {
-        if let Some(column_type) = &column.column_type {
-            match column_type {
-                InputType::Decimal => {
-                    let series_values: Result<Vec<_>, _> =
-                        column.values.iter().map(|v| v.as_f64()).collect();
-                    let series = Series::new(&name, series_values?);
-                    df_series.push(series)
-                }
-                InputType::Integer => {
-                    let series_values: Result<Vec<_>, _> =
-                        column.values.iter().map(|v| v.as_i64()).collect();
-                    let series = Series::new(&name, series_values?);
-                    df_series.push(series)
-                }
-                InputType::String => {
-                    let series_values: Result<Vec<_>, _> =
-                        column.values.iter().map(|v| v.as_string()).collect();
-                    let series = Series::new(&name, series_values?);
-                    df_series.push(series)
-                }
-                InputType::Boolean => {
-                    let series_values: Result<Vec<_>, _> =
-                        column.values.iter().map(|v| v.as_bool()).collect();
-                    let series = Series::new(&name, series_values?);
-                    df_series.push(series)
-                }
-                InputType::Object => {
-                    let mut builder =
-                        ObjectChunkedBuilder::<Value>::new(&name, column.values.len());
-
-                    for v in &column.values {
-                        builder.append_value(v.clone());
-                    }
-
-                    let res = builder.finish();
-                    df_series.push(res.into_series())
-                }
-                InputType::Date => {
-                    let it = column.values.iter().map(|v| {
-                        if let UntaggedValue::Primitive(Primitive::Date(date)) = &v.value {
-                            Some(date.timestamp_millis())
-                        } else {
-                            None
-                        }
-                    });
-
-                    let res: DatetimeChunked =
-                        ChunkedArray::<Int64Type>::new_from_opt_iter(&name, it).into();
-
-                    df_series.push(res.into_series())
-                }
-                InputType::Duration => {
-                    let it = column.values.iter().map(|v| {
-                        if let UntaggedValue::Primitive(Primitive::Duration(duration)) = &v.value {
-                            Some(duration.to_i64().expect("Not expecting NAN in duration"))
-                        } else {
-                            None
-                        }
-                    });
-
-                    let res = ChunkedArray::<Int64Type>::new_from_opt_iter(&name, it);
-
-                    df_series.push(res.into_series())
-                }
-            }
-        }
-    }
-
-    let df = DataFrame::new(df_series);
-
-    match df {
-        Ok(df) => Ok(NuDataFrame::new(df)),
-        Err(e) => Err(ShellError::labeled_error(
-            "Error while creating dataframe",
-            e.to_string(),
-            span,
-        )),
-    }
-}

crates/nu-protocol/src/dataframe/mod.rs

@@ -1,17 +0,0 @@
-pub mod compute_between;
-pub mod conversion;
-pub mod nu_dataframe;
-pub mod nu_groupby;
-pub mod operations;
-
-pub use compute_between::{compute_between_dataframes, compute_series_single_value};
-pub use conversion::Column;
-pub use nu_dataframe::NuDataFrame;
-pub use nu_groupby::NuGroupBy;
-pub use operations::Axis;
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
-pub enum FrameStruct {
-    GroupBy(NuGroupBy),
-}

crates/nu-protocol/src/dataframe/nu_dataframe.rs

@@ -1,401 +0,0 @@
-use indexmap::IndexMap;
-use std::cmp::Ordering;
-use std::fmt::Display;
-use std::hash::{Hash, Hasher};
-
-use nu_errors::ShellError;
-use nu_source::{Span, Tag};
-use polars::prelude::{DataFrame, DataType, PolarsObject, Series};
-use serde::{Deserialize, Serialize};
-
-use super::conversion::{
-    add_separator, create_column, from_parsed_columns, insert_row, insert_table, insert_value,
-    Column, ColumnMap,
-};
-use crate::{Dictionary, Primitive, ShellTypeName, UntaggedValue, Value};
-
-impl Display for Value {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.type_name())
-    }
-}
-
-impl Default for Value {
-    fn default() -> Self {
-        Self {
-            value: UntaggedValue::Primitive(Primitive::Nothing),
-            tag: Tag::default(),
-        }
-    }
-}
-
-impl PolarsObject for Value {
-    fn type_name() -> &'static str {
-        "object"
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct NuDataFrame {
-    dataframe: DataFrame,
-}
-
-// Dataframes are considered equal if they have the same shape, column name
-// and values
-impl PartialEq for NuDataFrame {
-    fn eq(&self, other: &Self) -> bool {
-        if self.as_ref().width() == 0 {
-            // checking for empty dataframe
-            return false;
-        }
-
-        if self.as_ref().get_column_names() != other.as_ref().get_column_names() {
-            // checking both dataframes share the same names
-            return false;
-        }
-
-        if self.as_ref().height() != other.as_ref().height() {
-            // checking both dataframes have the same row size
-            return false;
-        }
-
-        // sorting dataframe by the first column
-        let column_names = self.as_ref().get_column_names();
-        let first_col = column_names
-            .get(0)
-            .expect("already checked that dataframe is different than 0");
-
-        // if unable to sort, then unable to compare
-        let lhs = match self.as_ref().sort(*first_col, false) {
-            Ok(df) => df,
-            Err(_) => return false,
-        };
-
-        let rhs = match other.as_ref().sort(*first_col, false) {
-            Ok(df) => df,
-            Err(_) => return false,
-        };
-
-        for name in self.as_ref().get_column_names() {
-            let self_series = lhs.column(name).expect("name from dataframe names");
-
-            let other_series = rhs
-                .column(name)
-                .expect("already checked that name in other");
-
-            let self_series = match self_series.dtype() {
-                // Casting needed to compare other numeric types with nushell numeric type.
-                // In nushell we only have i64 integer numeric types and any array created
-                // with nushell untagged primitives will be of type i64
-                DataType::UInt32 => match self_series.cast(&DataType::Int64) {
-                    Ok(series) => series,
-                    Err(_) => return false,
-                },
-                _ => self_series.clone(),
-            };
-
-            if !self_series.series_equal(other_series) {
-                return false;
-            }
-        }
-
-        true
-    }
-}
-
-impl Eq for NuDataFrame {}
-
-impl PartialOrd for NuDataFrame {
-    fn partial_cmp(&self, _: &Self) -> Option<Ordering> {
-        Some(Ordering::Equal)
-    }
-}
-
-impl Ord for NuDataFrame {
-    fn cmp(&self, _: &Self) -> Ordering {
-        Ordering::Equal
-    }
-}
-
-impl Hash for NuDataFrame {
-    fn hash<H: Hasher>(&self, _: &mut H) {}
-}
-
-impl AsRef<DataFrame> for NuDataFrame {
-    fn as_ref(&self) -> &polars::prelude::DataFrame {
-        &self.dataframe
-    }
-}
-
-impl AsMut<DataFrame> for NuDataFrame {
-    fn as_mut(&mut self) -> &mut polars::prelude::DataFrame {
-        &mut self.dataframe
-    }
-}
-
-impl NuDataFrame {
-    pub fn new(dataframe: polars::prelude::DataFrame) -> Self {
-        NuDataFrame { dataframe }
-    }
-
-    pub fn try_from_stream<T>(input: &mut T, span: &Span) -> Result<(Self, Tag), ShellError>
-    where
-        T: Iterator<Item = Value>,
-    {
-        input
-            .next()
-            .and_then(|value| match value.value {
-                UntaggedValue::DataFrame(df) => Some((df, value.tag)),
-                _ => None,
-            })
-            .ok_or_else(|| {
-                ShellError::labeled_error(
-                    "No dataframe in stream",
-                    "no dataframe found in input stream",
-                    span,
-                )
-            })
-    }
-
-    pub fn try_from_iter<T>(iter: T, tag: &Tag) -> Result<Self, ShellError>
-    where
-        T: Iterator<Item = Value>,
-    {
-        // Dictionary to store the columnar data extracted from
-        // the input. During the iteration we check if the values
-        // have different type
-        let mut column_values: ColumnMap = IndexMap::new();
-
-        for value in iter {
-            match value.value {
-                UntaggedValue::Row(dictionary) => insert_row(&mut column_values, dictionary)?,
-                UntaggedValue::Table(table) => insert_table(&mut column_values, table)?,
-                UntaggedValue::Primitive(Primitive::Int(_))
-                | UntaggedValue::Primitive(Primitive::Decimal(_))
-                | UntaggedValue::Primitive(Primitive::String(_))
-                | UntaggedValue::Primitive(Primitive::Boolean(_))
-                | UntaggedValue::Primitive(Primitive::Date(_))
-                | UntaggedValue::DataFrame(_) => {
-                    let key = "0".to_string();
-                    insert_value(value, key, &mut column_values)?
-                }
-                _ => {
-                    return Err(ShellError::labeled_error_with_secondary(
-                        "Format not supported",
-                        "Value not supported for conversion",
-                        &value.tag,
-                        "Perhaps you want to use a List, a List of Tables or a Dictionary",
-                        &value.tag,
-                    ));
-                }
-            }
-        }
-
-        from_parsed_columns(column_values, &tag.span)
-    }
-
-    pub fn try_from_series(columns: Vec<Series>, span: &Span) -> Result<Self, ShellError> {
-        let dataframe = DataFrame::new(columns).map_err(|e| {
-            ShellError::labeled_error(
-                "DataFrame Creation",
-                format!("Unable to create DataFrame: {}", e),
-                span,
-            )
-        })?;
-
-        Ok(Self { dataframe })
-    }
-
-    pub fn try_from_columns(columns: Vec<Column>, span: &Span) -> Result<Self, ShellError> {
-        let mut column_values: ColumnMap = IndexMap::new();
-
-        for column in columns {
-            let name = column.name().to_string();
-            for value in column {
-                insert_value(value, name.clone(), &mut column_values)?;
-            }
-        }
-
-        from_parsed_columns(column_values, span)
-    }
-
-    pub fn into_value(self, tag: Tag) -> Value {
-        Value {
-            value: Self::into_untagged(self),
-            tag,
-        }
-    }
-
-    pub fn into_untagged(self) -> UntaggedValue {
-        UntaggedValue::DataFrame(self)
-    }
-
-    pub fn dataframe_to_value(df: DataFrame, tag: Tag) -> Value {
-        Value {
-            value: Self::dataframe_to_untagged(df),
-            tag,
-        }
-    }
-
-    pub fn dataframe_to_untagged(df: DataFrame) -> UntaggedValue {
-        UntaggedValue::DataFrame(Self::new(df))
-    }
-
-    pub fn series_to_untagged(series: Series, span: &Span) -> UntaggedValue {
-        match DataFrame::new(vec![series]) {
-            Ok(dataframe) => UntaggedValue::DataFrame(Self { dataframe }),
-            Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                "DataFrame Creation",
-                format!("Unable to create DataFrame: {}", e),
-                span,
-            )),
-        }
-    }
-
-    pub fn column(&self, column: &str, tag: &Tag) -> Result<Self, ShellError> {
-        let s = self
-            .as_ref()
-            .column(column)
-            .map_err(|e| ShellError::labeled_error("Column not found", e.to_string(), tag.span))?;
-
-        let dataframe = DataFrame::new(vec![s.clone()])
-            .map_err(|e| ShellError::labeled_error("DataFrame error", e.to_string(), tag.span))?;
-
-        Ok(Self { dataframe })
-    }
-
-    pub fn is_series(&self) -> bool {
-        self.as_ref().width() == 1
-    }
-
-    pub fn as_series(&self, span: &Span) -> Result<Series, ShellError> {
-        if !self.is_series() {
-            return Err(ShellError::labeled_error_with_secondary(
-                "Not a Series",
-                "DataFrame cannot be used as Series",
-                span,
-                "Note that a Series is a DataFrame with one column",
-                span,
-            ));
-        }
-
-        let series = self
-            .as_ref()
-            .get_columns()
-            .get(0)
-            .expect("We have already checked that the width is 1");
-
-        Ok(series.clone())
-    }
-
-    pub fn get_value(&self, row: usize, span: Span) -> Result<Value, ShellError> {
-        let series = self.as_series(&Span::default())?;
-        let column = create_column(&series, row, row + 1)?;
-
-        if column.len() == 0 {
-            Err(ShellError::labeled_error_with_secondary(
-                "Not a valid row",
-                format!("No value found for index {}", row),
-                span,
-                format!("Note that the column size is {}", series.len()),
-                span,
-            ))
-        } else {
-            let value = column
-                .into_iter()
-                .next()
-                .expect("already checked there is a value");
-            Ok(value)
-        }
-    }
-
-    // Print is made out a head and if the dataframe is too large, then a tail
-    pub fn print(&self) -> Result<Vec<Value>, ShellError> {
-        let df = &self.as_ref();
-        let size: usize = 20;
-
-        if df.height() > size {
-            let sample_size = size / 2;
-            let mut values = self.head(Some(sample_size))?;
-            add_separator(&mut values, df);
-            let remaining = df.height() - sample_size;
-            let tail_size = remaining.min(sample_size);
-            let mut tail_values = self.tail(Some(tail_size))?;
-            values.append(&mut tail_values);
-
-            Ok(values)
-        } else {
-            Ok(self.head(Some(size))?)
-        }
-    }
-
-    pub fn head(&self, rows: Option<usize>) -> Result<Vec<Value>, ShellError> {
-        let to_row = rows.unwrap_or(5);
-        let values = self.to_rows(0, to_row)?;
-
-        Ok(values)
-    }
-
-    pub fn tail(&self, rows: Option<usize>) -> Result<Vec<Value>, ShellError> {
-        let df = &self.as_ref();
-        let to_row = df.height();
-        let size = rows.unwrap_or(5);
-        let from_row = to_row.saturating_sub(size);
-
-        let values = self.to_rows(from_row, to_row)?;
-
-        Ok(values)
-    }
-
-    pub fn to_rows(&self, from_row: usize, to_row: usize) -> Result<Vec<Value>, ShellError> {
-        let df = self.as_ref();
-        let upper_row = to_row.min(df.height());
-
-        let mut size: usize = 0;
-        let columns = self
-            .as_ref()
-            .get_columns()
-            .iter()
-            .map(|col| match create_column(col, from_row, upper_row) {
-                Ok(col) => {
-                    size = col.len();
-                    Ok(col)
-                }
-                Err(e) => Err(e),
-            })
-            .collect::<Result<Vec<Column>, ShellError>>()?;
-
-        let mut iterators = columns
-            .into_iter()
-            .map(|col| (col.name().to_string(), col.into_iter()))
-            .collect::<Vec<(String, std::vec::IntoIter<Value>)>>();
-
-        let values = (0..size)
-            .into_iter()
-            .map(|i| {
-                let mut dictionary_row = Dictionary::default();
-
-                for (name, col) in &mut iterators {
-                    let dict_val = match col.next() {
-                        Some(v) => v,
-                        None => {
-                            println!("index: {}", i);
-                            Value {
-                                value: UntaggedValue::Primitive(Primitive::Nothing),
-                                tag: Tag::default(),
-                            }
-                        }
-                    };
-                    dictionary_row.insert(name.clone(), dict_val);
-                }
-
-                Value {
-                    value: UntaggedValue::Row(dictionary_row),
-                    tag: Tag::unknown(),
-                }
-            })
-            .collect::<Vec<Value>>();
-
-        Ok(values)
-    }
-}

crates/nu-protocol/src/dataframe/nu_groupby.rs

@@ -1,75 +0,0 @@
-use nu_source::{Span, Tag};
-use polars::frame::groupby::{GroupBy, GroupTuples};
-use serde::{Deserialize, Serialize};
-
-use super::{FrameStruct, NuDataFrame};
-use nu_errors::ShellError;
-
-use crate::{TaggedDictBuilder, UntaggedValue, Value};
-
-#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
-pub struct NuGroupBy {
-    dataframe: NuDataFrame,
-    by: Vec<String>,
-    groups: GroupTuples,
-}
-
-impl NuGroupBy {
-    pub fn new(dataframe: NuDataFrame, by: Vec<String>, groups: GroupTuples) -> Self {
-        NuGroupBy {
-            dataframe,
-            by,
-            groups,
-        }
-    }
-
-    pub fn by(&self) -> &[String] {
-        &self.by
-    }
-
-    pub fn try_from_stream<T>(input: &mut T, span: &Span) -> Result<NuGroupBy, ShellError>
-    where
-        T: Iterator<Item = Value>,
-    {
-        input
-            .next()
-            .and_then(|value| match value.value {
-                UntaggedValue::FrameStruct(FrameStruct::GroupBy(group)) => Some(group),
-                _ => None,
-            })
-            .ok_or_else(|| {
-                ShellError::labeled_error(
-                    "No groupby object in stream",
-                    "no groupby object found in input stream",
-                    span,
-                )
-            })
-    }
-
-    pub fn to_groupby(&self) -> Result<GroupBy, ShellError> {
-        let df = self.dataframe.as_ref();
-
-        let by = df.select_series(&self.by).map_err(|e| {
-            ShellError::labeled_error("Error creating groupby", e.to_string(), Tag::unknown())
-        })?;
-
-        Ok(GroupBy::new(df, by, self.groups.clone(), None))
-    }
-
-    pub fn print(&self) -> Result<Vec<Value>, ShellError> {
-        let mut values: Vec<Value> = Vec::new();
-
-        let mut data = TaggedDictBuilder::new(Tag::unknown());
-        data.insert_value("property", "group by");
-        data.insert_value("value", self.by.join(", "));
-        values.push(data.into_value());
-
-        Ok(values)
-    }
-}
-
-impl AsRef<polars::prelude::DataFrame> for NuGroupBy {
-    fn as_ref(&self) -> &polars::prelude::DataFrame {
-        self.dataframe.as_ref()
-    }
-}

crates/nu-protocol/src/dataframe/operations.rs

@@ -1,120 +0,0 @@
-use nu_errors::ShellError;
-use nu_source::Span;
-use polars::prelude::{DataFrame, Series};
-
-use super::NuDataFrame;
-
-pub enum Axis {
-    Row,
-    Column,
-}
-
-impl Axis {
-    pub fn try_from_str(axis: &str, span: &Span) -> Result<Axis, ShellError> {
-        match axis {
-            "row" => Ok(Axis::Row),
-            "col" => Ok(Axis::Column),
-            _ => Err(ShellError::labeled_error_with_secondary(
-                "Wrong axis",
-                "The selected axis does not exist",
-                span,
-                "The only axis options are 'row' or 'col'",
-                span,
-            )),
-        }
-    }
-}
-
-impl NuDataFrame {
-    pub fn append_df(
-        &self,
-        other: &NuDataFrame,
-        axis: Axis,
-        span: &Span,
-    ) -> Result<Self, ShellError> {
-        match axis {
-            Axis::Row => {
-                let mut columns: Vec<&str> = Vec::new();
-
-                let new_cols = self
-                    .as_ref()
-                    .get_columns()
-                    .iter()
-                    .chain(other.as_ref().get_columns())
-                    .map(|s| {
-                        let name = if columns.contains(&s.name()) {
-                            format!("{}_{}", s.name(), "x")
-                        } else {
-                            columns.push(s.name());
-                            s.name().to_string()
-                        };
-
-                        let mut series = s.clone();
-                        series.rename(&name);
-                        series
-                    })
-                    .collect::<Vec<Series>>();
-
-                let df_new = DataFrame::new(new_cols).map_err(|e| {
-                    ShellError::labeled_error("Appending error", e.to_string(), span)
-                })?;
-
-                Ok(NuDataFrame::new(df_new))
-            }
-            Axis::Column => {
-                if self.as_ref().width() != other.as_ref().width() {
-                    return Err(ShellError::labeled_error(
-                        "Appending error",
-                        "Dataframes with different number of columns",
-                        span,
-                    ));
-                }
-
-                if !self
-                    .as_ref()
-                    .get_column_names()
-                    .iter()
-                    .all(|col| other.as_ref().get_column_names().contains(col))
-                {
-                    return Err(ShellError::labeled_error(
-                        "Appending error",
-                        "Dataframes with different columns names",
-                        span,
-                    ));
-                }
-
-                let new_cols = self
-                    .as_ref()
-                    .get_columns()
-                    .iter()
-                    .map(|s| {
-                        let other_col = other
-                            .as_ref()
-                            .column(s.name())
-                            .expect("Already checked that dataframes have same columns");
-
-                        let mut tmp = s.clone();
-                        let res = tmp.append(other_col);
-
-                        match res {
-                            Ok(s) => Ok(s.clone()),
-                            Err(e) => Err({
-                                ShellError::labeled_error(
-                                    "Appending error",
-                                    format!("Unable to append dataframes: {}", e),
-                                    span,
-                                )
-                            }),
-                        }
-                    })
-                    .collect::<Result<Vec<Series>, ShellError>>()?;
-
-                let df_new = DataFrame::new(new_cols).map_err(|e| {
-                    ShellError::labeled_error("Appending error", e.to_string(), span)
-                })?;
-
-                Ok(NuDataFrame::new(df_new))
-            }
-        }
-    }
-}

crates/nu-protocol/src/engine/call_info.rs

@@ -0,0 +1,8 @@
+use crate::ast::Call;
+use crate::Span;
+
+#[derive(Debug, Clone)]
+pub struct UnevaluatedCallInfo {
+    pub args: Call,
+    pub name_span: Span,
+}

crates/nu-protocol/src/engine/capture_block.rs

@@ -0,0 +1,9 @@
+use std::collections::HashMap;
+
+use crate::{BlockId, Value, VarId};
+
+#[derive(Clone, Debug)]
+pub struct CaptureBlock {
+    pub block_id: BlockId,
+    pub captures: HashMap<VarId, Value>,
+}

crates/nu-protocol/src/engine/command.rs

@@ -0,0 +1,78 @@
+use std::path::PathBuf;
+
+use crate::{ast::Call, BlockId, Example, PipelineData, ShellError, Signature};
+
+use super::{EngineState, Stack};
+
+pub trait Command: Send + Sync + CommandClone {
+    fn name(&self) -> &str;
+
+    fn signature(&self) -> Signature;
+
+    fn usage(&self) -> &str;
+
+    fn extra_usage(&self) -> &str {
+        ""
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError>;
+
+    fn is_binary(&self) -> bool {
+        false
+    }
+
+    // Commands that are not meant to be run by users
+    fn is_private(&self) -> bool {
+        false
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        Vec::new()
+    }
+
+    // This is a built-in command
+    fn is_builtin(&self) -> bool {
+        true
+    }
+
+    // Is a sub command
+    fn is_sub(&self) -> bool {
+        self.name().contains(' ')
+    }
+
+    // Is a plugin command (returns plugin's path, encoding and type of shell
+    // if the declaration is a plugin)
+    fn is_plugin(&self) -> Option<(&PathBuf, &str, &Option<PathBuf>)> {
+        None
+    }
+
+    // If command is a block i.e. def blah [] { }, get the block id
+    fn get_block_id(&self) -> Option<BlockId> {
+        None
+    }
+}
+
+pub trait CommandClone {
+    fn clone_box(&self) -> Box<dyn Command>;
+}
+
+impl<T> CommandClone for T
+where
+    T: 'static + Command + Clone,
+{
+    fn clone_box(&self) -> Box<dyn Command> {
+        Box::new(self.clone())
+    }
+}
+
+impl Clone for Box<dyn Command> {
+    fn clone(&self) -> Box<dyn Command> {
+        self.clone_box()
+    }
+}

crates/nu-protocol/src/engine/mod.rs

@@ -0,0 +1,11 @@
+mod call_info;
+mod capture_block;
+mod command;
+mod engine_state;
+mod stack;
+
+pub use call_info::*;
+pub use capture_block::*;
+pub use command::*;
+pub use engine_state::*;
+pub use stack::*;

crates/nu-protocol/src/engine/stack.rs

@@ -0,0 +1,249 @@
+use std::collections::{HashMap, HashSet};
+
+use crate::engine::EngineState;
+use crate::{Config, ShellError, Span, Value, VarId, CONFIG_VARIABLE_ID};
+
+/// A runtime value stack used during evaluation
+///
+/// A note on implementation:
+///
+/// We previously set up the stack in a traditional way, where stack frames had parents which would
+/// represent other frames that you might return to when exiting a function.
+///
+/// While experimenting with blocks, we found that we needed to have closure captures of variables
+/// seen outside of the blocks, so that they blocks could be run in a way that was both thread-safe
+/// and followed the restrictions for closures applied to iterators. The end result left us with
+/// closure-captured single stack frames that blocks could see.
+///
+/// Blocks make up the only scope and stack definition abstraction in Nushell. As a result, we were
+/// creating closure captures at any point we wanted to have a Block value we could safely evaluate
+/// in any context. This meant that the parents were going largely unused, with captured variables
+/// taking their place. The end result is this, where we no longer have separate frames, but instead
+/// use the Stack as a way of representing the local and closure-captured state.
+#[derive(Debug, Clone)]
+pub struct Stack {
+    /// Variables
+    pub vars: HashMap<VarId, Value>,
+    /// Environment variables arranged as a stack to be able to recover values from parent scopes
+    pub env_vars: Vec<HashMap<String, Value>>,
+    /// Tells which environment variables from engine state are hidden. We don't need to track the
+    /// env vars in the stack since we can just delete them.
+    pub env_hidden: HashSet<String>,
+}
+
+impl Default for Stack {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Stack {
+    pub fn new() -> Stack {
+        Stack {
+            vars: HashMap::new(),
+            env_vars: vec![],
+            env_hidden: HashSet::new(),
+        }
+    }
+
+    pub fn with_env(&mut self, env_vars: &[HashMap<String, Value>], env_hidden: &HashSet<String>) {
+        // Do not clone the environment if it hasn't changed
+        if self.env_vars.iter().any(|scope| !scope.is_empty()) {
+            self.env_vars = env_vars.to_owned();
+        }
+
+        if !self.env_hidden.is_empty() {
+            self.env_hidden = env_hidden.clone();
+        }
+    }
+
+    pub fn get_var(&self, var_id: VarId, span: Span) -> Result<Value, ShellError> {
+        if let Some(v) = self.vars.get(&var_id) {
+            return Ok(v.clone().with_span(span));
+        }
+
+        Err(ShellError::VariableNotFoundAtRuntime(span))
+    }
+
+    pub fn get_var_with_origin(&self, var_id: VarId, span: Span) -> Result<Value, ShellError> {
+        if let Some(v) = self.vars.get(&var_id) {
+            return Ok(v.clone());
+        }
+
+        Err(ShellError::VariableNotFoundAtRuntime(span))
+    }
+
+    pub fn add_var(&mut self, var_id: VarId, value: Value) {
+        self.vars.insert(var_id, value);
+    }
+
+    pub fn add_env_var(&mut self, var: String, value: Value) {
+        // if the env var was hidden, let's activate it again
+        self.env_hidden.remove(&var);
+
+        if let Some(scope) = self.env_vars.last_mut() {
+            scope.insert(var, value);
+        } else {
+            self.env_vars.push(HashMap::from([(var, value)]));
+        }
+    }
+
+    pub fn captures_to_stack(&self, captures: &HashMap<VarId, Value>) -> Stack {
+        let mut output = Stack::new();
+
+        output.vars = captures.clone();
+
+        // FIXME: this is probably slow
+        output.env_vars = self.env_vars.clone();
+        output.env_vars.push(HashMap::new());
+
+        let config = self
+            .get_var(CONFIG_VARIABLE_ID, Span::new(0, 0))
+            .expect("internal error: config is missing");
+        output.vars.insert(CONFIG_VARIABLE_ID, config);
+
+        output
+    }
+
+    pub fn gather_captures(&self, captures: &[VarId]) -> Stack {
+        let mut output = Stack::new();
+
+        let fake_span = Span::new(0, 0);
+
+        for capture in captures {
+            // Note: this assumes we have calculated captures correctly and that commands
+            // that take in a var decl will manually set this into scope when running the blocks
+            if let Ok(value) = self.get_var(*capture, fake_span) {
+                output.vars.insert(*capture, value);
+            }
+        }
+
+        // FIXME: this is probably slow
+        output.env_vars = self.env_vars.clone();
+        output.env_vars.push(HashMap::new());
+
+        let config = self
+            .get_var(CONFIG_VARIABLE_ID, fake_span)
+            .expect("internal error: config is missing");
+        output.vars.insert(CONFIG_VARIABLE_ID, config);
+
+        output
+    }
+
+    /// Flatten the env var scope frames into one frame
+    pub fn get_env_vars(&self, engine_state: &EngineState) -> HashMap<String, Value> {
+        // TODO: We're collecting im::HashMap to HashMap here. It might make sense to make these
+        // the same data structure.
+        let mut result: HashMap<String, Value> = engine_state
+            .env_vars
+            .iter()
+            .filter(|(k, _)| !self.env_hidden.contains(*k))
+            .map(|(k, v)| (k.clone(), v.clone()))
+            .collect();
+
+        for scope in &self.env_vars {
+            result.extend(scope.clone());
+        }
+
+        result
+    }
+
+    /// Same as get_env_vars, but returns only the names as a HashSet
+    pub fn get_env_var_names(&self, engine_state: &EngineState) -> HashSet<String> {
+        let mut result: HashSet<String> = engine_state
+            .env_vars
+            .keys()
+            .filter(|k| !self.env_hidden.contains(*k))
+            .cloned()
+            .collect();
+
+        for scope in &self.env_vars {
+            let scope_keys: HashSet<String> = scope.keys().cloned().collect();
+            result.extend(scope_keys);
+        }
+
+        result
+    }
+
+    pub fn get_env_var(&self, engine_state: &EngineState, name: &str) -> Option<Value> {
+        for scope in self.env_vars.iter().rev() {
+            if let Some(v) = scope.get(name) {
+                return Some(v.clone());
+            }
+        }
+
+        if self.env_hidden.contains(name) {
+            None
+        } else {
+            engine_state.env_vars.get(name).cloned()
+        }
+    }
+
+    pub fn has_env_var(&self, engine_state: &EngineState, name: &str) -> bool {
+        for scope in self.env_vars.iter().rev() {
+            if scope.contains_key(name) {
+                return true;
+            }
+        }
+
+        if self.env_hidden.contains(name) {
+            false
+        } else {
+            engine_state.env_vars.contains_key(name)
+        }
+    }
+
+    pub fn remove_env_var(&mut self, engine_state: &EngineState, name: &str) -> Option<Value> {
+        for scope in self.env_vars.iter_mut().rev() {
+            if let Some(v) = scope.remove(name) {
+                return Some(v);
+            }
+        }
+
+        if self.env_hidden.contains(name) {
+            // the environment variable is already hidden
+            None
+        } else if let Some(val) = engine_state.env_vars.get(name) {
+            // the environment variable was found in the engine state => mark it as hidden
+            self.env_hidden.insert(name.to_string());
+            Some(val.clone())
+        } else {
+            None
+        }
+    }
+
+    pub fn get_config(&self) -> Result<Config, ShellError> {
+        let config = self.get_var(CONFIG_VARIABLE_ID, Span::new(0, 0));
+
+        match config {
+            Ok(config) => config.into_config(),
+            Err(e) => Err(e),
+        }
+    }
+
+    pub fn update_config(&mut self, name: &str, value: Value) {
+        if let Some(Value::Record { cols, vals, .. }) = self.vars.get_mut(&CONFIG_VARIABLE_ID) {
+            for col_val in cols.iter().zip(vals.iter_mut()) {
+                if col_val.0 == name {
+                    *col_val.1 = value;
+                    return;
+                }
+            }
+            cols.push(name.to_string());
+            vals.push(value);
+        }
+    }
+
+    pub fn print_stack(&self) {
+        println!("vars:");
+        for (var, val) in &self.vars {
+            println!("  {}: {:?}", var, val);
+        }
+        for (i, scope) in self.env_vars.iter().rev().enumerate() {
+            println!("env vars, scope {} (from the last);", i);
+            for (var, val) in scope {
+                println!("  {}: {:?}", var, val.clone().debug_value());
+            }
+        }
+    }
+}

crates/nu-protocol/src/example.rs

@@ -0,0 +1,8 @@
+use crate::Value;
+
+#[derive(Debug)]
+pub struct Example {
+    pub example: &'static str,
+    pub description: &'static str,
+    pub result: Option<Value>,
+}

crates/nu-protocol/src/exportable.rs

@@ -0,0 +1,6 @@
+use crate::{BlockId, DeclId};
+
+pub enum Exportable {
+    Decl(DeclId),
+    EnvVar(BlockId),
+}

crates/nu-protocol/src/id.rs

@@ -0,0 +1,4 @@
+pub type VarId = usize;
+pub type DeclId = usize;
+pub type BlockId = usize;
+pub type OverlayId = usize;

crates/nu-protocol/src/lib.rs

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 #[macro_use]
 mod macros;
 
@@ -33,3 +34,37 @@ pub use crate::value::primitive::{format_date, format_duration, format_primitive
 pub use crate::value::range::{Range, RangeInclusion};
 pub use crate::value::value_structure::{ValueResource, ValueStructure};
 pub use crate::value::{merge_descriptors, UntaggedValue, Value};
+=======
+pub mod ast;
+mod config;
+pub mod engine;
+mod example;
+mod exportable;
+mod id;
+mod overlay;
+mod pipeline_data;
+mod shell_error;
+mod signature;
+mod span;
+mod syntax_shape;
+mod ty;
+mod value;
+pub use value::Value;
+
+pub use config::*;
+pub use engine::{
+    CONFIG_VARIABLE_ID, ENV_VARIABLE_ID, IN_VARIABLE_ID, NU_VARIABLE_ID, SCOPE_VARIABLE_ID,
+};
+pub use example::*;
+pub use exportable::*;
+pub use id::*;
+pub use overlay::*;
+pub use pipeline_data::*;
+pub use shell_error::*;
+pub use signature::*;
+pub use span::*;
+pub use syntax_shape::*;
+pub use ty::*;
+pub use value::CustomValue;
+pub use value::*;
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce

crates/nu-protocol/src/macros.rs

@@ -1,49 +0,0 @@
-/// Outputs to standard out
-///
-/// Note: this exists to differentiate between intentional writing to stdout
-/// and stray printlns left by accident
-#[macro_export]
-macro_rules! out {
-    ($($tokens:tt)*) => {
-        use std::io::Write;
-        write!(std::io::stdout(), $($tokens)*).unwrap_or(());
-        let _ = std::io::stdout().flush();
-    }
-}
-
-/// Outputs to standard out with a newline added
-///
-/// Note: this exists to differentiate between intentional writing to stdout
-/// and stray printlns left by accident
-#[macro_export]
-macro_rules! outln {
-    ($($tokens:tt)*) => {
-        {
-            use std::io::Write;
-            writeln!(std::io::stdout(), $($tokens)*).unwrap_or(())
-        }
-    }
-}
-
-/// Outputs to standard error
-///
-/// Note: this exists to differentiate between intentional writing to stdout
-/// and stray printlns left by accident
-#[macro_export]
-macro_rules! errln {
-    ($($tokens:tt)*) => {
-        {
-            use std::io::Write;
-            writeln!(std::io::stderr(), $($tokens)*).unwrap_or(())
-        }
-    }
-}
-
-#[macro_export]
-macro_rules! row {
-    ($( $key: expr => $val: expr ),*) => {{
-         let mut map = ::indexmap::IndexMap::new();
-         $( map.insert($key, $val); )*
-         ::nu_protocol::UntaggedValue::row(map).into_untagged_value()
-    }}
-}

crates/nu-protocol/src/maybe_owned.rs

@@ -1,18 +0,0 @@
-#![allow(clippy::should_implement_trait)]
-
-/// Helper type to allow passing something that may potentially be owned, but could also be borrowed
-#[derive(Debug)]
-pub enum MaybeOwned<'a, T> {
-    Owned(T),
-    Borrowed(&'a T),
-}
-
-impl<T> MaybeOwned<'_, T> {
-    /// Allows the borrowing of an owned value or passes out the borrowed value
-    pub fn borrow(&self) -> &T {
-        match self {
-            MaybeOwned::Owned(v) => v,
-            MaybeOwned::Borrowed(v) => v,
-        }
-    }
-}

crates/nu-protocol/src/overlay.rs

@@ -0,0 +1,131 @@
+use crate::{BlockId, DeclId, Span};
+
+use indexmap::IndexMap;
+
+// TODO: Move the import pattern matching logic here from use/hide commands and
+// parse_use/parse_hide
+
+/// Collection of definitions that can be exported from a module
+#[derive(Debug, Clone)]
+pub struct Overlay {
+    pub decls: IndexMap<Vec<u8>, DeclId>,
+    pub env_vars: IndexMap<Vec<u8>, BlockId>,
+    pub span: Option<Span>,
+}
+
+impl Overlay {
+    pub fn new() -> Self {
+        Overlay {
+            decls: IndexMap::new(),
+            env_vars: IndexMap::new(),
+            span: None,
+        }
+    }
+
+    pub fn from_span(span: Span) -> Self {
+        Overlay {
+            decls: IndexMap::new(),
+            env_vars: IndexMap::new(),
+            span: Some(span),
+        }
+    }
+
+    pub fn add_decl(&mut self, name: &[u8], decl_id: DeclId) -> Option<DeclId> {
+        self.decls.insert(name.to_vec(), decl_id)
+    }
+
+    pub fn add_env_var(&mut self, name: &[u8], block_id: BlockId) -> Option<BlockId> {
+        self.env_vars.insert(name.to_vec(), block_id)
+    }
+
+    pub fn extend(&mut self, other: &Overlay) {
+        self.decls.extend(other.decls.clone());
+        self.env_vars.extend(other.env_vars.clone());
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.decls.is_empty() && self.env_vars.is_empty()
+    }
+
+    pub fn get_decl_id(&self, name: &[u8]) -> Option<DeclId> {
+        self.decls.get(name).copied()
+    }
+
+    pub fn has_decl(&self, name: &[u8]) -> bool {
+        self.decls.contains_key(name)
+    }
+
+    pub fn decl_with_head(&self, name: &[u8], head: &[u8]) -> Option<(Vec<u8>, DeclId)> {
+        if let Some(id) = self.get_decl_id(name) {
+            let mut new_name = head.to_vec();
+            new_name.push(b' ');
+            new_name.extend(name);
+            Some((new_name, id))
+        } else {
+            None
+        }
+    }
+
+    pub fn decls_with_head(&self, head: &[u8]) -> Vec<(Vec<u8>, DeclId)> {
+        self.decls
+            .iter()
+            .map(|(name, id)| {
+                let mut new_name = head.to_vec();
+                new_name.push(b' ');
+                new_name.extend(name);
+                (new_name, *id)
+            })
+            .collect()
+    }
+
+    pub fn decls(&self) -> Vec<(Vec<u8>, DeclId)> {
+        self.decls
+            .iter()
+            .map(|(name, id)| (name.clone(), *id))
+            .collect()
+    }
+
+    pub fn get_env_var_id(&self, name: &[u8]) -> Option<BlockId> {
+        self.env_vars.get(name).copied()
+    }
+
+    pub fn has_env_var(&self, name: &[u8]) -> bool {
+        self.env_vars.contains_key(name)
+    }
+
+    pub fn env_var_with_head(&self, name: &[u8], head: &[u8]) -> Option<(Vec<u8>, BlockId)> {
+        if let Some(id) = self.get_env_var_id(name) {
+            let mut new_name = head.to_vec();
+            new_name.push(b' ');
+            new_name.extend(name);
+            Some((new_name, id))
+        } else {
+            None
+        }
+    }
+
+    pub fn env_vars_with_head(&self, head: &[u8]) -> Vec<(Vec<u8>, BlockId)> {
+        self.env_vars
+            .iter()
+            .map(|(name, id)| {
+                let mut new_name = head.to_vec();
+                new_name.push(b' ');
+                new_name.extend(name);
+                (new_name, *id)
+            })
+            .collect()
+    }
+
+    pub fn env_vars(&self) -> Vec<(Vec<u8>, BlockId)> {
+        self.env_vars
+            .iter()
+            .map(|(name, id)| (name.clone(), *id))
+            .collect()
+    }
+}
+
+impl Default for Overlay {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/nu-protocol/src/pipeline_data.rs

@@ -0,0 +1,470 @@
+use std::sync::{atomic::AtomicBool, Arc};
+
+use crate::{ast::PathMember, Config, ListStream, RawStream, ShellError, Span, Value};
+
+/// The foundational abstraction for input and output to commands
+///
+/// This represents either a single Value or a stream of values coming into the command or leaving a command.
+///
+/// A note on implementation:
+///
+/// We've tried a few variations of this structure. Listing these below so we have a record.
+///
+/// * We tried always assuming a stream in Nushell. This was a great 80% solution, but it had some rough edges.
+/// Namely, how do you know the difference between a single string and a list of one string. How do you know
+/// when to flatten the data given to you from a data source into the stream or to keep it as an unflattened
+/// list?
+///
+/// * We tried putting the stream into Value. This had some interesting properties as now commands "just worked
+/// on values", but lead to a few unfortunate issues.
+///
+/// The first is that you can't easily clone Values in a way that felt largely immutable. For example, if
+/// you cloned a Value which contained a stream, and in one variable drained some part of it, then the second
+/// variable would see different values based on what you did to the first.
+///
+/// To make this kind of mutation thread-safe, we would have had to produce a lock for the stream, which in
+/// practice would have meant always locking the stream before reading from it. But more fundamentally, it
+/// felt wrong in practice that observation of a value at runtime could affect other values which happen to
+/// alias the same stream. By separating these, we don't have this effect. Instead, variables could get
+/// concrete list values rather than streams, and be able to view them without non-local effects.
+///
+/// * A balance of the two approaches is what we've landed on: Values are thread-safe to pass, and we can stream
+/// them into any sources. Streams are still available to model the infinite streams approach of original
+/// Nushell.
+#[derive(Debug)]
+pub enum PipelineData {
+    Value(Value, Option<PipelineMetadata>),
+    ListStream(ListStream, Option<PipelineMetadata>),
+    RawStream(RawStream, Span, Option<PipelineMetadata>),
+}
+
+#[derive(Debug, Clone)]
+pub struct PipelineMetadata {
+    pub data_source: DataSource,
+}
+
+#[derive(Debug, Clone)]
+pub enum DataSource {
+    Ls,
+}
+
+impl PipelineData {
+    pub fn new(span: Span) -> PipelineData {
+        PipelineData::Value(Value::Nothing { span }, None)
+    }
+
+    pub fn new_with_metadata(metadata: Option<PipelineMetadata>, span: Span) -> PipelineData {
+        PipelineData::Value(Value::Nothing { span }, metadata)
+    }
+
+    pub fn metadata(&self) -> Option<PipelineMetadata> {
+        match self {
+            PipelineData::ListStream(_, x) => x.clone(),
+            PipelineData::RawStream(_, _, x) => x.clone(),
+            PipelineData::Value(_, x) => x.clone(),
+        }
+    }
+
+    pub fn set_metadata(mut self, metadata: Option<PipelineMetadata>) -> Self {
+        match &mut self {
+            PipelineData::ListStream(_, x) => *x = metadata,
+            PipelineData::RawStream(_, _, x) => *x = metadata,
+            PipelineData::Value(_, x) => *x = metadata,
+        }
+
+        self
+    }
+
+    pub fn is_nothing(&self) -> bool {
+        matches!(self, PipelineData::Value(Value::Nothing { .. }, ..))
+    }
+
+    pub fn into_value(self, span: Span) -> Value {
+        match self {
+            PipelineData::Value(Value::Nothing { .. }, ..) => Value::nothing(span),
+            PipelineData::Value(v, ..) => v,
+            PipelineData::ListStream(s, ..) => Value::List {
+                vals: s.collect(),
+                span, // FIXME?
+            },
+            PipelineData::RawStream(mut s, ..) => {
+                let mut items = vec![];
+
+                for val in &mut s {
+                    match val {
+                        Ok(val) => {
+                            items.push(val);
+                        }
+                        Err(e) => {
+                            return Value::Error { error: e };
+                        }
+                    }
+                }
+
+                if s.is_binary {
+                    let mut output = vec![];
+                    for item in items {
+                        match item.as_binary() {
+                            Ok(item) => {
+                                output.extend(item);
+                            }
+                            Err(err) => {
+                                return Value::Error { error: err };
+                            }
+                        }
+                    }
+
+                    Value::Binary {
+                        val: output,
+                        span, // FIXME?
+                    }
+                } else {
+                    let mut output = String::new();
+                    for item in items {
+                        match item.as_string() {
+                            Ok(s) => output.push_str(&s),
+                            Err(err) => {
+                                return Value::Error { error: err };
+                            }
+                        }
+                    }
+                    Value::String {
+                        val: output,
+                        span, // FIXME?
+                    }
+                }
+            }
+        }
+    }
+
+    pub fn into_interruptible_iter(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineIterator {
+        let mut iter = self.into_iter();
+
+        if let PipelineIterator(PipelineData::ListStream(s, ..)) = &mut iter {
+            s.ctrlc = ctrlc;
+        }
+
+        iter
+    }
+
+    pub fn collect_string(self, separator: &str, config: &Config) -> Result<String, ShellError> {
+        match self {
+            PipelineData::Value(v, ..) => Ok(v.into_string(separator, config)),
+            PipelineData::ListStream(s, ..) => Ok(s.into_string(separator, config)),
+            PipelineData::RawStream(s, ..) => {
+                let mut items = vec![];
+
+                for val in s {
+                    match val {
+                        Ok(val) => {
+                            items.push(val);
+                        }
+                        Err(e) => {
+                            return Err(e);
+                        }
+                    }
+                }
+
+                let mut output = String::new();
+                for item in items {
+                    match item.as_string() {
+                        Ok(s) => output.push_str(&s),
+                        Err(err) => {
+                            return Err(err);
+                        }
+                    }
+                }
+
+                Ok(output)
+            }
+        }
+    }
+
+    pub fn follow_cell_path(
+        self,
+        cell_path: &[PathMember],
+        head: Span,
+    ) -> Result<Value, ShellError> {
+        match self {
+            // FIXME: there are probably better ways of doing this
+            PipelineData::ListStream(stream, ..) => Value::List {
+                vals: stream.collect(),
+                span: head,
+            }
+            .follow_cell_path(cell_path),
+            PipelineData::Value(v, ..) => v.follow_cell_path(cell_path),
+            _ => Err(ShellError::IOError("can't follow stream paths".into())),
+        }
+    }
+
+    pub fn update_cell_path(
+        &mut self,
+        cell_path: &[PathMember],
+        callback: Box<dyn FnOnce(&Value) -> Value>,
+        head: Span,
+    ) -> Result<(), ShellError> {
+        match self {
+            // FIXME: there are probably better ways of doing this
+            PipelineData::ListStream(stream, ..) => Value::List {
+                vals: stream.collect(),
+                span: head,
+            }
+            .update_cell_path(cell_path, callback),
+            PipelineData::Value(v, ..) => v.update_cell_path(cell_path, callback),
+            _ => Ok(()),
+        }
+    }
+
+    /// Simplified mapper to help with simple values also. For full iterator support use `.into_iter()` instead
+    pub fn map<F>(
+        self,
+        mut f: F,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> Result<PipelineData, ShellError>
+    where
+        Self: Sized,
+        F: FnMut(Value) -> Value + 'static + Send,
+    {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }, ..) => {
+                Ok(vals.into_iter().map(f).into_pipeline_data(ctrlc))
+            }
+            PipelineData::ListStream(stream, ..) => Ok(stream.map(f).into_pipeline_data(ctrlc)),
+            PipelineData::RawStream(stream, ..) => {
+                let collected = stream.into_bytes()?;
+
+                if let Ok(st) = String::from_utf8(collected.clone().item) {
+                    Ok(f(Value::String {
+                        val: st,
+                        span: collected.span,
+                    })
+                    .into_pipeline_data())
+                } else {
+                    Ok(f(Value::Binary {
+                        val: collected.item,
+                        span: collected.span,
+                    })
+                    .into_pipeline_data())
+                }
+            }
+
+            PipelineData::Value(Value::Range { val, .. }, ..) => {
+                Ok(val.into_range_iter()?.map(f).into_pipeline_data(ctrlc))
+            }
+            PipelineData::Value(v, ..) => match f(v) {
+                Value::Error { error } => Err(error),
+                v => Ok(v.into_pipeline_data()),
+            },
+        }
+    }
+
+    /// Simplified flatmapper. For full iterator support use `.into_iter()` instead
+    pub fn flat_map<U, F>(
+        self,
+        mut f: F,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> Result<PipelineData, ShellError>
+    where
+        Self: Sized,
+        U: IntoIterator<Item = Value>,
+        <U as IntoIterator>::IntoIter: 'static + Send,
+        F: FnMut(Value) -> U + 'static + Send,
+    {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }, ..) => {
+                Ok(vals.into_iter().map(f).flatten().into_pipeline_data(ctrlc))
+            }
+            PipelineData::ListStream(stream, ..) => {
+                Ok(stream.map(f).flatten().into_pipeline_data(ctrlc))
+            }
+            PipelineData::RawStream(stream, ..) => {
+                let collected = stream.into_bytes()?;
+
+                if let Ok(st) = String::from_utf8(collected.clone().item) {
+                    Ok(f(Value::String {
+                        val: st,
+                        span: collected.span,
+                    })
+                    .into_iter()
+                    .into_pipeline_data(ctrlc))
+                } else {
+                    Ok(f(Value::Binary {
+                        val: collected.item,
+                        span: collected.span,
+                    })
+                    .into_iter()
+                    .into_pipeline_data(ctrlc))
+                }
+            }
+            PipelineData::Value(Value::Range { val, .. }, ..) => match val.into_range_iter() {
+                Ok(iter) => Ok(iter.map(f).flatten().into_pipeline_data(ctrlc)),
+                Err(error) => Err(error),
+            },
+            PipelineData::Value(v, ..) => Ok(f(v).into_iter().into_pipeline_data(ctrlc)),
+        }
+    }
+
+    pub fn filter<F>(
+        self,
+        mut f: F,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> Result<PipelineData, ShellError>
+    where
+        Self: Sized,
+        F: FnMut(&Value) -> bool + 'static + Send,
+    {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }, ..) => {
+                Ok(vals.into_iter().filter(f).into_pipeline_data(ctrlc))
+            }
+            PipelineData::ListStream(stream, ..) => Ok(stream.filter(f).into_pipeline_data(ctrlc)),
+            PipelineData::RawStream(stream, ..) => {
+                let collected = stream.into_bytes()?;
+
+                if let Ok(st) = String::from_utf8(collected.clone().item) {
+                    let v = Value::String {
+                        val: st,
+                        span: collected.span,
+                    };
+
+                    if f(&v) {
+                        Ok(v.into_pipeline_data())
+                    } else {
+                        Ok(PipelineData::new(collected.span))
+                    }
+                } else {
+                    let v = Value::Binary {
+                        val: collected.item,
+                        span: collected.span,
+                    };
+
+                    if f(&v) {
+                        Ok(v.into_pipeline_data())
+                    } else {
+                        Ok(PipelineData::new(collected.span))
+                    }
+                }
+            }
+            PipelineData::Value(Value::Range { val, .. }, ..) => {
+                Ok(val.into_range_iter()?.filter(f).into_pipeline_data(ctrlc))
+            }
+            PipelineData::Value(v, ..) => {
+                if f(&v) {
+                    Ok(v.into_pipeline_data())
+                } else {
+                    Ok(Value::Nothing { span: v.span()? }.into_pipeline_data())
+                }
+            }
+        }
+    }
+}
+
+pub struct PipelineIterator(PipelineData);
+
+impl IntoIterator for PipelineData {
+    type Item = Value;
+
+    type IntoIter = PipelineIterator;
+
+    fn into_iter(self) -> Self::IntoIter {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }, metadata) => {
+                PipelineIterator(PipelineData::ListStream(
+                    ListStream {
+                        stream: Box::new(vals.into_iter()),
+                        ctrlc: None,
+                    },
+                    metadata,
+                ))
+            }
+            PipelineData::Value(Value::Range { val, .. }, metadata) => {
+                match val.into_range_iter() {
+                    Ok(iter) => PipelineIterator(PipelineData::ListStream(
+                        ListStream {
+                            stream: Box::new(iter),
+                            ctrlc: None,
+                        },
+                        metadata,
+                    )),
+                    Err(error) => PipelineIterator(PipelineData::ListStream(
+                        ListStream {
+                            stream: Box::new(std::iter::once(Value::Error { error })),
+                            ctrlc: None,
+                        },
+                        metadata,
+                    )),
+                }
+            }
+            x => PipelineIterator(x),
+        }
+    }
+}
+
+impl Iterator for PipelineIterator {
+    type Item = Value;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match &mut self.0 {
+            PipelineData::Value(Value::Nothing { .. }, ..) => None,
+            PipelineData::Value(v, ..) => Some(std::mem::take(v)),
+            PipelineData::ListStream(stream, ..) => stream.next(),
+            PipelineData::RawStream(stream, ..) => stream.next().map(|x| match x {
+                Ok(x) => x,
+                Err(err) => Value::Error { error: err },
+            }),
+        }
+    }
+}
+
+pub trait IntoPipelineData {
+    fn into_pipeline_data(self) -> PipelineData;
+}
+
+impl<V> IntoPipelineData for V
+where
+    V: Into<Value>,
+{
+    fn into_pipeline_data(self) -> PipelineData {
+        PipelineData::Value(self.into(), None)
+    }
+}
+
+pub trait IntoInterruptiblePipelineData {
+    fn into_pipeline_data(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineData;
+    fn into_pipeline_data_with_metadata(
+        self,
+        metadata: PipelineMetadata,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> PipelineData;
+}
+
+impl<I> IntoInterruptiblePipelineData for I
+where
+    I: IntoIterator + Send + 'static,
+    I::IntoIter: Send + 'static,
+    <I::IntoIter as Iterator>::Item: Into<Value>,
+{
+    fn into_pipeline_data(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineData {
+        PipelineData::ListStream(
+            ListStream {
+                stream: Box::new(self.into_iter().map(Into::into)),
+                ctrlc,
+            },
+            None,
+        )
+    }
+
+    fn into_pipeline_data_with_metadata(
+        self,
+        metadata: PipelineMetadata,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> PipelineData {
+        PipelineData::ListStream(
+            ListStream {
+                stream: Box::new(self.into_iter().map(Into::into)),
+                ctrlc,
+            },
+            Some(metadata),
+        )
+    }
+}

crates/nu-protocol/src/registry.rs

@@ -1,31 +0,0 @@
-use crate::{Signature, Value};
-use nu_source::Spanned;
-use std::fmt::Debug;
-
-pub trait VariableRegistry {
-    fn get_variable(&self, name: &Spanned<&str>) -> Option<Value>;
-    fn variables(&self) -> Vec<String>;
-}
-
-pub trait SignatureRegistry: Debug {
-    fn names(&self) -> Vec<String>;
-    fn has(&self, name: &str) -> bool;
-    fn get(&self, name: &str) -> Option<Signature>;
-    fn clone_box(&self) -> Box<dyn SignatureRegistry>;
-}
-
-impl SignatureRegistry for Box<dyn SignatureRegistry> {
-    fn names(&self) -> Vec<String> {
-        (&**self).names()
-    }
-
-    fn has(&self, name: &str) -> bool {
-        (&**self).has(name)
-    }
-    fn get(&self, name: &str) -> Option<Signature> {
-        (&**self).get(name)
-    }
-    fn clone_box(&self) -> Box<dyn SignatureRegistry> {
-        (&**self).clone_box()
-    }
-}

crates/nu-protocol/src/return_value.rs

@@ -1,140 +0,0 @@
-use crate::{value::Value, ConfigPath};
-use nu_errors::ShellError;
-use nu_source::{DbgDocBldr, DebugDocBuilder, PrettyDebug};
-use serde::{Deserialize, Serialize};
-
-/// The inner set of actions for the command processor. Each denotes a way to change state in the processor without changing it directly from the command itself.
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum CommandAction {
-    /// Change to a new directory or path (in non-filesystem situations)
-    ChangePath(String),
-    /// Exit out of Nu
-    Exit(i32),
-    /// Display an error
-    Error(ShellError),
-    /// Enter a new shell at the given path
-    EnterShell(String),
-    /// Convert the value given from one type to another
-    AutoConvert(Value, String),
-    /// Enter a value shell, one that allows exploring inside of a Value
-    EnterValueShell(Value),
-    /// Add plugins from path given
-    AddPlugins(String),
-    /// Unload the config specified by PathBuf if present
-    UnloadConfig(ConfigPath),
-    /// Load the config specified by PathBuf
-    LoadConfig(ConfigPath),
-    /// Go to the previous shell in the shell ring buffer
-    PreviousShell,
-    /// Go to the next shell in the shell ring buffer
-    NextShell,
-    /// Jump to the specified shell in the shell ring buffer
-    GotoShell(usize),
-    /// Leave the current shell. If it's the last shell, exit out of Nu
-    LeaveShell(i32),
-}
-
-impl PrettyDebug for CommandAction {
-    /// Get a command action ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            CommandAction::ChangePath(path) => {
-                DbgDocBldr::typed("change path", DbgDocBldr::description(path))
-            }
-            CommandAction::Exit(_) => DbgDocBldr::description("exit"),
-            CommandAction::Error(_) => DbgDocBldr::error("error"),
-            CommandAction::AutoConvert(_, extension) => {
-                DbgDocBldr::typed("auto convert", DbgDocBldr::description(extension))
-            }
-            CommandAction::EnterShell(s) => {
-                DbgDocBldr::typed("enter shell", DbgDocBldr::description(s))
-            }
-            CommandAction::EnterValueShell(v) => DbgDocBldr::typed("enter value shell", v.pretty()),
-            CommandAction::AddPlugins(..) => DbgDocBldr::description("add plugins"),
-            CommandAction::PreviousShell => DbgDocBldr::description("previous shell"),
-            CommandAction::NextShell => DbgDocBldr::description("next shell"),
-            CommandAction::GotoShell(_) => DbgDocBldr::description("goto shell"),
-            CommandAction::LeaveShell(_) => DbgDocBldr::description("leave shell"),
-            CommandAction::UnloadConfig(cfg) => {
-                DbgDocBldr::description(format!("unload config {:?}", cfg))
-            }
-            CommandAction::LoadConfig(cfg) => {
-                DbgDocBldr::description(format!("load config {:?}", cfg))
-            }
-        }
-    }
-}
-
-/// The fundamental success type in the pipeline. Commands return these values as their main responsibility
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum ReturnSuccess {
-    /// A value to be used or shown to the user
-    Value(Value),
-    /// A debug-enabled value to be used or shown to the user
-    DebugValue(Value),
-    /// An action to be performed as values pass out of the command. These are performed rather than passed to the next command in the pipeline
-    Action(CommandAction),
-}
-
-impl PrettyDebug for ReturnSuccess {
-    /// Get a return success ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            ReturnSuccess::Value(value) => DbgDocBldr::typed("value", value.pretty()),
-            ReturnSuccess::DebugValue(value) => DbgDocBldr::typed("debug value", value.pretty()),
-            ReturnSuccess::Action(action) => DbgDocBldr::typed("action", action.pretty()),
-        }
-    }
-}
-
-/// The core Result type for pipelines
-pub type ReturnValue = Result<ReturnSuccess, ShellError>;
-
-impl From<Value> for ReturnValue {
-    fn from(v: Value) -> Self {
-        Ok(ReturnSuccess::Value(v))
-    }
-}
-
-impl ReturnSuccess {
-    /// Get to the contained Value, if possible
-    pub fn raw_value(&self) -> Option<Value> {
-        match self {
-            ReturnSuccess::Value(raw) => Some(raw.clone()),
-            ReturnSuccess::DebugValue(raw) => Some(raw.clone()),
-            ReturnSuccess::Action(_) => None,
-        }
-    }
-
-    /// Helper function for an action to change the the path
-    pub fn change_cwd(path: String) -> ReturnValue {
-        Ok(ReturnSuccess::Action(CommandAction::ChangePath(path)))
-    }
-
-    /// Helper function to create simple values for returning
-    pub fn value(input: impl Into<Value>) -> ReturnValue {
-        Ok(ReturnSuccess::Value(input.into()))
-    }
-
-    /// Helper function to create simple debug-enabled values for returning
-    pub fn debug_value(input: impl Into<Value>) -> ReturnValue {
-        Ok(ReturnSuccess::DebugValue(input.into()))
-    }
-
-    /// Helper function for creating actions
-    pub fn action(input: CommandAction) -> ReturnValue {
-        Ok(ReturnSuccess::Action(input))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::{ReturnSuccess, ReturnValue, UntaggedValue};
-
-    #[test]
-    fn return_value_can_be_used_in_assert_eq() {
-        let v1: ReturnValue = ReturnSuccess::value(UntaggedValue::nothing());
-        let v2: ReturnValue = ReturnSuccess::value(UntaggedValue::nothing());
-        assert_eq!(v1, v2);
-    }
-}

crates/nu-protocol/src/shell_error.rs

@@ -0,0 +1,367 @@
+use miette::Diagnostic;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{ast::Operator, Span, Type};
+
+/// The fundamental error type for the evaluation engine. These cases represent different kinds of errors
+/// the evaluator might face, along with helpful spans to label. An error renderer will take this error value
+/// and pass it into an error viewer to display to the user.
+#[derive(Debug, Clone, Error, Diagnostic, Serialize, Deserialize)]
+pub enum ShellError {
+    #[error("Type mismatch during operation.")]
+    #[diagnostic(code(nu::shell::type_mismatch), url(docsrs))]
+    OperatorMismatch {
+        #[label = "type mismatch for operator"]
+        op_span: Span,
+        lhs_ty: Type,
+        #[label("{lhs_ty}")]
+        lhs_span: Span,
+        rhs_ty: Type,
+        #[label("{rhs_ty}")]
+        rhs_span: Span,
+    },
+
+    #[error("Operator overflow.")]
+    #[diagnostic(code(nu::shell::operator_overflow), url(docsrs))]
+    OperatorOverflow(String, #[label = "{0}"] Span),
+
+    #[error("Pipeline mismatch.")]
+    #[diagnostic(code(nu::shell::pipeline_mismatch), url(docsrs))]
+    PipelineMismatch(
+        String,
+        #[label("expected: {0}")] Span,
+        #[label("value originates from here")] Span,
+    ),
+
+    #[error("Type mismatch")]
+    #[diagnostic(code(nu::shell::type_mismatch), url(docsrs))]
+    TypeMismatch(String, #[label = "{0}"] Span),
+
+    #[error("Unsupported operator: {0}.")]
+    #[diagnostic(code(nu::shell::unsupported_operator), url(docsrs))]
+    UnsupportedOperator(Operator, #[label = "unsupported operator"] Span),
+
+    #[error("Unsupported operator: {0}.")]
+    #[diagnostic(code(nu::shell::unknown_operator), url(docsrs))]
+    UnknownOperator(String, #[label = "unsupported operator"] Span),
+
+    #[error("Missing parameter: {0}.")]
+    #[diagnostic(code(nu::shell::missing_parameter), url(docsrs))]
+    MissingParameter(String, #[label = "missing parameter: {0}"] Span),
+
+    // Be cautious, as flags can share the same span, resulting in a panic (ex: `rm -pt`)
+    #[error("Incompatible parameters.")]
+    #[diagnostic(code(nu::shell::incompatible_parameters), url(docsrs))]
+    IncompatibleParameters {
+        left_message: String,
+        #[label("{left_message}")]
+        left_span: Span,
+        right_message: String,
+        #[label("{right_message}")]
+        right_span: Span,
+    },
+
+    #[error("Delimiter error")]
+    #[diagnostic(code(nu::shell::delimiter_error), url(docsrs))]
+    DelimiterError(String, #[label("{0}")] Span),
+
+    #[error("Incompatible parameters.")]
+    #[diagnostic(code(nu::shell::incompatible_parameters), url(docsrs))]
+    IncompatibleParametersSingle(String, #[label = "{0}"] Span),
+
+    #[error("Feature not enabled.")]
+    #[diagnostic(code(nu::shell::feature_not_enabled), url(docsrs))]
+    FeatureNotEnabled(#[label = "feature not enabled"] Span),
+
+    #[error("External commands not yet supported")]
+    #[diagnostic(code(nu::shell::external_commands), url(docsrs))]
+    ExternalNotSupported(#[label = "external not supported"] Span),
+
+    #[error("Invalid Probability.")]
+    #[diagnostic(code(nu::shell::invalid_probability), url(docsrs))]
+    InvalidProbability(#[label = "invalid probability"] Span),
+
+    #[error("Invalid range {0}..{1}")]
+    #[diagnostic(code(nu::shell::invalid_range), url(docsrs))]
+    InvalidRange(String, String, #[label = "expected a valid range"] Span),
+
+    // Only use this one if we Nushell completely falls over and hits a state that isn't possible or isn't recoverable
+    #[error("Nushell failed: {0}.")]
+    #[diagnostic(code(nu::shell::nushell_failed), url(docsrs))]
+    NushellFailed(String),
+
+    // Only use this one if we Nushell completely falls over and hits a state that isn't possible or isn't recoverable
+    #[error("Nushell failed: {0}.")]
+    #[diagnostic(code(nu::shell::nushell_failed), url(docsrs))]
+    NushellFailedSpanned(String, String, #[label = "{1}"] Span),
+
+    #[error("Variable not found")]
+    #[diagnostic(code(nu::shell::variable_not_found), url(docsrs))]
+    VariableNotFoundAtRuntime(#[label = "variable not found"] Span),
+
+    #[error("Environment variable '{0}' not found")]
+    #[diagnostic(code(nu::shell::env_variable_not_found), url(docsrs))]
+    EnvVarNotFoundAtRuntime(String, #[label = "environment variable not found"] Span),
+
+    #[error("Not found.")]
+    #[diagnostic(code(nu::parser::not_found), url(docsrs))]
+    NotFound(#[label = "did not find anything under this name"] Span),
+
+    #[error("Can't convert to {0}.")]
+    #[diagnostic(code(nu::shell::cant_convert), url(docsrs))]
+    CantConvert(String, String, #[label("can't convert {1} to {0}")] Span),
+
+    #[error("Division by zero.")]
+    #[diagnostic(code(nu::shell::division_by_zero), url(docsrs))]
+    DivisionByZero(#[label("division by zero")] Span),
+
+    #[error("Can't convert range to countable values")]
+    #[diagnostic(code(nu::shell::range_to_countable), url(docsrs))]
+    CannotCreateRange(#[label = "can't convert to countable values"] Span),
+
+    #[error("Row number too large (max: {0}).")]
+    #[diagnostic(code(nu::shell::access_beyond_end), url(docsrs))]
+    AccessBeyondEnd(usize, #[label = "too large"] Span),
+
+    #[error("Row number too large.")]
+    #[diagnostic(code(nu::shell::access_beyond_end_of_stream), url(docsrs))]
+    AccessBeyondEndOfStream(#[label = "too large"] Span),
+
+    #[error("Data cannot be accessed with a cell path")]
+    #[diagnostic(code(nu::shell::incompatible_path_access), url(docsrs))]
+    IncompatiblePathAccess(String, #[label("{0} doesn't support cell paths")] Span),
+
+    #[error("Cannot find column")]
+    #[diagnostic(code(nu::shell::column_not_found), url(docsrs))]
+    CantFindColumn(
+        #[label = "cannot find column"] Span,
+        #[label = "value originates here"] Span,
+    ),
+
+    #[error("Not a list value")]
+    #[diagnostic(code(nu::shell::not_a_list), url(docsrs))]
+    NotAList(
+        #[label = "value not a list"] Span,
+        #[label = "value originates here"] Span,
+    ),
+
+    #[error("External command")]
+    #[diagnostic(code(nu::shell::external_command), url(docsrs), help("{1}"))]
+    ExternalCommand(String, String, #[label("{0}")] Span),
+
+    #[error("Unsupported input")]
+    #[diagnostic(code(nu::shell::unsupported_input), url(docsrs))]
+    UnsupportedInput(String, #[label("{0}")] Span),
+
+    #[error("Network failure")]
+    #[diagnostic(code(nu::shell::network_failure), url(docsrs))]
+    NetworkFailure(String, #[label("{0}")] Span),
+
+    #[error("Command not found")]
+    #[diagnostic(code(nu::shell::command_not_found), url(docsrs))]
+    CommandNotFound(#[label("command not found")] Span),
+
+    #[error("Flag not found")]
+    #[diagnostic(code(nu::shell::flag_not_found), url(docsrs))]
+    FlagNotFound(String, #[label("{0} not found")] Span),
+
+    #[error("File not found")]
+    #[diagnostic(code(nu::shell::file_not_found), url(docsrs))]
+    FileNotFound(#[label("file not found")] Span),
+
+    #[error("File not found")]
+    #[diagnostic(code(nu::shell::file_not_found), url(docsrs))]
+    FileNotFoundCustom(String, #[label("{0}")] Span),
+
+    #[error("Plugin failed to load: {0}")]
+    #[diagnostic(code(nu::shell::plugin_failed_to_load), url(docsrs))]
+    PluginFailedToLoad(String),
+
+    #[error("Plugin failed to encode: {0}")]
+    #[diagnostic(code(nu::shell::plugin_failed_to_encode), url(docsrs))]
+    PluginFailedToEncode(String),
+
+    #[error("Plugin failed to decode: {0}")]
+    #[diagnostic(code(nu::shell::plugin_failed_to_decode), url(docsrs))]
+    PluginFailedToDecode(String),
+
+    #[error("I/O error")]
+    #[diagnostic(code(nu::shell::io_error), url(docsrs), help("{0}"))]
+    IOError(String),
+
+    #[error("Cannot change to directory")]
+    #[diagnostic(code(nu::shell::cannot_cd_to_directory), url(docsrs))]
+    NotADirectory(#[label("is not a directory")] Span),
+
+    #[error("Directory not found")]
+    #[diagnostic(code(nu::shell::directory_not_found), url(docsrs))]
+    DirectoryNotFound(#[label("directory not found")] Span),
+
+    #[error("Directory not found")]
+    #[diagnostic(code(nu::shell::directory_not_found_custom), url(docsrs))]
+    DirectoryNotFoundCustom(String, #[label("{0}")] Span),
+
+    #[error("Directory not found")]
+    #[diagnostic(code(nu::shell::directory_not_found_help), url(docsrs), help("{1}"))]
+    DirectoryNotFoundHelp(#[label("directory not found")] Span, String),
+
+    #[error("Move not possible")]
+    #[diagnostic(code(nu::shell::move_not_possible), url(docsrs))]
+    MoveNotPossible {
+        source_message: String,
+        #[label("{source_message}")]
+        source_span: Span,
+        destination_message: String,
+        #[label("{destination_message}")]
+        destination_span: Span,
+    },
+
+    #[error("Move not possible")]
+    #[diagnostic(code(nu::shell::move_not_possible_single), url(docsrs))]
+    MoveNotPossibleSingle(String, #[label("{0}")] Span),
+
+    #[error("Create not possible")]
+    #[diagnostic(code(nu::shell::create_not_possible), url(docsrs))]
+    CreateNotPossible(String, #[label("{0}")] Span),
+
+    #[error("Remove not possible")]
+    #[diagnostic(code(nu::shell::remove_not_possible), url(docsrs))]
+    RemoveNotPossible(String, #[label("{0}")] Span),
+
+    #[error("No file to be removed")]
+    NoFileToBeRemoved(),
+    #[error("No file to be moved")]
+    NoFileToBeMoved(),
+    #[error("No file to be copied")]
+    NoFileToBeCopied(),
+
+    #[error("Name not found")]
+    #[diagnostic(code(nu::shell::name_not_found), url(docsrs))]
+    DidYouMean(String, #[label("did you mean '{0}'?")] Span),
+
+    #[error("Non-UTF8 string")]
+    #[diagnostic(code(nu::parser::non_utf8), url(docsrs))]
+    NonUtf8(#[label = "non-UTF8 string"] Span),
+
+    #[error("Casting error")]
+    #[diagnostic(code(nu::shell::downcast_not_possible), url(docsrs))]
+    DowncastNotPossible(String, #[label("{0}")] Span),
+
+    #[error("Unsupported config value")]
+    #[diagnostic(code(nu::shell::unsupported_config_value), url(docsrs))]
+    UnsupportedConfigValue(String, String, #[label = "expected {0}, got {1}"] Span),
+
+    #[error("Missing config value")]
+    #[diagnostic(code(nu::shell::missing_config_value), url(docsrs))]
+    MissingConfigValue(String, #[label = "missing {0}"] Span),
+
+    #[error("{0}")]
+    #[diagnostic()]
+    SpannedLabeledError(String, String, #[label("{1}")] Span),
+
+    #[error("{0}")]
+    #[diagnostic(help("{3}"))]
+    SpannedLabeledErrorHelp(String, String, #[label("{1}")] Span, String),
+
+    #[error("{0}")]
+    #[diagnostic(help("{1}"))]
+    LabeledError(String, String),
+}
+
+impl From<std::io::Error> for ShellError {
+    fn from(input: std::io::Error) -> ShellError {
+        ShellError::IOError(format!("{:?}", input))
+    }
+}
+
+impl std::convert::From<Box<dyn std::error::Error>> for ShellError {
+    fn from(input: Box<dyn std::error::Error>) -> ShellError {
+        ShellError::IOError(input.to_string())
+    }
+}
+
+impl From<Box<dyn std::error::Error + Send + Sync>> for ShellError {
+    fn from(input: Box<dyn std::error::Error + Send + Sync>) -> ShellError {
+        ShellError::IOError(format!("{:?}", input))
+    }
+}
+
+pub fn did_you_mean(possibilities: &[String], tried: &str) -> Option<String> {
+    let mut possible_matches: Vec<_> = possibilities
+        .iter()
+        .map(|word| {
+            let edit_distance = levenshtein_distance(word, tried);
+            (edit_distance, word.to_owned())
+        })
+        .collect();
+
+    possible_matches.sort();
+
+    if let Some((_, first)) = possible_matches.into_iter().next() {
+        Some(first)
+    } else {
+        None
+    }
+}
+
+// Borrowed from here https://github.com/wooorm/levenshtein-rs
+pub fn levenshtein_distance(a: &str, b: &str) -> usize {
+    let mut result = 0;
+
+    /* Shortcut optimizations / degenerate cases. */
+    if a == b {
+        return result;
+    }
+
+    let length_a = a.chars().count();
+    let length_b = b.chars().count();
+
+    if length_a == 0 {
+        return length_b;
+    }
+
+    if length_b == 0 {
+        return length_a;
+    }
+
+    /* Initialize the vector.
+     *
+     * This is why it’s fast, normally a matrix is used,
+     * here we use a single vector. */
+    let mut cache: Vec<usize> = (1..).take(length_a).collect();
+    let mut distance_a;
+    let mut distance_b;
+
+    /* Loop. */
+    for (index_b, code_b) in b.chars().enumerate() {
+        result = index_b;
+        distance_a = index_b;
+
+        for (index_a, code_a) in a.chars().enumerate() {
+            distance_b = if code_a == code_b {
+                distance_a
+            } else {
+                distance_a + 1
+            };
+
+            distance_a = cache[index_a];
+
+            result = if distance_a > result {
+                if distance_b > result {
+                    result + 1
+                } else {
+                    distance_b
+                }
+            } else if distance_b > distance_a {
+                distance_a + 1
+            } else {
+                distance_b
+            };
+
+            cache[index_a] = result;
+        }
+    }
+
+    result
+}

crates/nu-protocol/src/signature.rs

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 use crate::syntax_shape::SyntaxShape;
 use crate::type_shape::Type;
 use indexmap::IndexMap;
@@ -156,13 +157,118 @@ pub struct Signature {
     pub input: Option<Type>,
     /// If the command is expected to filter data, or to consume it (as a sink)
     pub is_filter: bool,
+=======
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::ast::Call;
+use crate::engine::Command;
+use crate::engine::EngineState;
+use crate::engine::Stack;
+use crate::BlockId;
+use crate::PipelineData;
+use crate::SyntaxShape;
+use crate::VarId;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Flag {
+    pub long: String,
+    pub short: Option<char>,
+    pub arg: Option<SyntaxShape>,
+    pub required: bool,
+    pub desc: String,
+    // For custom commands
+    pub var_id: Option<VarId>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PositionalArg {
+    pub name: String,
+    pub desc: String,
+    pub shape: SyntaxShape,
+    // For custom commands
+    pub var_id: Option<VarId>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum Category {
+    Default,
+    Conversions,
+    Core,
+    Date,
+    Env,
+    Experimental,
+    FileSystem,
+    Filters,
+    Formats,
+    Math,
+    Network,
+    Random,
+    Platform,
+    Shells,
+    Strings,
+    System,
+    Viewers,
+    Hash,
+    Generators,
+    Custom(String),
+}
+
+impl std::fmt::Display for Category {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let msg = match self {
+            Category::Default => "default",
+            Category::Conversions => "conversions",
+            Category::Core => "core",
+            Category::Date => "date",
+            Category::Env => "env",
+            Category::Experimental => "experimental",
+            Category::FileSystem => "filesystem",
+            Category::Filters => "filters",
+            Category::Formats => "formats",
+            Category::Math => "math",
+            Category::Network => "network",
+            Category::Random => "random",
+            Category::Platform => "platform",
+            Category::Shells => "shells",
+            Category::Strings => "strings",
+            Category::System => "system",
+            Category::Viewers => "viewers",
+            Category::Hash => "hash",
+            Category::Generators => "generators",
+            Category::Custom(name) => name,
+        };
+
+        write!(f, "{}", msg)
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Signature {
+    pub name: String,
+    pub usage: String,
+    pub extra_usage: String,
+    pub required_positional: Vec<PositionalArg>,
+    pub optional_positional: Vec<PositionalArg>,
+    pub rest_positional: Option<PositionalArg>,
+    pub named: Vec<Flag>,
+    pub is_filter: bool,
+    pub creates_scope: bool,
+    // Signature category used to classify commands stored in the list of declarations
+    pub category: Category,
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
 }
 
 impl PartialEq for Signature {
     fn eq(&self, other: &Self) -> bool {
         self.name == other.name
             && self.usage == other.usage
+<<<<<<< HEAD
             && self.positional == other.positional
+=======
+            && self.required_positional == other.required_positional
+            && self.optional_positional == other.optional_positional
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
             && self.rest_positional == other.rest_positional
             && self.is_filter == other.is_filter
     }
@@ -171,6 +277,7 @@ impl PartialEq for Signature {
 impl Eq for Signature {}
 
 impl Signature {
+<<<<<<< HEAD
     pub fn shift_positional(&mut self) {
         self.positional = Vec::from(&self.positional[1..]);
     }
@@ -224,10 +331,24 @@ impl PrettyDebugWithSource for Signature {
 impl Signature {
     /// Create a new command signature with the given name
     pub fn new(name: impl Into<String>) -> Signature {
+=======
+    pub fn new(name: impl Into<String>) -> Signature {
+        // default help flag
+        let flag = Flag {
+            long: "help".into(),
+            short: Some('h'),
+            arg: None,
+            desc: "Display this help message".into(),
+            required: false,
+            var_id: None,
+        };
+
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
         Signature {
             name: name.into(),
             usage: String::new(),
             extra_usage: String::new(),
+<<<<<<< HEAD
             positional: vec![],
             rest_positional: None,
             named: indexmap::indexmap! {"help".into() => (NamedType::Switch(Some('h')), "Display this help message".into())},
@@ -238,6 +359,17 @@ impl Signature {
     }
 
     /// Create a new signature
+=======
+            required_positional: vec![],
+            optional_positional: vec![],
+            rest_positional: None,
+            named: vec![flag],
+            is_filter: false,
+            creates_scope: false,
+            category: Category::Default,
+        }
+    }
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
     pub fn build(name: impl Into<String>) -> Signature {
         Signature::new(name.into())
     }
@@ -252,6 +384,7 @@ impl Signature {
     pub fn required(
         mut self,
         name: impl Into<String>,
+<<<<<<< HEAD
         ty: impl Into<SyntaxShape>,
         desc: impl Into<String>,
     ) -> Signature {
@@ -259,10 +392,22 @@ impl Signature {
             PositionalType::Mandatory(name.into(), ty.into()),
             desc.into(),
         ));
+=======
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.required_positional.push(PositionalArg {
+            name: name.into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
 
         self
     }
 
+<<<<<<< HEAD
     /// Add an optional positional argument to the signature
     pub fn optional(
         mut self,
@@ -274,6 +419,37 @@ impl Signature {
             PositionalType::Optional(name.into(), ty.into()),
             desc.into(),
         ));
+=======
+    /// Add a required positional argument to the signature
+    pub fn optional(
+        mut self,
+        name: impl Into<String>,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.optional_positional.push(PositionalArg {
+            name: name.into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    pub fn rest(
+        mut self,
+        name: &str,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.rest_positional = Some(PositionalArg {
+            name: name.into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
 
         self
     }
@@ -282,6 +458,7 @@ impl Signature {
     pub fn named(
         mut self,
         name: impl Into<String>,
+<<<<<<< HEAD
         ty: impl Into<SyntaxShape>,
         desc: impl Into<String>,
         short: Option<char>,
@@ -294,6 +471,22 @@ impl Signature {
             name.into(),
             (NamedType::Optional(s, ty.into()), desc.into()),
         );
+=======
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+        short: Option<char>,
+    ) -> Signature {
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name,
+            short: s,
+            arg: Some(shape.into()),
+            required: false,
+            desc: desc.into(),
+            var_id: None,
+        });
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
 
         self
     }
@@ -302,6 +495,7 @@ impl Signature {
     pub fn required_named(
         mut self,
         name: impl Into<String>,
+<<<<<<< HEAD
         ty: impl Into<SyntaxShape>,
         desc: impl Into<String>,
         short: Option<char>,
@@ -316,6 +510,23 @@ impl Signature {
             (NamedType::Mandatory(s, ty.into()), desc.into()),
         );
 
+=======
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+        short: Option<char>,
+    ) -> Signature {
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name,
+            short: s,
+            arg: Some(shape.into()),
+            required: true,
+            desc: desc.into(),
+            var_id: None,
+        });
+
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
         self
     }
 
@@ -326,6 +537,80 @@ impl Signature {
         desc: impl Into<String>,
         short: Option<char>,
     ) -> Signature {
+<<<<<<< HEAD
+=======
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name,
+            short: s,
+            arg: None,
+            required: false,
+            desc: desc.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Changes the signature category
+    pub fn category(mut self, category: Category) -> Signature {
+        self.category = category;
+
+        self
+    }
+
+    /// Sets that signature will create a scope as it parses
+    pub fn creates_scope(mut self) -> Signature {
+        self.creates_scope = true;
+        self
+    }
+
+    pub fn call_signature(&self) -> String {
+        let mut one_liner = String::new();
+        one_liner.push_str(&self.name);
+        one_liner.push(' ');
+
+        // Note: the call signature needs flags first because on the nu commandline,
+        // flags will precede the script file name. Flags for internal commands can come
+        // either before or after (or around) positional parameters, so there isn't a strong
+        // preference, so we default to the more constrained example.
+        if self.named.len() > 1 {
+            one_liner.push_str("{flags} ");
+        }
+
+        for positional in &self.required_positional {
+            one_liner.push_str(&get_positional_short_name(positional, true));
+        }
+        for positional in &self.optional_positional {
+            one_liner.push_str(&get_positional_short_name(positional, false));
+        }
+
+        if let Some(rest) = &self.rest_positional {
+            one_liner.push_str(&format!("...{}", get_positional_short_name(rest, false)));
+        }
+
+        // if !self.subcommands.is_empty() {
+        //     one_liner.push_str("<subcommand> ");
+        // }
+
+        one_liner
+    }
+
+    /// Get list of the short-hand flags
+    pub fn get_shorts(&self) -> Vec<char> {
+        self.named.iter().filter_map(|f| f.short).collect()
+    }
+
+    /// Get list of the long-hand flags
+    pub fn get_names(&self) -> Vec<&str> {
+        self.named.iter().map(|f| f.long.as_str()).collect()
+    }
+
+    /// Checks if short or long are already present
+    /// Panics if one of them is found
+    fn check_names(&self, name: impl Into<String>, short: Option<char>) -> (String, Option<char>) {
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
         let s = short.map(|c| {
             debug_assert!(
                 !self.get_shorts().contains(&c),
@@ -334,9 +619,95 @@ impl Signature {
             c
         });
 
+<<<<<<< HEAD
         self.named
             .insert(name.into(), (NamedType::Switch(s), desc.into()));
         self
+=======
+        let name = {
+            let name: String = name.into();
+            debug_assert!(
+                !self.get_names().contains(&name.as_str()),
+                "There may be duplicate name flags, such as --help"
+            );
+            name
+        };
+
+        (name, s)
+    }
+
+    pub fn get_positional(&self, position: usize) -> Option<PositionalArg> {
+        if position < self.required_positional.len() {
+            self.required_positional.get(position).cloned()
+        } else if position < (self.required_positional.len() + self.optional_positional.len()) {
+            self.optional_positional
+                .get(position - self.required_positional.len())
+                .cloned()
+        } else {
+            self.rest_positional.clone()
+        }
+    }
+
+    pub fn num_positionals(&self) -> usize {
+        let mut total = self.required_positional.len() + self.optional_positional.len();
+
+        for positional in &self.required_positional {
+            if let SyntaxShape::Keyword(..) = positional.shape {
+                // Keywords have a required argument, so account for that
+                total += 1;
+            }
+        }
+        for positional in &self.optional_positional {
+            if let SyntaxShape::Keyword(..) = positional.shape {
+                // Keywords have a required argument, so account for that
+                total += 1;
+            }
+        }
+        total
+    }
+
+    pub fn num_positionals_after(&self, idx: usize) -> usize {
+        let mut total = 0;
+
+        for (curr, positional) in self.required_positional.iter().enumerate() {
+            match positional.shape {
+                SyntaxShape::Keyword(..) => {
+                    // Keywords have a required argument, so account for that
+                    if curr > idx {
+                        total += 2;
+                    }
+                }
+                _ => {
+                    if curr > idx {
+                        total += 1;
+                    }
+                }
+            }
+        }
+        total
+    }
+
+    /// Find the matching long flag
+    pub fn get_long_flag(&self, name: &str) -> Option<Flag> {
+        for flag in &self.named {
+            if flag.long == name {
+                return Some(flag.clone());
+            }
+        }
+        None
+    }
+
+    /// Find the matching long flag
+    pub fn get_short_flag(&self, short: char) -> Option<Flag> {
+        for flag in &self.named {
+            if let Some(short_flag) = &flag.short {
+                if *short_flag == short {
+                    return Some(flag.clone());
+                }
+            }
+        }
+        None
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
     }
 
     /// Set the filter flag for the signature
@@ -345,6 +716,7 @@ impl Signature {
         self
     }
 
+<<<<<<< HEAD
     /// Set the type for the "rest" of the positional arguments
     /// Note: Not naming the field in your struct holding the rest values "rest", can
     /// cause errors when deserializing
@@ -379,5 +751,102 @@ impl Signature {
             }
         }
         shorts
+=======
+    /// Create a placeholder implementation of Command as a way to predeclare a definition's
+    /// signature so other definitions can see it. This placeholder is later replaced with the
+    /// full definition in a second pass of the parser.
+    pub fn predeclare(self) -> Box<dyn Command> {
+        Box::new(Predeclaration { signature: self })
+    }
+
+    /// Combines a signature and a block into a runnable block
+    pub fn into_block_command(self, block_id: BlockId) -> Box<dyn Command> {
+        Box::new(BlockCommand {
+            signature: self,
+            block_id,
+        })
+    }
+}
+
+#[derive(Clone)]
+struct Predeclaration {
+    signature: Signature,
+}
+
+impl Command for Predeclaration {
+    fn name(&self) -> &str {
+        &self.signature.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.signature.usage
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        _call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, crate::ShellError> {
+        panic!("Internal error: can't run a predeclaration without a body")
+    }
+}
+
+fn get_positional_short_name(arg: &PositionalArg, is_required: bool) -> String {
+    match &arg.shape {
+        SyntaxShape::Keyword(name, ..) => {
+            if is_required {
+                format!("{} <{}> ", String::from_utf8_lossy(name), arg.name)
+            } else {
+                format!("({} <{}>) ", String::from_utf8_lossy(name), arg.name)
+            }
+        }
+        _ => {
+            if is_required {
+                format!("<{}> ", arg.name)
+            } else {
+                format!("({}) ", arg.name)
+            }
+        }
+    }
+}
+
+#[derive(Clone)]
+struct BlockCommand {
+    signature: Signature,
+    block_id: BlockId,
+}
+
+impl Command for BlockCommand {
+    fn name(&self) -> &str {
+        &self.signature.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.signature.usage
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        _call: &Call,
+        _input: PipelineData,
+    ) -> Result<crate::PipelineData, crate::ShellError> {
+        panic!("Internal error: can't run custom command with 'run', use block_id");
+    }
+
+    fn get_block_id(&self) -> Option<BlockId> {
+        Some(self.block_id)
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
     }
 }

crates/nu-protocol/src/span.rs

@@ -0,0 +1,76 @@
+use miette::SourceSpan;
+use serde::{Deserialize, Serialize};
+
+/// A spanned area of interest, generic over what kind of thing is of interest
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Spanned<T>
+where
+    T: Clone + std::fmt::Debug,
+{
+    pub item: T,
+    pub span: Span,
+}
+
+/// Spans are a global offset across all seen files, which are cached in the engine's state. The start and
+/// end offset together make the inclusive start/exclusive end pair for where to underline to highlight
+/// a given point of interest.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+pub struct Span {
+    pub start: usize,
+    pub end: usize,
+}
+
+impl From<Span> for SourceSpan {
+    fn from(s: Span) -> Self {
+        Self::new(s.start.into(), (s.end - s.start).into())
+    }
+}
+
+impl Span {
+    pub fn new(start: usize, end: usize) -> Span {
+        Span { start, end }
+    }
+
+    /// Note: Only use this for test data, *not* live data, as it will point into unknown source
+    /// when used in errors.
+    pub fn test_data() -> Span {
+        Span { start: 0, end: 0 }
+    }
+
+    pub fn offset(&self, offset: usize) -> Span {
+        Span {
+            start: self.start - offset,
+            end: self.end - offset,
+        }
+    }
+
+    pub fn contains(&self, pos: usize) -> bool {
+        pos >= self.start && pos < self.end
+    }
+
+    /// Point to the space just past this span, useful for missing
+    /// values
+    pub fn past(&self) -> Span {
+        Span {
+            start: self.end,
+            end: self.end,
+        }
+    }
+}
+
+/// Used when you have a slice of spans of at least size 1
+pub fn span(spans: &[Span]) -> Span {
+    let length = spans.len();
+
+    if length == 0 {
+        // TODO: do this for now, but we might also want to protect against this case
+        Span { start: 0, end: 0 }
+    } else if length == 1 {
+        spans[0]
+    } else {
+        Span {
+            start: spans[0].start,
+            end: spans[length - 1].end,
+        }
+    }
+}

crates/nu-protocol/src/syntax_shape.rs

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 use nu_source::{DbgDocBldr, DebugDocBuilder, PrettyDebug};
 use serde::{Deserialize, Serialize};
 
@@ -58,13 +59,171 @@ impl SyntaxShape {
             SyntaxShape::Operator => "operator",
             SyntaxShape::RowCondition => "condition",
             SyntaxShape::MathExpression => "math expression",
+=======
+use std::fmt::Display;
+
+use serde::{Deserialize, Serialize};
+
+use crate::Type;
+
+/// The syntactic shapes that values must match to be passed into a command. You can think of this as the type-checking that occurs when you call a function.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum SyntaxShape {
+    /// A specific match to a word or symbol
+    Keyword(Vec<u8>, Box<SyntaxShape>),
+
+    /// Any syntactic form is allowed
+    Any,
+
+    /// Strings and string-like bare words are allowed
+    String,
+
+    /// A dotted path to navigate the table
+    CellPath,
+
+    /// A dotted path to navigate the table (including variable)
+    FullCellPath,
+
+    /// Only a numeric (integer or decimal) value is allowed
+    Number,
+
+    /// A range is allowed (eg, `1..3`)
+    Range,
+
+    /// Only an integer value is allowed
+    Int,
+
+    /// A filepath is allowed
+    Filepath,
+
+    /// A glob pattern is allowed, eg `foo*`
+    GlobPattern,
+
+    /// A module path pattern used for imports
+    ImportPattern,
+
+    /// A block is allowed, eg `{start this thing}`
+    Block(Option<Vec<SyntaxShape>>),
+
+    /// A table is allowed, eg `[[first, second]; [1, 2]]`
+    Table,
+
+    /// A table is allowed, eg `[first second]`
+    List(Box<SyntaxShape>),
+
+    /// A filesize value is allowed, eg `10kb`
+    Filesize,
+
+    /// A duration value is allowed, eg `19day`
+    Duration,
+
+    /// An operator
+    Operator,
+
+    /// A math expression which expands shorthand forms on the lefthand side, eg `foo > 1`
+    /// The shorthand allows us to more easily reach columns inside of the row being passed in
+    RowCondition,
+
+    /// A general math expression, eg `1 + 2`
+    MathExpression,
+
+    /// A variable name
+    Variable,
+
+    /// A variable with optional type, `x` or `x: int`
+    VarWithOptType,
+
+    /// A signature for a definition, `[x:int, --foo]`
+    Signature,
+
+    /// A general expression, eg `1 + 2` or `foo --bar`
+    Expression,
+
+    /// A boolean value
+    Boolean,
+
+    /// A record value
+    Record,
+
+    /// A custom shape with custom completion logic
+    Custom(Box<SyntaxShape>, String),
+}
+
+impl SyntaxShape {
+    pub fn to_type(&self) -> Type {
+        match self {
+            SyntaxShape::Any => Type::Unknown,
+            SyntaxShape::Block(_) => Type::Block,
+            SyntaxShape::CellPath => Type::Unknown,
+            SyntaxShape::Custom(custom, _) => custom.to_type(),
+            SyntaxShape::Duration => Type::Duration,
+            SyntaxShape::Expression => Type::Unknown,
+            SyntaxShape::Filepath => Type::String,
+            SyntaxShape::Filesize => Type::Filesize,
+            SyntaxShape::FullCellPath => Type::Unknown,
+            SyntaxShape::GlobPattern => Type::String,
+            SyntaxShape::ImportPattern => Type::Unknown,
+            SyntaxShape::Int => Type::Int,
+            SyntaxShape::List(x) => {
+                let contents = x.to_type();
+                Type::List(Box::new(contents))
+            }
+            SyntaxShape::Keyword(_, expr) => expr.to_type(),
+            SyntaxShape::MathExpression => Type::Unknown,
+            SyntaxShape::Number => Type::Number,
+            SyntaxShape::Operator => Type::Unknown,
+            SyntaxShape::Range => Type::Unknown,
+            SyntaxShape::Record => Type::Record(vec![]), // FIXME: Add actual record type
+            SyntaxShape::RowCondition => Type::Bool,
+            SyntaxShape::Boolean => Type::Bool,
+            SyntaxShape::Signature => Type::Signature,
+            SyntaxShape::String => Type::String,
+            SyntaxShape::Table => Type::List(Box::new(Type::Unknown)), // FIXME: Tables should have better types
+            SyntaxShape::VarWithOptType => Type::Unknown,
+            SyntaxShape::Variable => Type::Unknown,
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
         }
     }
 }
 
+<<<<<<< HEAD
 impl PrettyDebug for SyntaxShape {
     /// Prepare SyntaxShape for pretty-printing
     fn pretty(&self) -> DebugDocBuilder {
         DbgDocBldr::kind(self.syntax_shape_name().to_string())
+=======
+impl Display for SyntaxShape {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            SyntaxShape::Keyword(kw, shape) => {
+                write!(f, "\"{}\" {}", String::from_utf8_lossy(kw), shape)
+            }
+            SyntaxShape::Any => write!(f, "any"),
+            SyntaxShape::String => write!(f, "string"),
+            SyntaxShape::CellPath => write!(f, "cellpath"),
+            SyntaxShape::FullCellPath => write!(f, "cellpath"),
+            SyntaxShape::Number => write!(f, "number"),
+            SyntaxShape::Range => write!(f, "range"),
+            SyntaxShape::Int => write!(f, "int"),
+            SyntaxShape::Filepath => write!(f, "path"),
+            SyntaxShape::GlobPattern => write!(f, "glob"),
+            SyntaxShape::ImportPattern => write!(f, "import"),
+            SyntaxShape::Block(_) => write!(f, "block"),
+            SyntaxShape::Table => write!(f, "table"),
+            SyntaxShape::List(x) => write!(f, "list<{}>", x),
+            SyntaxShape::Record => write!(f, "record"),
+            SyntaxShape::Filesize => write!(f, "filesize"),
+            SyntaxShape::Duration => write!(f, "duration"),
+            SyntaxShape::Operator => write!(f, "operator"),
+            SyntaxShape::RowCondition => write!(f, "condition"),
+            SyntaxShape::MathExpression => write!(f, "variable"),
+            SyntaxShape::Variable => write!(f, "var"),
+            SyntaxShape::VarWithOptType => write!(f, "vardecl"),
+            SyntaxShape::Signature => write!(f, "signature"),
+            SyntaxShape::Expression => write!(f, "expression"),
+            SyntaxShape::Boolean => write!(f, "bool"),
+            SyntaxShape::Custom(x, _) => write!(f, "custom<{}>", x),
+        }
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
     }
 }

crates/nu-protocol/src/ty.rs

@@ -0,0 +1,64 @@
+use serde::{Deserialize, Serialize};
+
+use std::fmt::Display;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Type {
+    Int,
+    Float,
+    Range,
+    Bool,
+    String,
+    Block,
+    CellPath,
+    Duration,
+    Date,
+    Filesize,
+    List(Box<Type>),
+    Number,
+    Nothing,
+    Record(Vec<(String, Type)>),
+    Table,
+    ValueStream,
+    Unknown,
+    Error,
+    Binary,
+    Custom,
+    Signature,
+}
+
+impl Display for Type {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Type::Block => write!(f, "block"),
+            Type::Bool => write!(f, "bool"),
+            Type::CellPath => write!(f, "cell path"),
+            Type::Date => write!(f, "date"),
+            Type::Duration => write!(f, "duration"),
+            Type::Filesize => write!(f, "filesize"),
+            Type::Float => write!(f, "float"),
+            Type::Int => write!(f, "int"),
+            Type::Range => write!(f, "range"),
+            Type::Record(fields) => write!(
+                f,
+                "record<{}>",
+                fields
+                    .iter()
+                    .map(|(x, y)| format!("{}: {}", x, y))
+                    .collect::<Vec<String>>()
+                    .join(", "),
+            ),
+            Type::Table => write!(f, "table"),
+            Type::List(l) => write!(f, "list<{}>", l),
+            Type::Nothing => write!(f, "nothing"),
+            Type::Number => write!(f, "number"),
+            Type::String => write!(f, "string"),
+            Type::ValueStream => write!(f, "value stream"),
+            Type::Unknown => write!(f, "unknown"),
+            Type::Error => write!(f, "error"),
+            Type::Binary => write!(f, "binary"),
+            Type::Custom => write!(f, "custom"),
+            Type::Signature => write!(f, "signature"),
+        }
+    }
+}

crates/nu-protocol/src/type_name.rs

@@ -1,44 +0,0 @@
-use nu_source::{DebugDocBuilder, HasSpan, Spanned, SpannedItem, Tagged};
-
-/// A trait that allows structures to define a known .type_name() which pretty-prints the type
-pub trait ShellTypeName {
-    fn type_name(&self) -> &'static str;
-}
-
-impl<T: ShellTypeName> ShellTypeName for Spanned<T> {
-    /// Return the type_name of the spanned item
-    fn type_name(&self) -> &'static str {
-        self.item.type_name()
-    }
-}
-
-impl<T: ShellTypeName> ShellTypeName for &T {
-    /// Return the type_name for the borrowed reference
-    fn type_name(&self) -> &'static str {
-        (*self).type_name()
-    }
-}
-
-/// A trait that allows structures to define a known way to return a spanned type name
-pub trait SpannedTypeName {
-    fn spanned_type_name(&self) -> Spanned<&'static str>;
-}
-
-impl<T: ShellTypeName + HasSpan> SpannedTypeName for T {
-    /// Return the type name as a spanned string
-    fn spanned_type_name(&self) -> Spanned<&'static str> {
-        self.type_name().spanned(self.span())
-    }
-}
-
-impl<T: ShellTypeName> SpannedTypeName for Tagged<T> {
-    /// Return the spanned type name for a Tagged value
-    fn spanned_type_name(&self) -> Spanned<&'static str> {
-        self.item.type_name().spanned(self.tag.span)
-    }
-}
-
-/// A trait to enable pretty-printing of type information
-pub trait PrettyType {
-    fn pretty_type(&self) -> DebugDocBuilder;
-}

crates/nu-protocol/src/type_shape.rs

@@ -1,426 +0,0 @@
-///
-/// This file describes the structural types of the nushell system.
-///
-/// Its primary purpose today is to identify "equivalent" values for the purpose
-/// of merging rows into a single table or identify rows in a table that have the
-/// same shape for reflection.
-use crate::value::dict::Dictionary;
-use crate::value::primitive::Primitive;
-use crate::value::range::RangeInclusion;
-use crate::value::{UntaggedValue, Value};
-use derive_new::new;
-use indexmap::map::IndexMap;
-use nu_source::{DbgDocBldr, DebugDoc, DebugDocBuilder, PrettyDebug};
-use serde::{Deserialize, Serialize};
-use std::cmp::Ordering;
-use std::fmt::Debug;
-use std::hash::{Hash, Hasher};
-
-/// Representation of the type of ranges
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, new)]
-pub struct RangeType {
-    from: (Type, RangeInclusion),
-    to: (Type, RangeInclusion),
-}
-
-/// Representation of for the type of a value in Nu
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
-pub enum Type {
-    /// A value which has no value
-    Nothing,
-    /// An integer-based value
-    Int,
-    /// An big integer-based value
-    BigInt,
-    /// A range between two values
-    Range(Box<RangeType>),
-    /// A decimal (floating point) value
-    Decimal,
-    /// A filesize in bytes
-    Filesize,
-    /// A string of text
-    String,
-    /// A line of text (a string with trailing line ending)
-    Line,
-    /// A path through a table
-    ColumnPath,
-    /// A glob pattern (like foo*)
-    GlobPattern,
-    /// A boolean value
-    Boolean,
-    /// A date value (in UTC)
-    Date,
-    /// A data duration value
-    Duration,
-    /// A filepath value
-    FilePath,
-    /// A binary (non-text) buffer value
-    Binary,
-
-    /// A row of data
-    Row(Row),
-    /// A full table of data
-    Table(Vec<Type>),
-
-    /// A block of script (TODO)
-    Block,
-    /// An error value (TODO)
-    Error,
-
-    /// Beginning of stream marker (used as bookend markers rather than actual values)
-    BeginningOfStream,
-    /// End of stream marker (used as bookend markers rather than actual values)
-    EndOfStream,
-
-    /// Dataframe
-    #[cfg(feature = "dataframe")]
-    DataFrame,
-
-    /// Dataframe
-    #[cfg(feature = "dataframe")]
-    FrameStruct,
-}
-
-/// A shape representation of the type of a row
-#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Row {
-    map: IndexMap<Column, Type>,
-}
-
-#[allow(clippy::derive_hash_xor_eq)]
-impl Hash for Row {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let mut entries = self.map.clone();
-        entries.sort_keys();
-        entries.keys().collect::<Vec<&Column>>().hash(state);
-        entries.values().collect::<Vec<&Type>>().hash(state);
-    }
-}
-
-impl PartialOrd for Row {
-    /// Compare two dictionaries for sort ordering
-    fn partial_cmp(&self, other: &Row) -> Option<Ordering> {
-        let this: Vec<&Column> = self.map.keys().collect();
-        let that: Vec<&Column> = other.map.keys().collect();
-
-        if this != that {
-            return this.partial_cmp(&that);
-        }
-
-        let this: Vec<&Type> = self.map.values().collect();
-        let that: Vec<&Type> = self.map.values().collect();
-
-        this.partial_cmp(&that)
-    }
-}
-
-impl Ord for Row {
-    /// Compare two dictionaries for ordering
-    fn cmp(&self, other: &Row) -> Ordering {
-        let this: Vec<&Column> = self.map.keys().collect();
-        let that: Vec<&Column> = other.map.keys().collect();
-
-        if this != that {
-            return this.cmp(&that);
-        }
-
-        let this: Vec<&Type> = self.map.values().collect();
-        let that: Vec<&Type> = self.map.values().collect();
-
-        this.cmp(&that)
-    }
-}
-
-impl Type {
-    /// Convert a Primitive into its corresponding Type
-    pub fn from_primitive(primitive: &Primitive) -> Type {
-        match primitive {
-            Primitive::Nothing => Type::Nothing,
-            Primitive::Int(_) => Type::Int,
-            Primitive::BigInt(_) => Type::BigInt,
-            Primitive::Range(range) => {
-                let (left_value, left_inclusion) = &range.from;
-                let (right_value, right_inclusion) = &range.to;
-
-                let left_type = (Type::from_primitive(left_value), *left_inclusion);
-                let right_type = (Type::from_primitive(right_value), *right_inclusion);
-
-                let range = RangeType::new(left_type, right_type);
-                Type::Range(Box::new(range))
-            }
-            Primitive::Decimal(_) => Type::Decimal,
-            Primitive::Filesize(_) => Type::Filesize,
-            Primitive::String(_) => Type::String,
-            Primitive::ColumnPath(_) => Type::ColumnPath,
-            Primitive::GlobPattern(_) => Type::GlobPattern,
-            Primitive::Boolean(_) => Type::Boolean,
-            Primitive::Date(_) => Type::Date,
-            Primitive::Duration(_) => Type::Duration,
-            Primitive::FilePath(_) => Type::FilePath,
-            Primitive::Binary(_) => Type::Binary,
-            Primitive::BeginningOfStream => Type::BeginningOfStream,
-            Primitive::EndOfStream => Type::EndOfStream,
-        }
-    }
-
-    /// Convert a dictionary into its corresponding row Type
-    pub fn from_dictionary(dictionary: &Dictionary) -> Type {
-        let mut map = IndexMap::new();
-
-        for (key, value) in &dictionary.entries {
-            let column = Column::String(key.clone());
-            map.insert(column, Type::from_value(value));
-        }
-
-        Type::Row(Row { map })
-    }
-
-    /// Convert a table into its corresponding Type
-    pub fn from_table<'a>(table: impl IntoIterator<Item = &'a Value>) -> Type {
-        let mut vec = vec![];
-
-        for item in table {
-            vec.push(Type::from_value(item))
-        }
-
-        Type::Table(vec)
-    }
-
-    /// Convert a value into its corresponding Type
-    pub fn from_value<'a>(value: impl Into<&'a UntaggedValue>) -> Type {
-        match value.into() {
-            UntaggedValue::Primitive(p) => Type::from_primitive(p),
-            UntaggedValue::Row(row) => Type::from_dictionary(row),
-            UntaggedValue::Table(table) => Type::from_table(table),
-            UntaggedValue::Error(_) => Type::Error,
-            UntaggedValue::Block(_) => Type::Block,
-            #[cfg(feature = "dataframe")]
-            UntaggedValue::DataFrame(_) => Type::DataFrame,
-            #[cfg(feature = "dataframe")]
-            UntaggedValue::FrameStruct(_) => Type::DataFrame,
-        }
-    }
-}
-
-impl PrettyDebug for Type {
-    /// Prepare Type for pretty-printing
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            Type::Nothing => ty("nothing"),
-            Type::Int => ty("integer"),
-            Type::BigInt => ty("big integer"),
-            Type::Range(range) => {
-                let (left, left_inclusion) = &range.from;
-                let (right, right_inclusion) = &range.to;
-
-                let left_bracket = DbgDocBldr::delimiter(match left_inclusion {
-                    RangeInclusion::Exclusive => "(",
-                    RangeInclusion::Inclusive => "[",
-                });
-
-                let right_bracket = DbgDocBldr::delimiter(match right_inclusion {
-                    RangeInclusion::Exclusive => ")",
-                    RangeInclusion::Inclusive => "]",
-                });
-
-                DbgDocBldr::typed(
-                    "range",
-                    (left_bracket
-                        + left.pretty()
-                        + DbgDocBldr::operator(",")
-                        + DbgDocBldr::space()
-                        + right.pretty()
-                        + right_bracket)
-                        .group(),
-                )
-            }
-            Type::Decimal => ty("decimal"),
-            Type::Filesize => ty("filesize"),
-            Type::String => ty("string"),
-            Type::Line => ty("line"),
-            Type::ColumnPath => ty("column-path"),
-            Type::GlobPattern => ty("pattern"),
-            Type::Boolean => ty("boolean"),
-            Type::Date => ty("date"),
-            Type::Duration => ty("duration"),
-            Type::FilePath => ty("path"),
-            Type::Binary => ty("binary"),
-            Type::Error => DbgDocBldr::error("error"),
-            Type::BeginningOfStream => DbgDocBldr::keyword("beginning-of-stream"),
-            Type::EndOfStream => DbgDocBldr::keyword("end-of-stream"),
-            Type::Row(row) => (DbgDocBldr::kind("row")
-                + DbgDocBldr::space()
-                + DbgDocBldr::intersperse(
-                    row.map.iter().map(|(key, ty)| {
-                        (DbgDocBldr::key(match key {
-                            Column::String(string) => string.clone(),
-                            Column::Value => "".to_string(),
-                        }) + DbgDocBldr::delimit("(", ty.pretty(), ")").into_kind())
-                        .nest()
-                    }),
-                    DbgDocBldr::space(),
-                )
-                .nest())
-            .nest(),
-
-            Type::Table(table) => {
-                let mut group: Group<DebugDoc, Vec<(usize, usize)>> = Group::new();
-
-                for (i, item) in table.iter().enumerate() {
-                    group.add(item.to_doc(), i);
-                }
-
-                (DbgDocBldr::kind("table") + DbgDocBldr::space() + DbgDocBldr::keyword("of"))
-                    .group()
-                    + DbgDocBldr::space()
-                    + (if group.len() == 1 {
-                        let (doc, _) = group.into_iter().collect::<Vec<_>>()[0].clone();
-                        DebugDocBuilder::from_doc(doc)
-                    } else {
-                        DbgDocBldr::intersperse(
-                            group.into_iter().map(|(doc, rows)| {
-                                (DbgDocBldr::intersperse(
-                                    rows.iter().map(|(from, to)| {
-                                        if from == to {
-                                            DbgDocBldr::description(from)
-                                        } else {
-                                            (DbgDocBldr::description(from)
-                                                + DbgDocBldr::space()
-                                                + DbgDocBldr::keyword("to")
-                                                + DbgDocBldr::space()
-                                                + DbgDocBldr::description(to))
-                                            .group()
-                                        }
-                                    }),
-                                    DbgDocBldr::description(", "),
-                                ) + DbgDocBldr::description(":")
-                                    + DbgDocBldr::space()
-                                    + DebugDocBuilder::from_doc(doc))
-                                .nest()
-                            }),
-                            DbgDocBldr::space(),
-                        )
-                    })
-            }
-            Type::Block => ty("block"),
-            #[cfg(feature = "dataframe")]
-            Type::DataFrame | Type::FrameStruct => ty("data_type_formatter"),
-        }
-    }
-}
-
-/// A view into dictionaries for debug purposes
-#[derive(Debug, new)]
-struct DebugEntry<'a> {
-    key: &'a Column,
-    value: &'a Type,
-}
-
-impl<'a> PrettyDebug for DebugEntry<'a> {
-    /// Prepare debug entries for pretty-printing
-    fn pretty(&self) -> DebugDocBuilder {
-        DbgDocBldr::key(match self.key {
-            Column::String(string) => string.clone(),
-            Column::Value => "".to_string(),
-        }) + DbgDocBldr::delimit("(", self.value.pretty(), ")").into_kind()
-    }
-}
-
-/// Helper to create a pretty-print for the type
-fn ty(name: impl std::fmt::Display) -> DebugDocBuilder {
-    DbgDocBldr::kind(name)
-}
-
-pub trait GroupedValue: Debug + Clone {
-    type Item;
-
-    fn new() -> Self;
-    fn merge(&mut self, value: Self::Item);
-}
-
-impl GroupedValue for Vec<(usize, usize)> {
-    type Item = usize;
-
-    fn new() -> Vec<(usize, usize)> {
-        vec![]
-    }
-
-    fn merge(&mut self, new_value: usize) {
-        match self.last_mut() {
-            Some(value) if value.1 == new_value - 1 => {
-                value.1 += 1;
-            }
-
-            _ => self.push((new_value, new_value)),
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct Group<K: Debug + Eq + Hash, V: GroupedValue> {
-    values: indexmap::IndexMap<K, V>,
-}
-
-impl<K, G> Group<K, G>
-where
-    K: Debug + Eq + Hash,
-    G: GroupedValue,
-{
-    pub fn new() -> Group<K, G> {
-        Group {
-            values: indexmap::IndexMap::default(),
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        self.values.len()
-    }
-
-    pub fn into_iter(self) -> impl Iterator<Item = (K, G)> {
-        self.values.into_iter()
-    }
-
-    pub fn add(&mut self, key: impl Into<K>, value: impl Into<G::Item>) {
-        let key = key.into();
-        let value = value.into();
-
-        let group = self.values.get_mut(&key);
-
-        match group {
-            None => {
-                self.values.insert(key, {
-                    let mut group = G::new();
-                    group.merge(value);
-                    group
-                });
-            }
-            Some(group) => {
-                group.merge(value);
-            }
-        }
-    }
-}
-
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
-pub enum Column {
-    String(String),
-    Value,
-}
-
-impl From<String> for Column {
-    fn from(x: String) -> Self {
-        Column::String(x)
-    }
-}
-
-impl From<&String> for Column {
-    fn from(x: &String) -> Self {
-        Column::String(x.clone())
-    }
-}
-
-impl From<&str> for Column {
-    fn from(x: &str) -> Self {
-        Column::String(x.to_string())
-    }
-}

crates/nu-protocol/src/value/column_path.rs

@@ -1,363 +0,0 @@
-use derive_new::new;
-use getset::Getters;
-use nu_source::{
-    span_for_spanned_list, DbgDocBldr, DebugDocBuilder, HasFallibleSpan, PrettyDebug, Span,
-    Spanned, SpannedItem,
-};
-use serde::{Deserialize, Serialize};
-
-use crate::hir::{Expression, Literal, Member, SpannedExpression};
-use nu_errors::ParseError;
-
-/// A PathMember that has yet to be spanned so that it can be used in later processing
-#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
-pub enum UnspannedPathMember {
-    String(String),
-    Int(i64),
-}
-
-impl UnspannedPathMember {
-    /// Add the span information and get a full PathMember
-    pub fn into_path_member(self, span: impl Into<Span>) -> PathMember {
-        PathMember {
-            unspanned: self,
-            span: span.into(),
-        }
-    }
-}
-
-/// A basic piece of a ColumnPath, which describes the steps to take through a table to arrive a cell, row, or inner table
-#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
-pub struct PathMember {
-    pub unspanned: UnspannedPathMember,
-    pub span: Span,
-}
-
-impl PrettyDebug for &PathMember {
-    /// Gets the PathMember ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        match &self.unspanned {
-            UnspannedPathMember::String(string) => DbgDocBldr::primitive(format!("{:?}", string)),
-            UnspannedPathMember::Int(int) => DbgDocBldr::primitive(int),
-        }
-    }
-}
-
-/// The fundamental path primitive to describe how to navigate through a table to get to a sub-item. A path member can be either a word or a number. Words/strings are taken to mean
-/// a column name, and numbers are the row number. Taken together they describe which column or row to narrow to in order to get data.
-///
-/// Rows must follow column names, they can't come first. eg) `foo.1` is valid where `1.foo` is not.
-#[derive(
-    Debug, Hash, Serialize, Deserialize, Ord, PartialOrd, Eq, PartialEq, Getters, Clone, new,
-)]
-pub struct ColumnPath {
-    #[get = "pub"]
-    members: Vec<PathMember>,
-}
-
-impl ColumnPath {
-    /// Iterate over the members of the column path
-    pub fn iter(&self) -> impl Iterator<Item = &PathMember> {
-        self.members.iter()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.members.is_empty()
-    }
-
-    /// Returns the last member and a slice of the remaining members
-    pub fn split_last(&self) -> Option<(&PathMember, &[PathMember])> {
-        self.members.split_last()
-    }
-
-    /// Returns the last member
-    pub fn last(&self) -> Option<&PathMember> {
-        self.iter().last()
-    }
-
-    pub fn path(&self) -> String {
-        let sep = std::path::MAIN_SEPARATOR;
-        let mut members = self.iter();
-        let mut f = String::from(sep);
-
-        if let Some(member) = members.next() {
-            f.push_str(&member.as_string());
-        }
-
-        for member in members {
-            f.push(sep);
-            f.push_str(&member.as_string());
-        }
-
-        f
-    }
-
-    pub fn build(text: &Spanned<String>) -> ColumnPath {
-        if let (
-            SpannedExpression {
-                expr: Expression::Literal(Literal::ColumnPath(path)),
-                span: _,
-            },
-            _,
-        ) = parse(text)
-        {
-            ColumnPath {
-                members: path.iter().map(|member| member.to_path_member()).collect(),
-            }
-        } else {
-            ColumnPath { members: vec![] }
-        }
-    }
-
-    pub fn with_head(text: &Spanned<String>) -> Option<(String, ColumnPath)> {
-        match parse_full_column_path(text) {
-            (
-                SpannedExpression {
-                    expr: Expression::FullColumnPath(path),
-                    ..
-                },
-                _,
-            ) => {
-                if let crate::hir::FullColumnPath {
-                    head:
-                        SpannedExpression {
-                            expr: Expression::Variable(name, _),
-                            span: _,
-                        },
-                    tail,
-                } = *path
-                {
-                    Some((
-                        name,
-                        ColumnPath {
-                            members: tail.to_vec(),
-                        },
-                    ))
-                } else {
-                    None
-                }
-            }
-            _ => None,
-        }
-    }
-}
-
-impl PrettyDebug for ColumnPath {
-    /// Gets the ColumnPath ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        let members: Vec<DebugDocBuilder> =
-            self.members.iter().map(|member| member.pretty()).collect();
-
-        DbgDocBldr::delimit(
-            "(",
-            DbgDocBldr::description("path")
-                + DbgDocBldr::equals()
-                + DbgDocBldr::intersperse(members, DbgDocBldr::space()),
-            ")",
-        )
-        .nest()
-    }
-}
-
-impl HasFallibleSpan for ColumnPath {
-    /// Creates a span that will cover the column path, if possible
-    fn maybe_span(&self) -> Option<Span> {
-        if self.members.is_empty() {
-            None
-        } else {
-            Some(span_for_spanned_list(self.members.iter().map(|m| m.span)))
-        }
-    }
-}
-
-impl<'a> IntoIterator for &'a ColumnPath {
-    type Item = &'a PathMember;
-
-    type IntoIter = std::slice::Iter<'a, PathMember>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        self.members.iter()
-    }
-}
-
-impl PathMember {
-    /// Create a string path member
-    pub fn string(string: impl Into<String>, span: impl Into<Span>) -> PathMember {
-        UnspannedPathMember::String(string.into()).into_path_member(span)
-    }
-
-    /// Create a numeric path member
-    pub fn int(int: i64, span: impl Into<Span>) -> PathMember {
-        UnspannedPathMember::Int(int).into_path_member(span)
-    }
-
-    pub fn as_string(&self) -> String {
-        match &self.unspanned {
-            UnspannedPathMember::String(string) => string.clone(),
-            UnspannedPathMember::Int(int) => int.to_string(),
-        }
-    }
-}
-
-fn parse_full_column_path(
-    raw_column_path: &Spanned<String>,
-) -> (SpannedExpression, Option<ParseError>) {
-    let mut inside_delimiter = vec![];
-    let mut output = vec![];
-    let mut current_part = String::new();
-    let mut start_index = 0;
-    let mut last_index = 0;
-    let error = None;
-
-    let mut head = None;
-
-    for (idx, c) in raw_column_path.item.char_indices() {
-        last_index = idx;
-        if c == '(' {
-            inside_delimiter.push(')');
-        } else if let Some(delimiter) = inside_delimiter.last() {
-            if c == *delimiter {
-                inside_delimiter.pop();
-            }
-        } else if c == '\'' || c == '"' {
-            inside_delimiter.push(c);
-        } else if c == '.' {
-            let part_span = Span::new(
-                raw_column_path.span.start() + start_index,
-                raw_column_path.span.start() + idx,
-            );
-
-            if head.is_none() && current_part.starts_with('$') {
-                // We have the variable head
-                head = Some(Expression::variable(current_part.clone(), part_span))
-            } else if let Ok(row_number) = current_part.parse::<i64>() {
-                output.push(UnspannedPathMember::Int(row_number).into_path_member(part_span));
-            } else {
-                let current_part = trim_quotes(&current_part);
-                output.push(
-                    UnspannedPathMember::String(current_part.clone()).into_path_member(part_span),
-                );
-            }
-            current_part.clear();
-            // Note: I believe this is safe because of the delimiter we're using,
-            // but if we get fancy with Unicode we'll need to change this.
-            start_index = idx + '.'.len_utf8();
-            continue;
-        }
-        current_part.push(c);
-    }
-
-    if !current_part.is_empty() {
-        let part_span = Span::new(
-            raw_column_path.span.start() + start_index,
-            raw_column_path.span.start() + last_index + 1,
-        );
-
-        if head.is_none() {
-            if current_part.starts_with('$') {
-                head = Some(Expression::variable(current_part, raw_column_path.span));
-            } else if let Ok(row_number) = current_part.parse::<i64>() {
-                output.push(UnspannedPathMember::Int(row_number).into_path_member(part_span));
-            } else {
-                let current_part = trim_quotes(&current_part);
-                output.push(UnspannedPathMember::String(current_part).into_path_member(part_span));
-            }
-        } else if let Ok(row_number) = current_part.parse::<i64>() {
-            output.push(UnspannedPathMember::Int(row_number).into_path_member(part_span));
-        } else {
-            let current_part = trim_quotes(&current_part);
-            output.push(UnspannedPathMember::String(current_part).into_path_member(part_span));
-        }
-    }
-
-    if let Some(head) = head {
-        (
-            SpannedExpression::new(
-                Expression::path(SpannedExpression::new(head, raw_column_path.span), output),
-                raw_column_path.span,
-            ),
-            error,
-        )
-    } else {
-        (
-            SpannedExpression::new(
-                Expression::path(
-                    SpannedExpression::new(
-                        Expression::variable("$it".into(), raw_column_path.span),
-                        raw_column_path.span,
-                    ),
-                    output,
-                ),
-                raw_column_path.span,
-            ),
-            error,
-        )
-    }
-}
-
-fn parse(raw_column_path: &Spanned<String>) -> (SpannedExpression, Option<ParseError>) {
-    let mut delimiter = '.';
-    let mut inside_delimiter = false;
-    let mut output = vec![];
-    let mut current_part = String::new();
-    let mut start_index = 0;
-    let mut last_index = 0;
-
-    for (idx, c) in raw_column_path.item.char_indices() {
-        last_index = idx;
-        if inside_delimiter {
-            if c == delimiter {
-                inside_delimiter = false;
-            }
-        } else if c == '\'' || c == '"' || c == '`' {
-            inside_delimiter = true;
-            delimiter = c;
-        } else if c == '.' {
-            let part_span = Span::new(
-                raw_column_path.span.start() + start_index,
-                raw_column_path.span.start() + idx,
-            );
-
-            if let Ok(row_number) = current_part.parse::<i64>() {
-                output.push(Member::Int(row_number, part_span));
-            } else {
-                let trimmed = trim_quotes(&current_part);
-                output.push(Member::Bare(trimmed.clone().spanned(part_span)));
-            }
-            current_part.clear();
-            // Note: I believe this is safe because of the delimiter we're using,
-            // but if we get fancy with Unicode we'll need to change this.
-            start_index = idx + '.'.len_utf8();
-            continue;
-        }
-        current_part.push(c);
-    }
-
-    if !current_part.is_empty() {
-        let part_span = Span::new(
-            raw_column_path.span.start() + start_index,
-            raw_column_path.span.start() + last_index + 1,
-        );
-        if let Ok(row_number) = current_part.parse::<i64>() {
-            output.push(Member::Int(row_number, part_span));
-        } else {
-            let current_part = trim_quotes(&current_part);
-            output.push(Member::Bare(current_part.spanned(part_span)));
-        }
-    }
-
-    (
-        SpannedExpression::new(Expression::simple_column_path(output), raw_column_path.span),
-        None,
-    )
-}
-
-fn trim_quotes(input: &str) -> String {
-    let mut chars = input.chars();
-
-    match (chars.next(), chars.next_back()) {
-        (Some('\''), Some('\'')) => chars.collect(),
-        (Some('"'), Some('"')) => chars.collect(),
-        _ => input.to_string(),
-    }
-}

crates/nu-protocol/src/value/convert.rs

@@ -1,60 +0,0 @@
-use crate::type_name::SpannedTypeName;
-use crate::value::dict::Dictionary;
-use crate::value::primitive::Primitive;
-use crate::value::{UntaggedValue, Value};
-use nu_errors::{CoerceInto, ShellError};
-use nu_source::TaggedItem;
-
-impl std::convert::TryFrom<&Value> for i64 {
-    type Error = ShellError;
-
-    /// Convert to an i64 integer, if possible
-    fn try_from(value: &Value) -> Result<i64, ShellError> {
-        match &value.value {
-            UntaggedValue::Primitive(Primitive::Int(int)) => Ok(*int),
-            UntaggedValue::Primitive(Primitive::BigInt(int)) => {
-                int.tagged(&value.tag).coerce_into("converting to i64")
-            }
-            _ => Err(ShellError::type_error("Integer", value.spanned_type_name())),
-        }
-    }
-}
-
-impl std::convert::TryFrom<&Value> for String {
-    type Error = ShellError;
-
-    /// Convert to a string, if possible
-    fn try_from(value: &Value) -> Result<String, ShellError> {
-        match &value.value {
-            UntaggedValue::Primitive(Primitive::String(s)) => Ok(s.clone()),
-            _ => Err(ShellError::type_error("String", value.spanned_type_name())),
-        }
-    }
-}
-
-impl std::convert::TryFrom<&Value> for Vec<u8> {
-    type Error = ShellError;
-
-    /// Convert to a u8 vec, if possible
-    fn try_from(value: &Value) -> Result<Vec<u8>, ShellError> {
-        match &value.value {
-            UntaggedValue::Primitive(Primitive::Binary(b)) => Ok(b.clone()),
-            _ => Err(ShellError::type_error("Binary", value.spanned_type_name())),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Value> for &'a Dictionary {
-    type Error = ShellError;
-
-    /// Convert to a dictionary, if possible
-    fn try_from(value: &'a Value) -> Result<&'a Dictionary, ShellError> {
-        match &value.value {
-            UntaggedValue::Row(d) => Ok(d),
-            _ => Err(ShellError::type_error(
-                "Dictionary",
-                value.spanned_type_name(),
-            )),
-        }
-    }
-}

crates/nu-protocol/src/value/custom_value.rs

@@ -0,0 +1,60 @@
+use std::{cmp::Ordering, fmt};
+
+use crate::{ast::Operator, ShellError, Span, Value};
+
+// Trait definition for a custom value
+#[typetag::serde(tag = "type")]
+pub trait CustomValue: fmt::Debug + Send + Sync {
+    fn clone_value(&self, span: Span) -> Value;
+
+    //fn category(&self) -> Category;
+
+    // Define string representation of the custom value
+    fn value_string(&self) -> String;
+
+    // Converts the custom value to a base nushell value
+    // This is used to represent the custom value using the table representations
+    // That already exist in nushell
+    fn to_base_value(&self, span: Span) -> Result<Value, ShellError>;
+
+    // Json representation of custom value
+    fn to_json(&self) -> nu_json::Value {
+        nu_json::Value::Null
+    }
+
+    // Any representation used to downcast object to its original type
+    fn as_any(&self) -> &dyn std::any::Any;
+
+    // Follow cell path functions
+    fn follow_path_int(&self, _count: usize, span: Span) -> Result<Value, ShellError> {
+        Err(ShellError::IncompatiblePathAccess(
+            format!("{} does't support path access", self.value_string()),
+            span,
+        ))
+    }
+
+    fn follow_path_string(&self, _column_name: String, span: Span) -> Result<Value, ShellError> {
+        Err(ShellError::IncompatiblePathAccess(
+            format!("{} does't support path access", self.value_string()),
+            span,
+        ))
+    }
+
+    // ordering with other value
+    fn partial_cmp(&self, _other: &Value) -> Option<Ordering> {
+        None
+    }
+
+    // Definition of an operation between the object that implements the trait
+    // and another Value.
+    // The Operator enum is used to indicate the expected operation
+    fn operation(
+        &self,
+        _lhs_span: Span,
+        operator: Operator,
+        op: Span,
+        _right: &Value,
+    ) -> Result<Value, ShellError> {
+        Err(ShellError::UnsupportedOperator(operator, op))
+    }
+}

crates/nu-protocol/src/value/debug.rs

@@ -1,111 +0,0 @@
-use crate::type_name::PrettyType;
-use crate::value::primitive::Primitive;
-use crate::value::{UntaggedValue, Value};
-use nu_source::{DbgDocBldr, DebugDocBuilder, PrettyDebug};
-
-impl PrettyDebug for &Value {
-    /// Get a borrowed Value ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        PrettyDebug::pretty(*self)
-    }
-}
-
-impl PrettyDebug for Value {
-    /// Get a Value ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        match &self.value {
-            UntaggedValue::Primitive(p) => p.pretty(),
-            UntaggedValue::Row(row) => row.pretty_builder().nest(1).group().into(),
-            UntaggedValue::Table(table) => DbgDocBldr::delimit(
-                "[",
-                DbgDocBldr::intersperse(table, DbgDocBldr::space()),
-                "]",
-            )
-            .nest(),
-            UntaggedValue::Error(_) => DbgDocBldr::error("error"),
-            UntaggedValue::Block(_) => DbgDocBldr::opaque("block"),
-            #[cfg(feature = "dataframe")]
-            UntaggedValue::DataFrame(_) | UntaggedValue::FrameStruct(_) => {
-                DbgDocBldr::opaque("dataframe")
-            }
-        }
-    }
-}
-
-impl PrettyType for Primitive {
-    /// Find the type of the Value and prepare it for pretty-printing
-    fn pretty_type(&self) -> DebugDocBuilder {
-        match self {
-            Primitive::Nothing => ty("nothing"),
-            Primitive::Int(_) => ty("integer"),
-            Primitive::BigInt(_) => ty("big-integer"),
-            Primitive::Range(_) => ty("range"),
-            Primitive::Decimal(_) => ty("decimal"),
-            Primitive::Filesize(_) => ty("filesize"),
-            Primitive::String(_) => ty("string"),
-            Primitive::ColumnPath(_) => ty("column-path"),
-            Primitive::GlobPattern(_) => ty("pattern"),
-            Primitive::Boolean(_) => ty("boolean"),
-            Primitive::Date(_) => ty("date"),
-            Primitive::Duration(_) => ty("duration"),
-            Primitive::FilePath(_) => ty("path"),
-            Primitive::Binary(_) => ty("binary"),
-            Primitive::BeginningOfStream => DbgDocBldr::keyword("beginning-of-stream"),
-            Primitive::EndOfStream => DbgDocBldr::keyword("end-of-stream"),
-        }
-    }
-}
-
-impl PrettyDebug for Primitive {
-    /// Get a Primitive value ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            Primitive::Nothing => DbgDocBldr::primitive("nothing"),
-            Primitive::Int(int) => prim(format_args!("{}", int)),
-            Primitive::BigInt(int) => prim(format_args!("{}", int)),
-            Primitive::Decimal(decimal) => prim(format_args!("{}", decimal)),
-            Primitive::Range(range) => {
-                let (left, left_inclusion) = &range.from;
-                let (right, right_inclusion) = &range.to;
-
-                DbgDocBldr::typed(
-                    "range",
-                    (left_inclusion.debug_left_bracket()
-                        + left.pretty()
-                        + DbgDocBldr::operator(",")
-                        + DbgDocBldr::space()
-                        + right.pretty()
-                        + right_inclusion.debug_right_bracket())
-                    .group(),
-                )
-            }
-            Primitive::Filesize(bytes) => primitive_doc(bytes, "filesize"),
-            Primitive::String(string) => prim(string),
-            Primitive::ColumnPath(path) => path.pretty(),
-            Primitive::GlobPattern(pattern) => primitive_doc(pattern, "pattern"),
-            Primitive::Boolean(boolean) => match boolean {
-                true => DbgDocBldr::primitive("$yes"),
-                false => DbgDocBldr::primitive("$no"),
-            },
-            Primitive::Date(date) => primitive_doc(date, "date"),
-            Primitive::Duration(duration) => primitive_doc(duration, "nanoseconds"),
-            Primitive::FilePath(path) => primitive_doc(path, "path"),
-            Primitive::Binary(_) => DbgDocBldr::opaque("binary"),
-            Primitive::BeginningOfStream => DbgDocBldr::keyword("beginning-of-stream"),
-            Primitive::EndOfStream => DbgDocBldr::keyword("end-of-stream"),
-        }
-    }
-}
-
-fn prim(name: impl std::fmt::Debug) -> DebugDocBuilder {
-    DbgDocBldr::primitive(format!("{:?}", name))
-}
-
-fn primitive_doc(name: impl std::fmt::Debug, ty: impl Into<String>) -> DebugDocBuilder {
-    DbgDocBldr::primitive(format!("{:?}", name))
-        + DbgDocBldr::delimit("(", DbgDocBldr::kind(ty.into()), ")")
-}
-
-fn ty(name: impl std::fmt::Debug) -> DebugDocBuilder {
-    DbgDocBldr::kind(format!("{:?}", name))
-}

crates/nu-protocol/src/value/dict.rs

@@ -1,270 +0,0 @@
-use crate::maybe_owned::MaybeOwned;
-use crate::value::primitive::Primitive;
-use crate::value::{UntaggedValue, Value};
-use derive_new::new;
-use getset::Getters;
-use indexmap::IndexMap;
-use nu_source::{DbgDocBldr, DebugDocBuilder, PrettyDebug, Spanned, SpannedItem, Tag};
-use serde::{Deserialize, Serialize};
-use std::cmp::{Ord, Ordering, PartialOrd};
-use std::hash::{Hash, Hasher};
-
-/// A dictionary that can hold a mapping from names to Values
-#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, Clone, Getters, new)]
-pub struct Dictionary {
-    #[get = "pub"]
-    pub entries: IndexMap<String, Value>,
-}
-
-#[allow(clippy::derive_hash_xor_eq)]
-impl Hash for Dictionary {
-    /// Create the hash function to allow the Hash trait for dictionaries
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let mut entries = self.entries.clone();
-        entries.sort_keys();
-        entries.keys().collect::<Vec<&String>>().hash(state);
-        entries.values().collect::<Vec<&Value>>().hash(state);
-    }
-}
-
-impl PartialOrd for Dictionary {
-    /// Compare two dictionaries for sort ordering
-    fn partial_cmp(&self, other: &Dictionary) -> Option<Ordering> {
-        let this: Vec<&String> = self.entries.keys().collect();
-        let that: Vec<&String> = other.entries.keys().collect();
-
-        if this != that {
-            return this.partial_cmp(&that);
-        }
-
-        let this: Vec<&Value> = self.entries.values().collect();
-        let that: Vec<&Value> = self.entries.values().collect();
-
-        this.partial_cmp(&that)
-    }
-}
-
-impl Ord for Dictionary {
-    /// Compare two dictionaries for ordering
-    fn cmp(&self, other: &Dictionary) -> Ordering {
-        let this: Vec<&String> = self.entries.keys().collect();
-        let that: Vec<&String> = other.entries.keys().collect();
-
-        if this != that {
-            return this.cmp(&that);
-        }
-
-        let this: Vec<&Value> = self.entries.values().collect();
-        let that: Vec<&Value> = self.entries.values().collect();
-
-        this.cmp(&that)
-    }
-}
-
-impl PartialEq<Value> for Dictionary {
-    /// Test a dictionary against a Value for equality
-    fn eq(&self, other: &Value) -> bool {
-        matches!(&other.value, UntaggedValue::Row(d) if self == d)
-    }
-}
-
-/// A key-value pair specifically meant to be used in debug and pretty-printing
-#[derive(Debug, new)]
-struct DebugEntry<'a> {
-    key: &'a str,
-    value: &'a Value,
-}
-
-impl<'a> PrettyDebug for DebugEntry<'a> {
-    /// Build the the information to pretty-print the DebugEntry
-    fn pretty(&self) -> DebugDocBuilder {
-        (DbgDocBldr::key(self.key.to_string())
-            + DbgDocBldr::equals()
-            + self.value.pretty().into_value())
-        .group()
-    }
-}
-
-impl PrettyDebug for Dictionary {
-    /// Get a Dictionary ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        DbgDocBldr::delimit(
-            "(",
-            DbgDocBldr::intersperse(
-                self.entries()
-                    .iter()
-                    .map(|(key, value)| DebugEntry::new(key, value)),
-                DbgDocBldr::space(),
-            ),
-            ")",
-        )
-    }
-}
-
-impl From<IndexMap<String, Value>> for Dictionary {
-    /// Create a dictionary from a map of strings to Values
-    fn from(input: IndexMap<String, Value>) -> Dictionary {
-        let mut out = IndexMap::default();
-
-        for (key, value) in input {
-            out.insert(key, value);
-        }
-
-        Dictionary::new(out)
-    }
-}
-
-impl Dictionary {
-    /// Find the matching Value for a given key, if possible. If not, return a Primitive::Nothing
-    pub fn get_data(&self, desc: &str) -> MaybeOwned<'_, Value> {
-        match self.entries.get(desc) {
-            Some(v) => MaybeOwned::Borrowed(v),
-            None => MaybeOwned::Owned(
-                UntaggedValue::Primitive(Primitive::Nothing).into_untagged_value(),
-            ),
-        }
-    }
-
-    pub fn insert(&mut self, key: String, value: Value) -> Option<Value> {
-        self.entries.insert_full(key, value).1
-    }
-
-    pub fn merge_from(&self, other: &Dictionary) -> Dictionary {
-        let mut obj = self.clone();
-
-        for column in other.keys() {
-            let key = column.clone();
-            let value_key = key.as_str();
-            let value_spanned_key = value_key.spanned_unknown();
-
-            let other_column = match other.get_data_by_key(value_spanned_key) {
-                Some(value) => value,
-                None => UntaggedValue::Primitive(Primitive::Nothing).into_untagged_value(),
-            };
-
-            obj.entries.insert(key, other_column);
-        }
-
-        obj
-    }
-
-    /// Iterate the keys in the Dictionary
-    pub fn keys(&self) -> impl Iterator<Item = &String> {
-        self.entries.keys()
-    }
-
-    /// Iterate the values in the Dictionary
-    pub fn values(&self) -> impl Iterator<Item = &Value> {
-        self.entries.values()
-    }
-
-    /// Checks if given key exists
-    pub fn contains_key(&self, key: &str) -> bool {
-        self.entries.contains_key(key)
-    }
-
-    /// Find the matching Value for a key, if possible
-    pub fn get_data_by_key(&self, name: Spanned<&str>) -> Option<Value> {
-        let result = self
-            .entries
-            .iter()
-            .find(|(desc_name, _)| *desc_name == name.item)?
-            .1;
-
-        Some(
-            result
-                .value
-                .clone()
-                .into_value(Tag::new(result.tag.anchor(), name.span)),
-        )
-    }
-
-    /// Get a mutable entry that matches a key, if possible
-    pub fn get_mut_data_by_key(&mut self, name: &str) -> Option<&mut Value> {
-        self.entries
-            .iter_mut()
-            .find(|(desc_name, _)| *desc_name == name)
-            .map(|(_, v)| v)
-    }
-
-    /// Insert a new key/value pair into the dictionary
-    pub fn insert_data_at_key(&mut self, name: &str, value: Value) {
-        self.entries.insert(name.to_string(), value);
-    }
-
-    /// Return size of dictionary
-    pub fn length(&self) -> usize {
-        self.entries.len()
-    }
-}
-
-/// A helper to help create dictionaries for you. It has the ability to insert values into the dictionary while maintaining the tags that need to be applied to the individual members
-#[derive(Debug, Clone)]
-pub struct TaggedDictBuilder {
-    tag: Tag,
-    dict: IndexMap<String, Value>,
-}
-
-impl TaggedDictBuilder {
-    /// Create a new builder
-    pub fn new(tag: impl Into<Tag>) -> TaggedDictBuilder {
-        TaggedDictBuilder {
-            tag: tag.into(),
-            dict: IndexMap::default(),
-        }
-    }
-
-    /// Build the contents of the builder into a Value
-    pub fn build(tag: impl Into<Tag>, block: impl FnOnce(&mut TaggedDictBuilder)) -> Value {
-        let mut builder = TaggedDictBuilder::new(tag);
-        block(&mut builder);
-        builder.into_value()
-    }
-
-    /// Create a new builder with a pre-defined capacity
-    pub fn with_capacity(tag: impl Into<Tag>, n: usize) -> TaggedDictBuilder {
-        TaggedDictBuilder {
-            tag: tag.into(),
-            dict: IndexMap::with_capacity(n),
-        }
-    }
-
-    /// Insert an untagged key/value pair into the dictionary, to later be tagged when built
-    pub fn insert_untagged(&mut self, key: impl Into<String>, value: impl Into<UntaggedValue>) {
-        self.dict
-            .insert(key.into(), value.into().into_value(&self.tag));
-    }
-
-    ///  Insert a key/value pair into the dictionary
-    pub fn insert_value(&mut self, key: impl Into<String>, value: impl Into<Value>) {
-        self.dict.insert(key.into(), value.into());
-    }
-
-    /// Convert the dictionary into a tagged Value using the original tag
-    pub fn into_value(self) -> Value {
-        let tag = self.tag.clone();
-        self.into_untagged_value().into_value(tag)
-    }
-
-    /// Convert the dictionary into an UntaggedValue
-    pub fn into_untagged_value(self) -> UntaggedValue {
-        UntaggedValue::Row(Dictionary { entries: self.dict })
-    }
-
-    /// Returns true if the dictionary is empty, false otherwise
-    pub fn is_empty(&self) -> bool {
-        self.dict.is_empty()
-    }
-
-    /// Checks if given key exists
-    pub fn contains_key(&self, key: &str) -> bool {
-        self.dict.contains_key(key)
-    }
-}
-
-impl From<TaggedDictBuilder> for Value {
-    /// Convert a builder into a tagged Value
-    fn from(input: TaggedDictBuilder) -> Value {
-        input.into_value()
-    }
-}

crates/nu-protocol/src/value/did_you_mean.rs

@@ -1,132 +0,0 @@
-use crate::Value;
-
-/// Prepares a list of "sounds like" matches (using edit distance) for the string you're trying to find
-pub fn did_you_mean(obj_source: &Value, field_tried: String) -> Option<Vec<String>> {
-    let possibilities = obj_source.data_descriptors();
-
-    let mut possible_matches: Vec<_> = possibilities
-        .into_iter()
-        .map(|word| {
-            let edit_distance = levenshtein_distance(&word, &field_tried);
-            (edit_distance, word)
-        })
-        .collect();
-
-    if !possible_matches.is_empty() {
-        possible_matches.sort();
-        let words_matched: Vec<String> = possible_matches.into_iter().map(|m| m.1).collect();
-        Some(words_matched)
-    } else {
-        None
-    }
-}
-
-// Borrowed from here https://github.com/wooorm/levenshtein-rs
-pub fn levenshtein_distance(a: &str, b: &str) -> usize {
-    let mut result = 0;
-
-    /* Shortcut optimizations / degenerate cases. */
-    if a == b {
-        return result;
-    }
-
-    let length_a = a.chars().count();
-    let length_b = b.chars().count();
-
-    if length_a == 0 {
-        return length_b;
-    }
-
-    if length_b == 0 {
-        return length_a;
-    }
-
-    /* Initialize the vector.
-     *
-     * This is why it’s fast, normally a matrix is used,
-     * here we use a single vector. */
-    let mut cache: Vec<usize> = (1..).take(length_a).collect();
-    let mut distance_a;
-    let mut distance_b;
-
-    /* Loop. */
-    for (index_b, code_b) in b.chars().enumerate() {
-        result = index_b;
-        distance_a = index_b;
-
-        for (index_a, code_a) in a.chars().enumerate() {
-            distance_b = if code_a == code_b {
-                distance_a
-            } else {
-                distance_a + 1
-            };
-
-            distance_a = cache[index_a];
-
-            result = if distance_a > result {
-                if distance_b > result {
-                    result + 1
-                } else {
-                    distance_b
-                }
-            } else if distance_b > distance_a {
-                distance_a + 1
-            } else {
-                distance_b
-            };
-
-            cache[index_a] = result;
-        }
-    }
-
-    result
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::UntaggedValue;
-    use indexmap::indexmap;
-    use nu_source::Tag;
-
-    #[test]
-    fn did_you_mean_returns_possible_column_matches() {
-        let value = UntaggedValue::row(indexmap! {
-           "dog".to_string() => UntaggedValue::int(1).into(),
-           "cat".to_string() => UntaggedValue::int(1).into(),
-           "alt".to_string() => UntaggedValue::int(1).into(),
-        });
-
-        let source = Value {
-            tag: Tag::unknown(),
-            value,
-        };
-
-        assert_eq!(
-            Some(vec![
-                "cat".to_string(),
-                "alt".to_string(),
-                "dog".to_string()
-            ]),
-            did_you_mean(&source, "hat".to_string())
-        )
-    }
-
-    #[test]
-    fn did_you_mean_returns_no_matches_when_empty() {
-        let empty_source = Value {
-            tag: Tag::unknown(),
-            value: UntaggedValue::row(indexmap! {}),
-        };
-
-        assert_eq!(None, did_you_mean(&empty_source, "hat".to_string()))
-    }
-
-    #[test]
-    fn test_levenshtein_distance() {
-        assert_eq!(super::levenshtein_distance("hello world", "hello world"), 0);
-        assert_eq!(super::levenshtein_distance("hello", "hello world"), 6);
-        assert_eq!(super::levenshtein_distance("°C", "°C"), 0);
-        assert_eq!(super::levenshtein_distance("°", "°C"), 1);
-    }
-}

crates/nu-protocol/src/value/from.rs

@@ -0,0 +1,25 @@
+use crate::{ShellError, Value};
+
+impl Value {
+    pub fn as_f64(&self) -> Result<f64, ShellError> {
+        match self {
+            Value::Float { val, .. } => Ok(*val),
+            x => Err(ShellError::CantConvert(
+                "f64".into(),
+                x.get_type().to_string(),
+                self.span()?,
+            )),
+        }
+    }
+
+    pub fn as_i64(&self) -> Result<i64, ShellError> {
+        match self {
+            Value::Int { val, .. } => Ok(*val),
+            x => Err(ShellError::CantConvert(
+                "rf64".into(),
+                x.get_type().to_string(),
+                self.span()?,
+            )),
+        }
+    }
+}

crates/nu-protocol/src/value/from_value.rs

@@ -0,0 +1,404 @@
+use std::path::PathBuf;
+use std::str::FromStr;
+
+use crate::ast::{CellPath, PathMember};
+use crate::engine::CaptureBlock;
+use crate::ShellError;
+use crate::{Range, Spanned, Value};
+use chrono::{DateTime, FixedOffset};
+
+pub trait FromValue: Sized {
+    fn from_value(v: &Value) -> Result<Self, ShellError>;
+}
+
+impl FromValue for Value {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        Ok(v.clone())
+    }
+}
+
+impl FromValue for Spanned<i64> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Int { val, span } => Ok(Spanned {
+                item: *val,
+                span: *span,
+            }),
+            Value::Filesize { val, span } => Ok(Spanned {
+                item: *val as i64,
+                span: *span,
+            }),
+            Value::Duration { val, span } => Ok(Spanned {
+                item: *val as i64,
+                span: *span,
+            }),
+
+            v => Err(ShellError::CantConvert(
+                "integer".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for i64 {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Int { val, .. } => Ok(*val),
+            Value::Filesize { val, .. } => Ok(*val as i64),
+            Value::Duration { val, .. } => Ok(*val as i64),
+
+            v => Err(ShellError::CantConvert(
+                "integer".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<f64> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Int { val, span } => Ok(Spanned {
+                item: *val as f64,
+                span: *span,
+            }),
+            Value::Float { val, span } => Ok(Spanned {
+                item: *val,
+                span: *span,
+            }),
+
+            v => Err(ShellError::CantConvert(
+                "float".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for f64 {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Float { val, .. } => Ok(*val),
+            Value::Int { val, .. } => Ok(*val as f64),
+            v => Err(ShellError::CantConvert(
+                "float".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<usize> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Int { val, span } => Ok(Spanned {
+                item: *val as usize,
+                span: *span,
+            }),
+            Value::Filesize { val, span } => Ok(Spanned {
+                item: *val as usize,
+                span: *span,
+            }),
+            Value::Duration { val, span } => Ok(Spanned {
+                item: *val as usize,
+                span: *span,
+            }),
+
+            v => Err(ShellError::CantConvert(
+                "integer".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for usize {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Int { val, .. } => Ok(*val as usize),
+            Value::Filesize { val, .. } => Ok(*val as usize),
+            Value::Duration { val, .. } => Ok(*val as usize),
+
+            v => Err(ShellError::CantConvert(
+                "integer".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for String {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        // FIXME: we may want to fail a little nicer here
+        match v {
+            Value::CellPath { val, .. } => Ok(val.into_string()),
+            Value::String { val, .. } => Ok(val.clone()),
+            v => Err(ShellError::CantConvert(
+                "string".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<String> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        Ok(Spanned {
+            item: match v {
+                Value::CellPath { val, .. } => val.into_string(),
+                Value::String { val, .. } => val.clone(),
+                v => {
+                    return Err(ShellError::CantConvert(
+                        "string".into(),
+                        v.get_type().to_string(),
+                        v.span()?,
+                    ))
+                }
+            },
+            span: v.span()?,
+        })
+    }
+}
+
+impl FromValue for Vec<String> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        // FIXME: we may want to fail a little nicer here
+        match v {
+            Value::List { vals, .. } => vals
+                .iter()
+                .map(|val| match val {
+                    Value::String { val, .. } => Ok(val.clone()),
+                    c => Err(ShellError::CantConvert(
+                        "string".into(),
+                        c.get_type().to_string(),
+                        c.span()?,
+                    )),
+                })
+                .collect::<Result<Vec<String>, ShellError>>(),
+            v => Err(ShellError::CantConvert(
+                "string".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for CellPath {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        let span = v.span()?;
+        match v {
+            Value::CellPath { val, .. } => Ok(val.clone()),
+            Value::String { val, .. } => Ok(CellPath {
+                members: vec![PathMember::String {
+                    val: val.clone(),
+                    span,
+                }],
+            }),
+            Value::Int { val, .. } => Ok(CellPath {
+                members: vec![PathMember::Int {
+                    val: *val as usize,
+                    span,
+                }],
+            }),
+            x => Err(ShellError::CantConvert(
+                "cell path".into(),
+                x.get_type().to_string(),
+                span,
+            )),
+        }
+    }
+}
+
+impl FromValue for bool {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Bool { val, .. } => Ok(*val),
+            v => Err(ShellError::CantConvert(
+                "bool".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<bool> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Bool { val, span } => Ok(Spanned {
+                item: *val,
+                span: *span,
+            }),
+            v => Err(ShellError::CantConvert(
+                "bool".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for DateTime<FixedOffset> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Date { val, .. } => Ok(*val),
+            v => Err(ShellError::CantConvert(
+                "date".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<DateTime<FixedOffset>> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Date { val, span } => Ok(Spanned {
+                item: *val,
+                span: *span,
+            }),
+            v => Err(ShellError::CantConvert(
+                "date".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Range {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Range { val, .. } => Ok((**val).clone()),
+            v => Err(ShellError::CantConvert(
+                "range".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<Range> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Range { val, span } => Ok(Spanned {
+                item: (**val).clone(),
+                span: *span,
+            }),
+            v => Err(ShellError::CantConvert(
+                "range".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Vec<u8> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Binary { val, .. } => Ok(val.clone()),
+            Value::String { val, .. } => Ok(val.bytes().collect()),
+            v => Err(ShellError::CantConvert(
+                "binary data".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<PathBuf> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::String { val, span } => Ok(Spanned {
+                item: PathBuf::from_str(val)
+                    .map_err(|err| ShellError::FileNotFoundCustom(err.to_string(), *span))?,
+                span: *span,
+            }),
+            v => Err(ShellError::CantConvert(
+                "range".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Vec<Value> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        // FIXME: we may want to fail a little nicer here
+        match v {
+            Value::List { vals, .. } => Ok(vals.clone()),
+            v => Err(ShellError::CantConvert(
+                "Vector of values".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+// A record
+impl FromValue for (Vec<String>, Vec<Value>) {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Record { cols, vals, .. } => Ok((cols.clone(), vals.clone())),
+            v => Err(ShellError::CantConvert(
+                "Record".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for CaptureBlock {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Block { val, captures, .. } => Ok(CaptureBlock {
+                block_id: *val,
+                captures: captures.clone(),
+            }),
+            v => Err(ShellError::CantConvert(
+                "Block".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<CaptureBlock> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Block {
+                val,
+                captures,
+                span,
+            } => Ok(Spanned {
+                item: CaptureBlock {
+                    block_id: *val,
+                    captures: captures.clone(),
+                },
+                span: *span,
+            }),
+            v => Err(ShellError::CantConvert(
+                "Block".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}

crates/nu-protocol/src/value/iter.rs

@@ -1,52 +0,0 @@
-use crate::value::{UntaggedValue, Value};
-
-#[derive(Debug)]
-pub enum RowValueIter<'a> {
-    Empty,
-    Entries(indexmap::map::Iter<'a, String, Value>),
-}
-
-#[derive(Debug)]
-pub enum TableValueIter<'a> {
-    Empty,
-    Entries(std::slice::Iter<'a, Value>),
-}
-
-impl<'a> Iterator for RowValueIter<'a> {
-    type Item = (&'a String, &'a Value);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self {
-            RowValueIter::Empty => None,
-            RowValueIter::Entries(iter) => iter.next(),
-        }
-    }
-}
-
-impl<'a> Iterator for TableValueIter<'a> {
-    type Item = &'a Value;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self {
-            TableValueIter::Empty => None,
-            TableValueIter::Entries(iter) => iter.next(),
-        }
-    }
-}
-
-pub fn table_entries(value: &Value) -> TableValueIter<'_> {
-    match &value.value {
-        UntaggedValue::Table(t) => TableValueIter::Entries(t.iter()),
-        _ => TableValueIter::Empty,
-    }
-}
-
-pub fn row_entries(value: &Value) -> RowValueIter<'_> {
-    match &value.value {
-        UntaggedValue::Row(o) => {
-            let iter = o.entries.iter();
-            RowValueIter::Entries(iter)
-        }
-        _ => RowValueIter::Empty,
-    }
-}

crates/nu-protocol/src/value/primitive.rs

@@ -1,599 +0,0 @@
-use crate::type_name::ShellTypeName;
-use crate::value::column_path::ColumnPath;
-use crate::value::range::{Range, RangeInclusion};
-use crate::value::{serde_bigdecimal, serde_bigint};
-use bigdecimal::BigDecimal;
-use chrono::{DateTime, FixedOffset};
-use chrono_humanize::HumanTime;
-use nu_errors::{ExpectedRange, ShellError};
-use nu_source::{PrettyDebug, Span, SpannedItem};
-use num_bigint::BigInt;
-use num_integer::Integer;
-use num_traits::cast::{FromPrimitive, ToPrimitive};
-use num_traits::identities::Zero;
-use num_traits::sign::Signed;
-use serde::{Deserialize, Serialize};
-use std::path::PathBuf;
-
-const NANOS_PER_SEC: u32 = 1_000_000_000;
-
-/// The most fundamental of structured values in Nu are the Primitive values. These values represent types like integers, strings, booleans, dates, etc
-/// that are then used as the building blocks of more complex structures.
-///
-/// Primitives also include marker values BeginningOfStream and EndOfStream which denote a change of condition in the stream
-#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Deserialize, Serialize)]
-pub enum Primitive {
-    /// An empty value
-    Nothing,
-    /// A common integer
-    Int(i64),
-    /// A "big int", an integer with arbitrarily large size (aka not limited to 64-bit)
-    #[serde(with = "serde_bigint")]
-    BigInt(BigInt),
-    /// A "big decimal", an decimal number with arbitrarily large size (aka not limited to 64-bit)
-    #[serde(with = "serde_bigdecimal")]
-    Decimal(BigDecimal),
-    /// A count in the number of bytes, used as a filesize
-    Filesize(u64),
-    /// A string value
-    String(String),
-    /// A path to travel to reach a value in a table
-    ColumnPath(ColumnPath),
-    /// A glob pattern, eg foo*
-    GlobPattern(String),
-    /// A boolean value
-    Boolean(bool),
-    /// A date value
-    Date(DateTime<FixedOffset>),
-    /// A count in the number of nanoseconds
-    #[serde(with = "serde_bigint")]
-    Duration(BigInt),
-    /// A range of values
-    Range(Box<Range>),
-    /// A file path
-    FilePath(PathBuf),
-    /// A vector of raw binary data
-    #[serde(with = "serde_bytes")]
-    Binary(Vec<u8>),
-
-    /// Beginning of stream marker, a pseudo-value not intended for tables
-    BeginningOfStream,
-    /// End of stream marker, a pseudo-value not intended for tables
-    EndOfStream,
-}
-
-impl Primitive {
-    /// Converts a primitive value to a char, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_char(&self, span: Span) -> Result<char, ShellError> {
-        match self {
-            Primitive::String(s) => {
-                if s.len() > 1 {
-                    return Err(ShellError::type_error(
-                        "char",
-                        self.type_name().spanned(span),
-                    ));
-                }
-                s.chars()
-                    .next()
-                    .ok_or_else(|| ShellError::type_error("char", self.type_name().spanned(span)))
-            }
-            other => Err(ShellError::type_error(
-                "char",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Converts a primitive value to a u64, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_usize(&self, span: Span) -> Result<usize, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_usize().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U64,
-                    &int.to_string().spanned(span),
-                    "converting an integer into an unsigned 64-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_usize().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U64,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into an unsigned 64-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Converts a primitive value to a u64, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_u64(&self, span: Span) -> Result<u64, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_u64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U64,
-                    &int.to_string().spanned(span),
-                    "converting an integer into an unsigned 64-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_u64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U64,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into an unsigned 64-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Converts a primitive value to a f64, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_f64(&self, span: Span) -> Result<f64, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_f64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::F64,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a 64-bit floating point",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_f64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::F64,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a 64-bit floating point",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Converts a primitive value to a i64, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_i64(&self, span: Span) -> Result<i64, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_i64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I64,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a signed 64-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_i64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I64,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a signed 64-bit integer",
-                )
-            }),
-            Primitive::Duration(duration) => duration.to_i64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I64,
-                    &duration.to_string().spanned(span),
-                    "converting a duration into a signed 64-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Converts a primitive value to a u32, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_u32(&self, span: Span) -> Result<u32, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_u32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U32,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a unsigned 32-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_u32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U32,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a unsigned 32-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    pub fn as_i32(&self, span: Span) -> Result<i32, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_i32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I32,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a signed 32-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_i32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I32,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a signed 32-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    pub fn as_i16(&self, span: Span) -> Result<i16, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_i16().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I16,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a signed 16-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_i16().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I16,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a signed 16-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    pub fn as_f32(&self, span: Span) -> Result<f32, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_f32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::F32,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a signed 32-bit float",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_f32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::F32,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a signed 32-bit float",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    // FIXME: This is a bad name, but no other way to differentiate with our own Duration.
-    pub fn into_chrono_duration(self, span: Span) -> Result<chrono::Duration, ShellError> {
-        match self {
-            Primitive::Duration(duration) => {
-                // Divide into seconds because BigInt can be larger than i64
-                let (secs, nanos) = duration.div_rem(
-                    &BigInt::from_u32(NANOS_PER_SEC)
-                        .expect("Internal error: conversion from u32 failed"),
-                );
-                let secs = match secs.to_i64() {
-                    //The duration crate doesn't accept seconds bigger than i64::MAX / 1000
-                    Some(secs) => match secs.checked_mul(1000) {
-                        Some(_) => secs,
-                        None => {
-                            return Err(ShellError::labeled_error(
-                                "Internal duration conversion overflow.",
-                                "duration overflow",
-                                span,
-                            ))
-                        }
-                    },
-                    None => {
-                        return Err(ShellError::labeled_error(
-                            "Internal duration conversion overflow.",
-                            "duration overflow",
-                            span,
-                        ))
-                    }
-                };
-                // This should never fail since NANOS_PER_SEC won't overflow
-                let nanos = nanos.to_i64().expect("Unexpected i64 overflow");
-                // This should also never fail since we are adding less than NANOS_PER_SEC.
-                chrono::Duration::seconds(secs)
-                    .checked_add(&chrono::Duration::nanoseconds(nanos))
-                    .ok_or_else(|| ShellError::unexpected("Unexpected duration overflow"))
-            }
-            other => Err(ShellError::type_error(
-                "duration",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    pub fn into_string(self, span: Span) -> Result<String, ShellError> {
-        match self {
-            Primitive::String(s) => Ok(s),
-            other => Err(ShellError::type_error(
-                "string",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Returns true if the value is empty
-    pub fn is_empty(&self) -> bool {
-        match self {
-            Primitive::Nothing => true,
-            Primitive::String(s) => s.is_empty(),
-            _ => false,
-        }
-    }
-}
-
-impl From<bool> for Primitive {
-    /// Helper to convert from boolean to a primitive
-    fn from(b: bool) -> Primitive {
-        Primitive::Boolean(b)
-    }
-}
-
-impl From<&str> for Primitive {
-    /// Helper to convert from string slices to a primitive
-    fn from(s: &str) -> Primitive {
-        Primitive::String(s.to_string())
-    }
-}
-
-impl From<String> for Primitive {
-    /// Helper to convert from Strings to a primitive
-    fn from(s: String) -> Primitive {
-        Primitive::String(s)
-    }
-}
-
-impl From<BigDecimal> for Primitive {
-    /// Helper to convert from decimals to a Primitive value
-    fn from(decimal: BigDecimal) -> Primitive {
-        Primitive::Decimal(decimal)
-    }
-}
-
-impl From<BigInt> for Primitive {
-    /// Helper to convert from integers to a Primitive value
-    fn from(int: BigInt) -> Primitive {
-        Primitive::BigInt(int)
-    }
-}
-
-// Macro to define the From trait for native types to primitives
-// The from trait requires a converter that will be applied to the
-// native type.
-macro_rules! from_native_to_primitive {
-    ($native_type:ty, $primitive_type:expr, $converter: expr) => {
-        // e.g. from u32 -> Primitive
-        impl From<$native_type> for Primitive {
-            fn from(value: $native_type) -> Primitive {
-                if let Some(i) = $converter(value) {
-                    $primitive_type(i)
-                } else {
-                    unreachable!("Internal error: protocol did not use compatible decimal")
-                }
-            }
-        }
-    };
-}
-
-from_native_to_primitive!(i8, Primitive::Int, i64::from_i8);
-from_native_to_primitive!(i16, Primitive::Int, i64::from_i16);
-from_native_to_primitive!(i32, Primitive::Int, i64::from_i32);
-from_native_to_primitive!(i64, Primitive::Int, i64::from_i64);
-from_native_to_primitive!(u8, Primitive::Int, i64::from_u8);
-from_native_to_primitive!(u16, Primitive::Int, i64::from_u16);
-from_native_to_primitive!(u32, Primitive::Int, i64::from_u32);
-from_native_to_primitive!(u64, Primitive::BigInt, BigInt::from_u64);
-from_native_to_primitive!(f32, Primitive::Decimal, BigDecimal::from_f32);
-from_native_to_primitive!(f64, Primitive::Decimal, BigDecimal::from_f64);
-
-impl From<chrono::Duration> for Primitive {
-    fn from(duration: chrono::Duration) -> Primitive {
-        // FIXME: This is a hack since chrono::Duration does not give access to its 'nanos' field.
-        let secs: i64 = duration.num_seconds();
-        // This will never fail.
-        let nanos: u32 = duration
-            .checked_sub(&chrono::Duration::seconds(secs))
-            .expect("Unexpected overflow")
-            .num_nanoseconds()
-            .expect("Unexpected overflow") as u32;
-        Primitive::Duration(
-            BigInt::from_i64(secs * NANOS_PER_SEC as i64 + nanos as i64)
-                .expect("Internal error: can't convert from i64"),
-        )
-    }
-}
-
-impl std::fmt::Display for Primitive {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "{:?}", self)
-    }
-}
-
-impl ShellTypeName for Primitive {
-    /// Get the name of the type of a Primitive value
-    fn type_name(&self) -> &'static str {
-        match self {
-            Primitive::Nothing => "nothing",
-            Primitive::Int(_) => "integer",
-            Primitive::BigInt(_) => "big integer",
-            Primitive::Range(_) => "range",
-            Primitive::Decimal(_) => "decimal",
-            Primitive::Filesize(_) => "filesize(in bytes)",
-            Primitive::String(_) => "string",
-            Primitive::ColumnPath(_) => "column path",
-            Primitive::GlobPattern(_) => "pattern",
-            Primitive::Boolean(_) => "boolean",
-            Primitive::Date(_) => "date",
-            Primitive::Duration(_) => "duration",
-            Primitive::FilePath(_) => "file path",
-            Primitive::Binary(_) => "binary",
-            Primitive::BeginningOfStream => "marker<beginning of stream>",
-            Primitive::EndOfStream => "marker<end of stream>",
-        }
-    }
-}
-
-/// Format a Primitive value into a string
-pub fn format_primitive(primitive: &Primitive, field_name: Option<&String>) -> String {
-    match primitive {
-        Primitive::Nothing => String::new(),
-        Primitive::BeginningOfStream => String::new(),
-        Primitive::EndOfStream => String::new(),
-        Primitive::FilePath(p) => p.display().to_string(),
-        Primitive::Filesize(num_bytes) => {
-            if let Some(value) = num_bytes.to_u128() {
-                let byte = byte_unit::Byte::from_bytes(value);
-
-                if byte.get_bytes() == 0u128 {
-                    return "—".to_string();
-                }
-
-                let byte = byte.get_appropriate_unit(false);
-
-                match byte.get_unit() {
-                    byte_unit::ByteUnit::B => format!("{} B ", byte.get_value()),
-                    _ => byte.format(1),
-                }
-            } else {
-                format!("{} B", num_bytes)
-            }
-        }
-        Primitive::Duration(duration) => format_duration(duration),
-        Primitive::Int(i) => i.to_string(),
-        Primitive::BigInt(i) => i.to_string(),
-        Primitive::Decimal(decimal) => {
-            // TODO: We should really pass the precision in here instead of hard coding it
-            let decimal_string = decimal.to_string();
-            let decimal_places: Vec<&str> = decimal_string.split('.').collect();
-            if decimal_places.len() == 2 && decimal_places[1].len() > 4 {
-                format!("{:.4}", decimal)
-            } else {
-                decimal.to_string()
-            }
-        }
-        Primitive::Range(range) => format!(
-            "{}..{}{}",
-            format_primitive(&range.from.0.item, None),
-            if range.to.1 == RangeInclusion::Exclusive {
-                "<"
-            } else {
-                ""
-            },
-            format_primitive(&range.to.0.item, None)
-        ),
-        Primitive::GlobPattern(s) => s.to_string(),
-        Primitive::String(s) => s.to_owned(),
-        Primitive::ColumnPath(p) => {
-            let mut members = p.iter();
-            let mut f = String::new();
-
-            f.push_str(
-                &members
-                    .next()
-                    .expect("BUG: column path with zero members")
-                    .display(),
-            );
-
-            for member in members {
-                f.push('.');
-                f.push_str(&member.display())
-            }
-
-            f
-        }
-        Primitive::Boolean(b) => match (b, field_name) {
-            (true, None) => "true",
-            (false, None) => "false",
-            (true, Some(s)) if !s.is_empty() => s,
-            (false, Some(s)) if !s.is_empty() => "",
-            (true, Some(_)) => "true",
-            (false, Some(_)) => "false",
-        }
-        .to_owned(),
-        Primitive::Binary(_) => "<binary>".to_owned(),
-        Primitive::Date(d) => format_date(d),
-    }
-}
-
-/// Format a duration in nanoseconds into a string
-pub fn format_duration(duration: &BigInt) -> String {
-    let is_zero = duration.is_zero();
-    // FIXME: This involves a lot of allocation, but it seems inevitable with BigInt.
-    let big_int_1000 = BigInt::from(1000);
-    let big_int_60 = BigInt::from(60);
-    let big_int_24 = BigInt::from(24);
-    // We only want the biggest subdivision to have the negative sign.
-    let (sign, duration) = if duration.is_zero() || duration.is_positive() {
-        (1, duration.clone())
-    } else {
-        (-1, -duration)
-    };
-    let (micros, nanos): (BigInt, BigInt) = duration.div_rem(&big_int_1000);
-    let (millis, micros): (BigInt, BigInt) = micros.div_rem(&big_int_1000);
-    let (secs, millis): (BigInt, BigInt) = millis.div_rem(&big_int_1000);
-    let (mins, secs): (BigInt, BigInt) = secs.div_rem(&big_int_60);
-    let (hours, mins): (BigInt, BigInt) = mins.div_rem(&big_int_60);
-    let (days, hours): (BigInt, BigInt) = hours.div_rem(&big_int_24);
-
-    let mut output_prep = vec![];
-
-    if !days.is_zero() {
-        output_prep.push(format!("{}day", days));
-    }
-
-    if !hours.is_zero() {
-        output_prep.push(format!("{}hr", hours));
-    }
-
-    if !mins.is_zero() {
-        output_prep.push(format!("{}min", mins));
-    }
-    // output 0sec for zero duration
-    if is_zero || !secs.is_zero() {
-        output_prep.push(format!("{}sec", secs));
-    }
-
-    if !millis.is_zero() {
-        output_prep.push(format!("{}ms", millis));
-    }
-
-    if !micros.is_zero() {
-        output_prep.push(format!("{}us", micros));
-    }
-
-    if !nanos.is_zero() {
-        output_prep.push(format!("{}ns", nanos));
-    }
-
-    format!(
-        "{}{}",
-        if sign == -1 { "-" } else { "" },
-        output_prep.join(" ")
-    )
-}
-
-/// Format a date value into a humanized string (eg "1 week ago" instead of a formal date string)
-pub fn format_date(d: &DateTime<FixedOffset>) -> String {
-    HumanTime::from(*d).to_string()
-}

crates/nu-protocol/src/value/range.rs

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 use crate::value::Primitive;
 use derive_new::new;
 use nu_errors::ShellError;
@@ -150,5 +151,225 @@ impl Range {
         }
 
         // How would inclusive vs. exclusive range work here?
+=======
+use serde::{Deserialize, Serialize};
+use std::cmp::Ordering;
+
+/// A Range is an iterator over integers.
+use crate::{
+    ast::{RangeInclusion, RangeOperator},
+    *,
+};
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Range {
+    pub from: Value,
+    pub incr: Value,
+    pub to: Value,
+    pub inclusion: RangeInclusion,
+}
+
+impl Range {
+    pub fn new(
+        expr_span: Span,
+        from: Value,
+        next: Value,
+        to: Value,
+        operator: &RangeOperator,
+    ) -> Result<Range, ShellError> {
+        // Select from & to values if they're not specified
+        // TODO: Replace the placeholder values with proper min/max for range based on data type
+        let from = if let Value::Nothing { .. } = from {
+            Value::Int {
+                val: 0i64,
+                span: expr_span,
+            }
+        } else {
+            from
+        };
+
+        let to = if let Value::Nothing { .. } = to {
+            if let Ok(Value::Bool { val: true, .. }) = next.lt(expr_span, &from) {
+                Value::Int {
+                    val: -100i64,
+                    span: expr_span,
+                }
+            } else {
+                Value::Int {
+                    val: 100i64,
+                    span: expr_span,
+                }
+            }
+        } else {
+            to
+        };
+
+        // Check if the range counts up or down
+        let moves_up = matches!(from.lte(expr_span, &to), Ok(Value::Bool { val: true, .. }));
+
+        // Convert the next value into the inctement
+        let incr = if let Value::Nothing { .. } = next {
+            if moves_up {
+                Value::Int {
+                    val: 1i64,
+                    span: expr_span,
+                }
+            } else {
+                Value::Int {
+                    val: -1i64,
+                    span: expr_span,
+                }
+            }
+        } else {
+            next.sub(operator.next_op_span, &from)?
+        };
+
+        let zero = Value::Int {
+            val: 0i64,
+            span: expr_span,
+        };
+
+        // Increment must be non-zero, otherwise we iterate forever
+        if matches!(incr.eq(expr_span, &zero), Ok(Value::Bool { val: true, .. })) {
+            return Err(ShellError::CannotCreateRange(expr_span));
+        }
+
+        // If to > from, then incr > 0, otherwise we iterate forever
+        if let (Value::Bool { val: true, .. }, Value::Bool { val: false, .. }) = (
+            to.gt(operator.span, &from)?,
+            incr.gt(operator.next_op_span, &zero)?,
+        ) {
+            return Err(ShellError::CannotCreateRange(expr_span));
+        }
+
+        // If to < from, then incr < 0, otherwise we iterate forever
+        if let (Value::Bool { val: true, .. }, Value::Bool { val: false, .. }) = (
+            to.lt(operator.span, &from)?,
+            incr.lt(operator.next_op_span, &zero)?,
+        ) {
+            return Err(ShellError::CannotCreateRange(expr_span));
+        }
+
+        Ok(Range {
+            from,
+            incr,
+            to,
+            inclusion: operator.inclusion,
+        })
+    }
+
+    #[inline]
+    fn moves_up(&self) -> bool {
+        self.from <= self.to
+    }
+
+    #[inline]
+    fn is_end_inclusive(&self) -> bool {
+        matches!(self.inclusion, RangeInclusion::Inclusive)
+    }
+
+    pub fn contains(&self, item: &Value) -> bool {
+        match (item.partial_cmp(&self.from), item.partial_cmp(&self.to)) {
+            (Some(Ordering::Greater | Ordering::Equal), Some(Ordering::Less)) => self.moves_up(),
+            (Some(Ordering::Less | Ordering::Equal), Some(Ordering::Greater)) => !self.moves_up(),
+            (Some(_), Some(Ordering::Equal)) => self.is_end_inclusive(),
+            (_, _) => false,
+        }
+    }
+
+    pub fn into_range_iter(self) -> Result<RangeIterator, ShellError> {
+        let span = self.from.span()?;
+
+        Ok(RangeIterator::new(self, span))
+    }
+}
+
+pub struct RangeIterator {
+    curr: Value,
+    end: Value,
+    span: Span,
+    is_end_inclusive: bool,
+    moves_up: bool,
+    incr: Value,
+    done: bool,
+}
+
+impl RangeIterator {
+    pub fn new(range: Range, span: Span) -> RangeIterator {
+        let moves_up = range.moves_up();
+        let is_end_inclusive = range.is_end_inclusive();
+
+        let start = match range.from {
+            Value::Nothing { .. } => Value::Int { val: 0, span },
+            x => x,
+        };
+
+        let end = match range.to {
+            Value::Nothing { .. } => Value::Int {
+                val: i64::MAX,
+                span,
+            },
+            x => x,
+        };
+
+        RangeIterator {
+            moves_up,
+            curr: start,
+            end,
+            span,
+            is_end_inclusive,
+            done: false,
+            incr: range.incr,
+        }
+    }
+}
+
+impl Iterator for RangeIterator {
+    type Item = Value;
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.done {
+            return None;
+        }
+
+        let ordering = if matches!(self.end, Value::Nothing { .. }) {
+            Some(Ordering::Less)
+        } else {
+            self.curr.partial_cmp(&self.end)
+        };
+
+        let ordering = if let Some(ord) = ordering {
+            ord
+        } else {
+            self.done = true;
+            return Some(Value::Error {
+                error: ShellError::CannotCreateRange(self.span),
+            });
+        };
+
+        let desired_ordering = if self.moves_up {
+            Ordering::Less
+        } else {
+            Ordering::Greater
+        };
+
+        if (ordering == desired_ordering) || (self.is_end_inclusive && ordering == Ordering::Equal)
+        {
+            let next_value = self.curr.add(self.span, &self.incr);
+
+            let mut next = match next_value {
+                Ok(result) => result,
+
+                Err(error) => {
+                    self.done = true;
+                    return Some(Value::Error { error });
+                }
+            };
+            std::mem::swap(&mut self.curr, &mut next);
+
+            Some(next)
+        } else {
+            None
+        }
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
     }
 }

crates/nu-protocol/src/value/serde_bigdecimal.rs

@@ -1,19 +0,0 @@
-use bigdecimal::BigDecimal;
-
-/// Enable big decimal serialization by providing a `serialize` function
-pub fn serialize<S>(big_decimal: &BigDecimal, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: serde::Serializer,
-{
-    serde::Serialize::serialize(&big_decimal.to_string(), serializer)
-}
-
-/// Enable big decimal deserialization by providing a `deserialize` function
-pub fn deserialize<'de, D>(deserializer: D) -> Result<BigDecimal, D::Error>
-where
-    D: serde::Deserializer<'de>,
-{
-    let x: String = serde::Deserialize::deserialize(deserializer)?;
-    BigDecimal::parse_bytes(x.as_bytes(), 10)
-        .ok_or_else(|| serde::de::Error::custom("expected a bigdecimal"))
-}

crates/nu-protocol/src/value/serde_bigint.rs

@@ -1,20 +0,0 @@
-use num_bigint::BigInt;
-
-/// Enable big int serialization by providing a `serialize` function
-pub fn serialize<S>(big_int: &BigInt, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: serde::Serializer,
-{
-    serde::Serialize::serialize(&big_int.to_string(), serializer)
-}
-
-/// Enable big int deserialization by providing a `deserialize` function
-pub fn deserialize<'de, D>(deserializer: D) -> Result<BigInt, D::Error>
-where
-    D: serde::Deserializer<'de>,
-{
-    let x: String = serde::Deserialize::deserialize(deserializer)?;
-
-    BigInt::parse_bytes(x.as_bytes(), 10)
-        .ok_or_else(|| serde::de::Error::custom("expected a bignum"))
-}

crates/nu-protocol/src/value/stream.rs

@@ -0,0 +1,204 @@
+use crate::*;
+use std::{
+    fmt::Debug,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+};
+
+pub struct RawStream {
+    pub stream: Box<dyn Iterator<Item = Result<Vec<u8>, ShellError>> + Send + 'static>,
+    pub leftover: Vec<u8>,
+    pub ctrlc: Option<Arc<AtomicBool>>,
+    pub is_binary: bool,
+    pub span: Span,
+}
+
+impl RawStream {
+    pub fn new(
+        stream: Box<dyn Iterator<Item = Result<Vec<u8>, ShellError>> + Send + 'static>,
+        ctrlc: Option<Arc<AtomicBool>>,
+        span: Span,
+    ) -> Self {
+        Self {
+            stream,
+            leftover: vec![],
+            ctrlc,
+            is_binary: false,
+            span,
+        }
+    }
+
+    pub fn into_bytes(self) -> Result<Spanned<Vec<u8>>, ShellError> {
+        let mut output = vec![];
+
+        for item in self.stream {
+            output.extend(item?);
+        }
+
+        Ok(Spanned {
+            item: output,
+            span: self.span,
+        })
+    }
+
+    pub fn into_string(self) -> Result<Spanned<String>, ShellError> {
+        let mut output = String::new();
+        let span = self.span;
+
+        for item in self {
+            output.push_str(&item?.as_string()?);
+        }
+
+        Ok(Spanned { item: output, span })
+    }
+}
+impl Debug for RawStream {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RawStream").finish()
+    }
+}
+impl Iterator for RawStream {
+    type Item = Result<Value, ShellError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // If we know we're already binary, just output that
+        if self.is_binary {
+            match self.stream.next() {
+                Some(buffer) => match buffer {
+                    Ok(mut v) => {
+                        if !self.leftover.is_empty() {
+                            while let Some(b) = self.leftover.pop() {
+                                v.insert(0, b);
+                            }
+                        }
+                        Some(Ok(Value::Binary {
+                            val: v,
+                            span: self.span,
+                        }))
+                    }
+                    Err(e) => Some(Err(e)),
+                },
+                None => None,
+            }
+        } else {
+            // We *may* be text. We're only going to try utf-8. Other decodings
+            // needs to be taken as binary first, then passed through `decode`.
+            match self.stream.next() {
+                Some(buffer) => match buffer {
+                    Ok(mut v) => {
+                        if !self.leftover.is_empty() {
+                            while let Some(b) = self.leftover.pop() {
+                                v.insert(0, b);
+                            }
+                        }
+
+                        match String::from_utf8(v.clone()) {
+                            Ok(s) => {
+                                // Great, we have a complete string, let's output it
+                                Some(Ok(Value::String {
+                                    val: s,
+                                    span: self.span,
+                                }))
+                            }
+                            Err(err) => {
+                                // Okay, we *might* have a string but we've also got some errors
+                                if v.is_empty() {
+                                    // We can just end here
+                                    None
+                                } else if v.len() > 3
+                                    && (v.len() - err.utf8_error().valid_up_to() > 3)
+                                {
+                                    // As UTF-8 characters are max 4 bytes, if we have more than that in error we know
+                                    // that it's not just a character spanning two frames.
+                                    // We now know we are definitely binary, so switch to binary and stay there.
+                                    self.is_binary = true;
+                                    Some(Ok(Value::Binary {
+                                        val: v,
+                                        span: self.span,
+                                    }))
+                                } else {
+                                    // Okay, we have a tiny bit of error at the end of the buffer. This could very well be
+                                    // a character that spans two frames. Since this is the case, remove the error from
+                                    // the current frame an dput it in the leftover buffer.
+                                    self.leftover = v[err.utf8_error().valid_up_to()..].to_vec();
+
+                                    let buf = v[0..err.utf8_error().valid_up_to()].to_vec();
+
+                                    match String::from_utf8(buf) {
+                                        Ok(s) => Some(Ok(Value::String {
+                                            val: s,
+                                            span: self.span,
+                                        })),
+                                        Err(_) => {
+                                            // Something is definitely wrong. Switch to binary, and stay there
+                                            self.is_binary = true;
+                                            Some(Ok(Value::Binary {
+                                                val: v,
+                                                span: self.span,
+                                            }))
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    Err(e) => Some(Err(e)),
+                },
+                None => None,
+            }
+        }
+    }
+}
+
+/// A potentially infinite stream of values, optinally with a mean to send a Ctrl-C signal to stop
+/// the stream from continuing.
+///
+/// In practice, a "stream" here means anything which can be iterated and produce Values as it iterates.
+/// Like other iterators in Rust, observing values from this stream will drain the items as you view them
+/// and the stream cannot be replayed.
+pub struct ListStream {
+    pub stream: Box<dyn Iterator<Item = Value> + Send + 'static>,
+    pub ctrlc: Option<Arc<AtomicBool>>,
+}
+
+impl ListStream {
+    pub fn into_string(self, separator: &str, config: &Config) -> String {
+        self.map(|x: Value| x.into_string(", ", config))
+            .collect::<Vec<String>>()
+            .join(separator)
+    }
+
+    pub fn from_stream(
+        input: impl Iterator<Item = Value> + Send + 'static,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> ListStream {
+        ListStream {
+            stream: Box::new(input),
+            ctrlc,
+        }
+    }
+}
+
+impl Debug for ListStream {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ValueStream").finish()
+    }
+}
+
+impl Iterator for ListStream {
+    type Item = Value;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(ctrlc) = &self.ctrlc {
+            if ctrlc.load(Ordering::SeqCst) {
+                None
+            } else {
+                self.stream.next()
+            }
+        } else {
+            self.stream.next()
+        }
+    }
+}

crates/nu-protocol/src/value/unit.rs

@@ -0,0 +1,27 @@
+#[derive(Debug, Clone, Copy)]
+pub enum Unit {
+    // Filesize units: metric
+    Byte,
+    Kilobyte,
+    Megabyte,
+    Gigabyte,
+    Terabyte,
+    Petabyte,
+
+    // Filesize units: ISO/IEC 80000
+    Kibibyte,
+    Mebibyte,
+    Gibibyte,
+    Tebibyte,
+    Pebibyte,
+
+    // Duration units
+    Nanosecond,
+    Microsecond,
+    Millisecond,
+    Second,
+    Minute,
+    Hour,
+    Day,
+    Week,
+}

crates/nu-protocol/src/value/value_structure.rs

@@ -1,83 +0,0 @@
-use crate::{UntaggedValue, Value};
-use nu_errors::ShellError;
-use std::path::{Component, Path, PathBuf};
-
-fn is_value_tagged_dir(value: &Value) -> bool {
-    matches!(
-        &value.value,
-        UntaggedValue::Row(_) | UntaggedValue::Table(_)
-    )
-}
-
-#[derive(Debug, Eq, Ord, PartialEq, PartialOrd)]
-pub struct ValueResource {
-    pub at: usize,
-    pub loc: PathBuf,
-}
-
-impl ValueResource {}
-
-#[derive(Default, Debug)]
-pub struct ValueStructure {
-    pub resources: Vec<ValueResource>,
-}
-
-impl ValueStructure {
-    pub fn new() -> ValueStructure {
-        ValueStructure {
-            resources: Vec::<ValueResource>::new(),
-        }
-    }
-
-    pub fn exists(&self, path: &Path) -> bool {
-        if path == Path::new("/") {
-            return true;
-        }
-
-        let path = if path.starts_with("/") {
-            path.strip_prefix("/").unwrap_or(path)
-        } else {
-            path
-        };
-
-        let comps: Vec<_> = path.components().map(Component::as_os_str).collect();
-
-        let mut is_there = true;
-
-        for (at, fragment) in comps.iter().enumerate() {
-            is_there = is_there
-                && self
-                    .resources
-                    .iter()
-                    .any(|resource| at == resource.at && *fragment == resource.loc.as_os_str());
-        }
-
-        is_there
-    }
-
-    pub fn walk_decorate(&mut self, start: &Value) -> Result<(), ShellError> {
-        self.resources = Vec::<ValueResource>::new();
-        self.build(start, 0)?;
-        self.resources.sort();
-
-        Ok(())
-    }
-
-    fn build(&mut self, src: &Value, lvl: usize) -> Result<(), ShellError> {
-        for entry in src.row_entries() {
-            let value = entry.1;
-            let path = entry.0;
-
-            self.resources.push(ValueResource {
-                at: lvl,
-                loc: PathBuf::from(path),
-            });
-
-            if is_value_tagged_dir(value) {
-                self.build(value, lvl + 1)?;
-            }
-        }
-
-        Ok(())
-    }
-}

crates/nu-protocol/tests/test_signature.rs

@@ -0,0 +1,170 @@
+use nu_protocol::{Flag, PositionalArg, Signature, SyntaxShape};
+
+#[test]
+fn test_signature() {
+    let signature = Signature::new("new_signature");
+    let from_build = Signature::build("new_signature");
+
+    // asserting partial eq implementation
+    assert_eq!(signature, from_build);
+
+    // constructing signature with description
+    let signature = Signature::new("signature").desc("example usage");
+    assert_eq!(signature.usage, "example usage".to_string())
+}
+
+#[test]
+fn test_signature_chained() {
+    let signature = Signature::new("new_signature")
+        .desc("description")
+        .required("required", SyntaxShape::String, "required description")
+        .optional("optional", SyntaxShape::String, "optional description")
+        .required_named(
+            "req_named",
+            SyntaxShape::String,
+            "required named description",
+            Some('r'),
+        )
+        .named("named", SyntaxShape::String, "named description", Some('n'))
+        .switch("switch", "switch description", None)
+        .rest("rest", SyntaxShape::String, "rest description");
+
+    assert_eq!(signature.required_positional.len(), 1);
+    assert_eq!(signature.optional_positional.len(), 1);
+    assert_eq!(signature.named.len(), 4); // The 3 above + help
+    assert!(signature.rest_positional.is_some());
+    assert_eq!(signature.get_shorts(), vec!['h', 'r', 'n']);
+    assert_eq!(
+        signature.get_names(),
+        vec!["help", "req_named", "named", "switch"]
+    );
+    assert_eq!(signature.num_positionals(), 2);
+
+    assert_eq!(
+        signature.get_positional(0),
+        Some(PositionalArg {
+            name: "required".to_string(),
+            desc: "required description".to_string(),
+            shape: SyntaxShape::String,
+            var_id: None
+        })
+    );
+    assert_eq!(
+        signature.get_positional(1),
+        Some(PositionalArg {
+            name: "optional".to_string(),
+            desc: "optional description".to_string(),
+            shape: SyntaxShape::String,
+            var_id: None
+        })
+    );
+    assert_eq!(
+        signature.get_positional(2),
+        Some(PositionalArg {
+            name: "rest".to_string(),
+            desc: "rest description".to_string(),
+            shape: SyntaxShape::String,
+            var_id: None
+        })
+    );
+
+    assert_eq!(
+        signature.get_long_flag("req_named"),
+        Some(Flag {
+            long: "req_named".to_string(),
+            short: Some('r'),
+            arg: Some(SyntaxShape::String),
+            required: true,
+            desc: "required named description".to_string(),
+            var_id: None
+        })
+    );
+
+    assert_eq!(
+        signature.get_short_flag('r'),
+        Some(Flag {
+            long: "req_named".to_string(),
+            short: Some('r'),
+            arg: Some(SyntaxShape::String),
+            required: true,
+            desc: "required named description".to_string(),
+            var_id: None
+        })
+    );
+}
+
+#[test]
+#[should_panic(expected = "There may be duplicate short flags, such as -h")]
+fn test_signature_same_short() {
+    // Creating signature with same short name should panic
+    Signature::new("new_signature")
+        .required_named(
+            "required_named",
+            SyntaxShape::String,
+            "required named description",
+            Some('n'),
+        )
+        .named("named", SyntaxShape::String, "named description", Some('n'));
+}
+
+#[test]
+#[should_panic(expected = "There may be duplicate name flags, such as --help")]
+fn test_signature_same_name() {
+    // Creating signature with same short name should panic
+    Signature::new("new_signature")
+        .required_named(
+            "name",
+            SyntaxShape::String,
+            "required named description",
+            Some('r'),
+        )
+        .named("name", SyntaxShape::String, "named description", Some('n'));
+}
+
+#[test]
+fn test_signature_round_trip() {
+    let signature = Signature::new("new_signature")
+        .desc("description")
+        .required("first", SyntaxShape::String, "first required")
+        .required("second", SyntaxShape::Int, "second required")
+        .optional("optional", SyntaxShape::String, "optional description")
+        .required_named(
+            "req_named",
+            SyntaxShape::String,
+            "required named description",
+            Some('r'),
+        )
+        .named("named", SyntaxShape::String, "named description", Some('n'))
+        .switch("switch", "switch description", None)
+        .rest("rest", SyntaxShape::String, "rest description")
+        .category(nu_protocol::Category::Conversions);
+
+    let string = serde_json::to_string_pretty(&signature).unwrap();
+    let returned: Signature = serde_json::from_str(&string).unwrap();
+
+    assert_eq!(signature.name, returned.name);
+    assert_eq!(signature.usage, returned.usage);
+    assert_eq!(signature.extra_usage, returned.extra_usage);
+    assert_eq!(signature.is_filter, returned.is_filter);
+    assert_eq!(signature.category, returned.category);
+
+    signature
+        .required_positional
+        .iter()
+        .zip(returned.required_positional.iter())
+        .for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
+
+    signature
+        .optional_positional
+        .iter()
+        .zip(returned.optional_positional.iter())
+        .for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
+
+    signature
+        .named
+        .iter()
+        .zip(returned.named.iter())
+        .for_each(|(lhs, rhs)| assert_eq!(lhs, rhs));
+
+    assert_eq!(signature.rest_positional, returned.rest_positional,);
+}

crates/nu-protocol/tests/test_value.rs

@@ -0,0 +1,45 @@
+use nu_protocol::{Span, Value};
+
+#[test]
+fn test_comparison_nothing() {
+    let values = vec![
+        Value::Int {
+            val: 1,
+            span: Span::test_data(),
+        },
+        Value::String {
+            val: "string".into(),
+            span: Span::test_data(),
+        },
+        Value::Float {
+            val: 1.0,
+            span: Span::test_data(),
+        },
+    ];
+
+    let nothing = Value::Nothing {
+        span: Span::test_data(),
+    };
+
+    for value in values {
+        assert!(matches!(
+            value.eq(Span::test_data(), &nothing),
+            Ok(Value::Bool { val: false, .. })
+        ));
+
+        assert!(matches!(
+            value.ne(Span::test_data(), &nothing),
+            Ok(Value::Bool { val: true, .. })
+        ));
+
+        assert!(matches!(
+            nothing.eq(Span::test_data(), &value),
+            Ok(Value::Bool { val: false, .. })
+        ));
+
+        assert!(matches!(
+            nothing.ne(Span::test_data(), &value),
+            Ok(Value::Bool { val: true, .. })
+        ));
+    }
+}