engine-q merge

crates/nu-protocol/src/ast/block.rs

@@ -0,0 +1,71 @@
+use std::ops::{Index, IndexMut};
+
+use crate::{Signature, Span, VarId};
+
+use super::Statement;
+
+#[derive(Debug, Clone)]
+pub struct Block {
+    pub signature: Box<Signature>,
+    pub stmts: Vec<Statement>,
+    pub captures: Vec<VarId>,
+    pub redirect_env: bool,
+    pub span: Option<Span>, // None option encodes no span to avoid using test_span()
+}
+
+impl Block {
+    pub fn len(&self) -> usize {
+        self.stmts.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.stmts.is_empty()
+    }
+}
+
+impl Index<usize> for Block {
+    type Output = Statement;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        &self.stmts[index]
+    }
+}
+
+impl IndexMut<usize> for Block {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        &mut self.stmts[index]
+    }
+}
+
+impl Default for Block {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Block {
+    pub fn new() -> Self {
+        Self {
+            signature: Box::new(Signature::new("")),
+            stmts: vec![],
+            captures: vec![],
+            redirect_env: false,
+            span: None,
+        }
+    }
+}
+
+impl<T> From<T> for Block
+where
+    T: Iterator<Item = Statement>,
+{
+    fn from(stmts: T) -> Self {
+        Self {
+            signature: Box::new(Signature::new("")),
+            stmts: stmts.collect(),
+            captures: vec![],
+            redirect_env: false,
+            span: None,
+        }
+    }
+}

crates/nu-protocol/src/ast/call.rs

@@ -0,0 +1,56 @@
+use super::Expression;
+use crate::{DeclId, Span, Spanned};
+
+#[derive(Debug, Clone)]
+pub struct Call {
+    /// identifier of the declaration to call
+    pub decl_id: DeclId,
+    pub head: Span,
+    pub positional: Vec<Expression>,
+    pub named: Vec<(Spanned<String>, Option<Expression>)>,
+}
+
+impl Call {
+    pub fn new(head: Span) -> Call {
+        Self {
+            decl_id: 0,
+            head,
+            positional: vec![],
+            named: vec![],
+        }
+    }
+
+    pub fn has_flag(&self, flag_name: &str) -> bool {
+        for name in &self.named {
+            if flag_name == name.0.item {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    pub fn get_flag_expr(&self, flag_name: &str) -> Option<Expression> {
+        for name in &self.named {
+            if flag_name == name.0.item {
+                return name.1.clone();
+            }
+        }
+
+        None
+    }
+
+    pub fn get_named_arg(&self, flag_name: &str) -> Option<Spanned<String>> {
+        for name in &self.named {
+            if flag_name == name.0.item {
+                return Some(name.0.clone());
+            }
+        }
+
+        None
+    }
+
+    pub fn nth(&self, pos: usize) -> Option<Expression> {
+        self.positional.get(pos).cloned()
+    }
+}

crates/nu-protocol/src/ast/cell_path.rs

@@ -0,0 +1,48 @@
+use super::Expression;
+use crate::Span;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum PathMember {
+    String { val: String, span: Span },
+    Int { val: usize, span: Span },
+}
+
+impl PartialEq for PathMember {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Self::String { val: l_val, .. }, Self::String { val: r_val, .. }) => l_val == r_val,
+            (Self::Int { val: l_val, .. }, Self::Int { val: r_val, .. }) => l_val == r_val,
+            _ => false,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CellPath {
+    pub members: Vec<PathMember>,
+}
+
+impl CellPath {
+    pub fn into_string(&self) -> String {
+        let mut output = String::new();
+
+        for (idx, elem) in self.members.iter().enumerate() {
+            if idx > 0 {
+                output.push('.');
+            }
+            match elem {
+                PathMember::Int { val, .. } => output.push_str(&format!("{}", val)),
+                PathMember::String { val, .. } => output.push_str(val),
+            }
+        }
+
+        output
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct FullCellPath {
+    pub head: Expression,
+    pub tail: Vec<PathMember>,
+}

crates/nu-protocol/src/ast/expr.rs

@@ -0,0 +1,39 @@
+use super::{Call, CellPath, Expression, FullCellPath, Operator, RangeOperator};
+use crate::{ast::ImportPattern, BlockId, Signature, Span, Spanned, Unit, VarId};
+
+#[derive(Debug, Clone)]
+pub enum Expr {
+    Bool(bool),
+    Int(i64),
+    Float(f64),
+    Range(
+        Option<Box<Expression>>, // from
+        Option<Box<Expression>>, // next value after "from"
+        Option<Box<Expression>>, // to
+        RangeOperator,
+    ),
+    Var(VarId),
+    VarDecl(VarId),
+    Call(Box<Call>),
+    ExternalCall(Box<Expression>, Vec<Expression>),
+    Operator(Operator),
+    RowCondition(BlockId),
+    BinaryOp(Box<Expression>, Box<Expression>, Box<Expression>), //lhs, op, rhs
+    Subexpression(BlockId),
+    Block(BlockId),
+    List(Vec<Expression>),
+    Table(Vec<Expression>, Vec<Vec<Expression>>),
+    Record(Vec<(Expression, Expression)>),
+    Keyword(Vec<u8>, Span, Box<Expression>),
+    ValueWithUnit(Box<Expression>, Spanned<Unit>),
+    Filepath(String),
+    GlobPattern(String),
+    String(String),
+    CellPath(CellPath),
+    FullCellPath(Box<FullCellPath>),
+    ImportPattern(ImportPattern),
+    Signature(Box<Signature>),
+    StringInterpolation(Vec<Expression>),
+    Nothing,
+    Garbage,
+}

crates/nu-protocol/src/ast/expression.rs

@@ -0,0 +1,405 @@
+use super::{Expr, Operator, Statement};
+use crate::ast::ImportPattern;
+use crate::{engine::StateWorkingSet, BlockId, Signature, Span, Type, VarId, IN_VARIABLE_ID};
+
+#[derive(Debug, Clone)]
+pub struct Expression {
+    pub expr: Expr,
+    pub span: Span,
+    pub ty: Type,
+    pub custom_completion: Option<String>,
+}
+
+impl Expression {
+    pub fn garbage(span: Span) -> Expression {
+        Expression {
+            expr: Expr::Garbage,
+            span,
+            ty: Type::Unknown,
+            custom_completion: None,
+        }
+    }
+
+    pub fn precedence(&self) -> usize {
+        match &self.expr {
+            Expr::Operator(operator) => {
+                // Higher precedence binds tighter
+
+                match operator {
+                    Operator::Pow => 100,
+                    Operator::Multiply | Operator::Divide | Operator::Modulo => 95,
+                    Operator::Plus | Operator::Minus => 90,
+                    Operator::NotContains
+                    | Operator::Contains
+                    | Operator::LessThan
+                    | Operator::LessThanOrEqual
+                    | Operator::GreaterThan
+                    | Operator::GreaterThanOrEqual
+                    | Operator::Equal
+                    | Operator::NotEqual
+                    | Operator::In
+                    | Operator::NotIn => 80,
+                    Operator::And => 50,
+                    Operator::Or => 40,
+                }
+            }
+            _ => 0,
+        }
+    }
+
+    pub fn as_block(&self) -> Option<BlockId> {
+        match self.expr {
+            Expr::Block(block_id) => Some(block_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_row_condition_block(&self) -> Option<BlockId> {
+        match self.expr {
+            Expr::RowCondition(block_id) => Some(block_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_signature(&self) -> Option<Box<Signature>> {
+        match &self.expr {
+            Expr::Signature(sig) => Some(sig.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_list(&self) -> Option<Vec<Expression>> {
+        match &self.expr {
+            Expr::List(list) => Some(list.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_keyword(&self) -> Option<&Expression> {
+        match &self.expr {
+            Expr::Keyword(_, _, expr) => Some(expr),
+            _ => None,
+        }
+    }
+
+    pub fn as_var(&self) -> Option<VarId> {
+        match self.expr {
+            Expr::Var(var_id) => Some(var_id),
+            Expr::VarDecl(var_id) => Some(var_id),
+            _ => None,
+        }
+    }
+
+    pub fn as_string(&self) -> Option<String> {
+        match &self.expr {
+            Expr::String(string) => Some(string.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn as_import_pattern(&self) -> Option<ImportPattern> {
+        match &self.expr {
+            Expr::ImportPattern(pattern) => Some(pattern.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn has_in_variable(&self, working_set: &StateWorkingSet) -> bool {
+        match &self.expr {
+            Expr::BinaryOp(left, _, right) => {
+                left.has_in_variable(working_set) || right.has_in_variable(working_set)
+            }
+            Expr::Block(block_id) => {
+                let block = working_set.get_block(*block_id);
+
+                if block.captures.contains(&IN_VARIABLE_ID) {
+                    return true;
+                }
+
+                if let Some(Statement::Pipeline(pipeline)) = block.stmts.get(0) {
+                    match pipeline.expressions.get(0) {
+                        Some(expr) => expr.has_in_variable(working_set),
+                        None => false,
+                    }
+                } else {
+                    false
+                }
+            }
+            Expr::Bool(_) => false,
+            Expr::Call(call) => {
+                for positional in &call.positional {
+                    if positional.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                for named in &call.named {
+                    if let Some(expr) = &named.1 {
+                        if expr.has_in_variable(working_set) {
+                            return true;
+                        }
+                    }
+                }
+                false
+            }
+            Expr::CellPath(_) => false,
+            Expr::ExternalCall(head, args) => {
+                if head.has_in_variable(working_set) {
+                    return true;
+                }
+                for arg in args {
+                    if arg.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Expr::ImportPattern(_) => false,
+            Expr::Filepath(_) => false,
+            Expr::Float(_) => false,
+            Expr::FullCellPath(full_cell_path) => {
+                if full_cell_path.head.has_in_variable(working_set) {
+                    return true;
+                }
+                false
+            }
+            Expr::Garbage => false,
+            Expr::Nothing => false,
+            Expr::GlobPattern(_) => false,
+            Expr::Int(_) => false,
+            Expr::Keyword(_, _, expr) => expr.has_in_variable(working_set),
+            Expr::List(list) => {
+                for l in list {
+                    if l.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Expr::StringInterpolation(items) => {
+                for i in items {
+                    if i.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Expr::Operator(_) => false,
+            Expr::Range(left, middle, right, ..) => {
+                if let Some(left) = &left {
+                    if left.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                if let Some(middle) = &middle {
+                    if middle.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                if let Some(right) = &right {
+                    if right.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Expr::Record(fields) => {
+                for (field_name, field_value) in fields {
+                    if field_name.has_in_variable(working_set) {
+                        return true;
+                    }
+                    if field_value.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+                false
+            }
+            Expr::Signature(_) => false,
+            Expr::String(_) => false,
+            Expr::RowCondition(block_id) | Expr::Subexpression(block_id) => {
+                let block = working_set.get_block(*block_id);
+
+                if let Some(Statement::Pipeline(pipeline)) = block.stmts.get(0) {
+                    if let Some(expr) = pipeline.expressions.get(0) {
+                        expr.has_in_variable(working_set)
+                    } else {
+                        false
+                    }
+                } else {
+                    false
+                }
+            }
+            Expr::Table(headers, cells) => {
+                for header in headers {
+                    if header.has_in_variable(working_set) {
+                        return true;
+                    }
+                }
+
+                for row in cells {
+                    for cell in row.iter() {
+                        if cell.has_in_variable(working_set) {
+                            return true;
+                        }
+                    }
+                }
+
+                false
+            }
+
+            Expr::ValueWithUnit(expr, _) => expr.has_in_variable(working_set),
+            Expr::Var(var_id) => *var_id == IN_VARIABLE_ID,
+            Expr::VarDecl(_) => false,
+        }
+    }
+
+    pub fn replace_in_variable(&mut self, working_set: &mut StateWorkingSet, new_var_id: VarId) {
+        match &mut self.expr {
+            Expr::BinaryOp(left, _, right) => {
+                left.replace_in_variable(working_set, new_var_id);
+                right.replace_in_variable(working_set, new_var_id);
+            }
+            Expr::Block(block_id) => {
+                let block = working_set.get_block(*block_id);
+
+                let new_expr = if let Some(Statement::Pipeline(pipeline)) = block.stmts.get(0) {
+                    if let Some(expr) = pipeline.expressions.get(0) {
+                        let mut new_expr = expr.clone();
+                        new_expr.replace_in_variable(working_set, new_var_id);
+                        Some(new_expr)
+                    } else {
+                        None
+                    }
+                } else {
+                    None
+                };
+
+                let block = working_set.get_block_mut(*block_id);
+
+                if let Some(new_expr) = new_expr {
+                    if let Some(Statement::Pipeline(pipeline)) = block.stmts.get_mut(0) {
+                        if let Some(expr) = pipeline.expressions.get_mut(0) {
+                            *expr = new_expr
+                        }
+                    }
+                }
+
+                block.captures = block
+                    .captures
+                    .iter()
+                    .map(|x| if *x != IN_VARIABLE_ID { *x } else { new_var_id })
+                    .collect();
+            }
+            Expr::Bool(_) => {}
+            Expr::Call(call) => {
+                for positional in &mut call.positional {
+                    positional.replace_in_variable(working_set, new_var_id);
+                }
+                for named in &mut call.named {
+                    if let Some(expr) = &mut named.1 {
+                        expr.replace_in_variable(working_set, new_var_id)
+                    }
+                }
+            }
+            Expr::CellPath(_) => {}
+            Expr::ExternalCall(head, args) => {
+                head.replace_in_variable(working_set, new_var_id);
+                for arg in args {
+                    arg.replace_in_variable(working_set, new_var_id)
+                }
+            }
+            Expr::Filepath(_) => {}
+            Expr::Float(_) => {}
+            Expr::FullCellPath(full_cell_path) => {
+                full_cell_path
+                    .head
+                    .replace_in_variable(working_set, new_var_id);
+            }
+            Expr::ImportPattern(_) => {}
+            Expr::Garbage => {}
+            Expr::Nothing => {}
+            Expr::GlobPattern(_) => {}
+            Expr::Int(_) => {}
+            Expr::Keyword(_, _, expr) => expr.replace_in_variable(working_set, new_var_id),
+            Expr::List(list) => {
+                for l in list {
+                    l.replace_in_variable(working_set, new_var_id)
+                }
+            }
+            Expr::Operator(_) => {}
+            Expr::Range(left, middle, right, ..) => {
+                if let Some(left) = left {
+                    left.replace_in_variable(working_set, new_var_id)
+                }
+                if let Some(middle) = middle {
+                    middle.replace_in_variable(working_set, new_var_id)
+                }
+                if let Some(right) = right {
+                    right.replace_in_variable(working_set, new_var_id)
+                }
+            }
+            Expr::Record(fields) => {
+                for (field_name, field_value) in fields {
+                    field_name.replace_in_variable(working_set, new_var_id);
+                    field_value.replace_in_variable(working_set, new_var_id);
+                }
+            }
+            Expr::Signature(_) => {}
+            Expr::String(_) => {}
+            Expr::StringInterpolation(items) => {
+                for i in items {
+                    i.replace_in_variable(working_set, new_var_id)
+                }
+            }
+            Expr::RowCondition(block_id) | Expr::Subexpression(block_id) => {
+                let block = working_set.get_block(*block_id);
+
+                let new_expr = if let Some(Statement::Pipeline(pipeline)) = block.stmts.get(0) {
+                    if let Some(expr) = pipeline.expressions.get(0) {
+                        let mut new_expr = expr.clone();
+                        new_expr.replace_in_variable(working_set, new_var_id);
+                        Some(new_expr)
+                    } else {
+                        None
+                    }
+                } else {
+                    None
+                };
+
+                let block = working_set.get_block_mut(*block_id);
+
+                if let Some(new_expr) = new_expr {
+                    if let Some(Statement::Pipeline(pipeline)) = block.stmts.get_mut(0) {
+                        if let Some(expr) = pipeline.expressions.get_mut(0) {
+                            *expr = new_expr
+                        }
+                    }
+                }
+
+                block.captures = block
+                    .captures
+                    .iter()
+                    .map(|x| if *x != IN_VARIABLE_ID { *x } else { new_var_id })
+                    .collect();
+            }
+            Expr::Table(headers, cells) => {
+                for header in headers {
+                    header.replace_in_variable(working_set, new_var_id)
+                }
+
+                for row in cells {
+                    for cell in row.iter_mut() {
+                        cell.replace_in_variable(working_set, new_var_id)
+                    }
+                }
+            }
+
+            Expr::ValueWithUnit(expr, _) => expr.replace_in_variable(working_set, new_var_id),
+            Expr::Var(x) => {
+                if *x == IN_VARIABLE_ID {
+                    *x = new_var_id
+                }
+            }
+            Expr::VarDecl(_) => {}
+        }
+    }
+}

crates/nu-protocol/src/ast/import_pattern.rs

@@ -0,0 +1,69 @@
+use crate::{span, Span};
+use std::collections::HashSet;
+
+#[derive(Debug, Clone)]
+pub enum ImportPatternMember {
+    Glob { span: Span },
+    Name { name: Vec<u8>, span: Span },
+    List { names: Vec<(Vec<u8>, Span)> },
+}
+
+#[derive(Debug, Clone)]
+pub struct ImportPatternHead {
+    pub name: Vec<u8>,
+    pub span: Span,
+}
+
+#[derive(Debug, Clone)]
+pub struct ImportPattern {
+    pub head: ImportPatternHead,
+    pub members: Vec<ImportPatternMember>,
+    // communicate to eval which decls/aliases were hidden during `parse_hide()` so it does not
+    // interpret these as env var names:
+    pub hidden: HashSet<Vec<u8>>,
+}
+
+impl ImportPattern {
+    pub fn new() -> Self {
+        ImportPattern {
+            head: ImportPatternHead {
+                name: vec![],
+                span: Span { start: 0, end: 0 },
+            },
+            members: vec![],
+            hidden: HashSet::new(),
+        }
+    }
+
+    pub fn span(&self) -> Span {
+        let mut spans = vec![self.head.span];
+
+        for member in &self.members {
+            match member {
+                ImportPatternMember::Glob { span } => spans.push(*span),
+                ImportPatternMember::Name { name: _, span } => spans.push(*span),
+                ImportPatternMember::List { names } => {
+                    for (_, span) in names {
+                        spans.push(*span);
+                    }
+                }
+            }
+        }
+
+        span(&spans)
+    }
+
+    pub fn with_hidden(self, hidden: HashSet<Vec<u8>>) -> Self {
+        ImportPattern {
+            head: self.head,
+            members: self.members,
+            hidden,
+        }
+    }
+}
+
+impl Default for ImportPattern {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/nu-protocol/src/ast/mod.rs

@@ -0,0 +1,19 @@
+mod block;
+mod call;
+mod cell_path;
+mod expr;
+mod expression;
+mod import_pattern;
+mod operator;
+mod pipeline;
+mod statement;
+
+pub use block::*;
+pub use call::*;
+pub use cell_path::*;
+pub use expr::*;
+pub use expression::*;
+pub use import_pattern::*;
+pub use operator::*;
+pub use pipeline::*;
+pub use statement::*;

crates/nu-protocol/src/ast/operator.rs

@@ -0,0 +1,73 @@
+use crate::Span;
+
+use serde::{Deserialize, Serialize};
+use std::fmt::Display;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Operator {
+    Equal,
+    NotEqual,
+    LessThan,
+    GreaterThan,
+    LessThanOrEqual,
+    GreaterThanOrEqual,
+    Contains,
+    NotContains,
+    Plus,
+    Minus,
+    Multiply,
+    Divide,
+    In,
+    NotIn,
+    Modulo,
+    And,
+    Or,
+    Pow,
+}
+
+impl Display for Operator {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Operator::Equal => write!(f, "=="),
+            Operator::NotEqual => write!(f, "!="),
+            Operator::LessThan => write!(f, "<"),
+            Operator::GreaterThan => write!(f, ">"),
+            Operator::Contains => write!(f, "=~"),
+            Operator::NotContains => write!(f, "!~"),
+            Operator::Plus => write!(f, "+"),
+            Operator::Minus => write!(f, "-"),
+            Operator::Multiply => write!(f, "*"),
+            Operator::Divide => write!(f, "/"),
+            Operator::In => write!(f, "in"),
+            Operator::NotIn => write!(f, "not-in"),
+            Operator::Modulo => write!(f, "mod"),
+            Operator::And => write!(f, "&&"),
+            Operator::Or => write!(f, "||"),
+            Operator::Pow => write!(f, "**"),
+            Operator::LessThanOrEqual => write!(f, "<="),
+            Operator::GreaterThanOrEqual => write!(f, ">="),
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
+pub enum RangeInclusion {
+    Inclusive,
+    RightExclusive,
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct RangeOperator {
+    pub inclusion: RangeInclusion,
+    pub span: Span,
+    pub next_op_span: Span,
+}
+
+impl Display for RangeOperator {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.inclusion {
+            RangeInclusion::Inclusive => write!(f, ".."),
+            RangeInclusion::RightExclusive => write!(f, "..<"),
+        }
+    }
+}

crates/nu-protocol/src/ast/pipeline.rs

@@ -0,0 +1,24 @@
+use crate::ast::Expression;
+
+#[derive(Debug, Clone)]
+pub struct Pipeline {
+    pub expressions: Vec<Expression>,
+}
+
+impl Default for Pipeline {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Pipeline {
+    pub fn new() -> Self {
+        Self {
+            expressions: vec![],
+        }
+    }
+
+    pub fn from_vec(expressions: Vec<Expression>) -> Pipeline {
+        Self { expressions }
+    }
+}

crates/nu-protocol/src/ast/statement.rs

@@ -0,0 +1,8 @@
+use super::Pipeline;
+use crate::DeclId;
+
+#[derive(Debug, Clone)]
+pub enum Statement {
+    Declaration(DeclId),
+    Pipeline(Pipeline),
+}

crates/nu-protocol/src/call_info.rs

@@ -1,121 +0,0 @@
-use crate::value::Value;
-use derive_new::new;
-use indexmap::IndexMap;
-use nu_errors::ShellError;
-use nu_source::Tag;
-use serde::{Deserialize, Serialize};
-
-/// Associated information for the call of a command, including the args passed to the command and a tag that spans the name of the command being called
-#[derive(Deserialize, Serialize, Debug, Clone)]
-pub struct CallInfo {
-    /// The arguments associated with this call
-    pub args: EvaluatedArgs,
-    /// The tag (underline-able position) of the name of the call itself
-    pub name_tag: Tag,
-}
-
-/// The set of positional and named arguments, after their values have been evaluated.
-///
-/// * Positional arguments are those who are given as values, without any associated flag. For example, in `foo arg1 arg2`, both `arg1` and `arg2` are positional arguments.
-/// * Named arguments are those associated with a flag. For example, `foo --given bar` the named argument would be name `given` and the value `bar`.
-#[derive(Debug, Default, new, Serialize, Deserialize, Clone)]
-pub struct EvaluatedArgs {
-    pub positional: Option<Vec<Value>>,
-    pub named: Option<IndexMap<String, Value>>,
-}
-
-impl EvaluatedArgs {
-    /// Retrieve a subset of positional arguments starting at a given position
-    pub fn slice_from(&self, from: usize) -> Vec<Value> {
-        let positional = &self.positional;
-
-        match positional {
-            None => vec![],
-            Some(list) => list[from..].to_vec(),
-        }
-    }
-
-    /// Get the nth positional argument, if possible
-    pub fn nth(&self, pos: usize) -> Option<&Value> {
-        match &self.positional {
-            None => None,
-            Some(array) => array.get(pos),
-        }
-    }
-
-    /// Get the nth positional argument, error if not possible
-    pub fn expect_nth(&self, pos: usize) -> Result<&Value, ShellError> {
-        match &self.positional {
-            None => Err(ShellError::unimplemented("Better error: expect_nth")),
-            Some(array) => match array.get(pos) {
-                None => Err(ShellError::unimplemented("Better error: expect_nth")),
-                Some(item) => Ok(item),
-            },
-        }
-    }
-
-    /// Get the number of positional arguments available
-    pub fn len(&self) -> usize {
-        match &self.positional {
-            None => 0,
-            Some(array) => array.len(),
-        }
-    }
-
-    /// Return if there are no positional arguments
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-
-    /// Return true if the set of named arguments contains the name provided
-    pub fn has(&self, name: &str) -> bool {
-        matches!(&self.named, Some(named) if named.contains_key(name))
-    }
-
-    /// Gets the corresponding Value for the named argument given, if possible
-    pub fn get(&self, name: &str) -> Option<&Value> {
-        match &self.named {
-            None => None,
-            Some(named) => named.get(name),
-        }
-    }
-
-    /// Gets the corresponding Value for the named argument given, error if not possible
-    pub fn expect_get(&self, name: &str) -> Result<&Value, ShellError> {
-        match &self.named {
-            None => Err(ShellError::unimplemented("Better error: expect_get")),
-            Some(named) => named
-                .get(name)
-                .ok_or_else(|| ShellError::unimplemented("Better error: expect_get")),
-        }
-    }
-
-    /// Iterates over the positional arguments
-    pub fn positional_iter(&self) -> PositionalIter<'_> {
-        match &self.positional {
-            None => PositionalIter::Empty,
-            Some(v) => {
-                let iter = v.iter();
-                PositionalIter::Array(iter)
-            }
-        }
-    }
-}
-
-/// An iterator to help iterate over positional arguments
-pub enum PositionalIter<'a> {
-    Empty,
-    Array(std::slice::Iter<'a, Value>),
-}
-
-impl<'a> Iterator for PositionalIter<'a> {
-    type Item = &'a Value;
-
-    /// The required `next` function to implement the Iterator trait
-    fn next(&mut self) -> Option<Self::Item> {
-        match self {
-            PositionalIter::Empty => None,
-            PositionalIter::Array(iter) => iter.next(),
-        }
-    }
-}

crates/nu-protocol/src/config.rs

@@ -0,0 +1,372 @@
+use crate::{BlockId, ShellError, Span, Value};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+const ANIMATE_PROMPT_DEFAULT: bool = true;
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct EnvConversion {
+    pub from_string: Option<(BlockId, Span)>,
+    pub to_string: Option<(BlockId, Span)>,
+}
+
+impl EnvConversion {
+    pub fn from_record(value: &Value) -> Result<Self, ShellError> {
+        let record = value.as_record()?;
+
+        let mut conv_map = HashMap::new();
+
+        for (k, v) in record.0.iter().zip(record.1) {
+            if (k == "from_string") || (k == "to_string") {
+                conv_map.insert(k.as_str(), (v.as_block()?, v.span()?));
+            } else {
+                return Err(ShellError::UnsupportedConfigValue(
+                    "'from_string' and 'to_string' fields".into(),
+                    k.into(),
+                    value.span()?,
+                ));
+            }
+        }
+
+        let from_string = conv_map.get("from_string").cloned();
+        let to_string = conv_map.get("to_string").cloned();
+
+        Ok(EnvConversion {
+            from_string,
+            to_string,
+        })
+    }
+}
+
+/// Definition of a parsed keybinding from the config object
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct ParsedKeybinding {
+    pub modifier: Value,
+    pub keycode: Value,
+    pub event: Value,
+    pub mode: Value,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct Config {
+    pub filesize_metric: bool,
+    pub table_mode: String,
+    pub use_ls_colors: bool,
+    pub color_config: HashMap<String, Value>,
+    pub use_grid_icons: bool,
+    pub footer_mode: FooterMode,
+    pub animate_prompt: bool,
+    pub float_precision: i64,
+    pub filesize_format: String,
+    pub use_ansi_coloring: bool,
+    pub quick_completions: bool,
+    pub env_conversions: HashMap<String, EnvConversion>,
+    pub edit_mode: String,
+    pub max_history_size: i64,
+    pub log_level: String,
+    pub menu_config: HashMap<String, Value>,
+    pub keybindings: Vec<ParsedKeybinding>,
+    pub history_config: HashMap<String, Value>,
+}
+
+impl Default for Config {
+    fn default() -> Config {
+        Config {
+            filesize_metric: false,
+            table_mode: "rounded".into(),
+            use_ls_colors: true,
+            color_config: HashMap::new(),
+            use_grid_icons: false,
+            footer_mode: FooterMode::Never,
+            animate_prompt: ANIMATE_PROMPT_DEFAULT,
+            float_precision: 4,
+            filesize_format: "auto".into(),
+            use_ansi_coloring: true,
+            quick_completions: false,
+            env_conversions: HashMap::new(), // TODO: Add default conversoins
+            edit_mode: "emacs".into(),
+            max_history_size: 1000,
+            log_level: String::new(),
+            menu_config: HashMap::new(),
+            keybindings: Vec::new(),
+            history_config: HashMap::new(),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub enum FooterMode {
+    /// Never show the footer
+    Never,
+    /// Always show the footer
+    Always,
+    /// Only show the footer if there are more than RowCount rows
+    RowCount(u64),
+    /// Calculate the screen height, calculate row count, if display will be bigger than screen, add the footer
+    Auto,
+}
+
+impl Value {
+    pub fn into_config(self) -> Result<Config, ShellError> {
+        let v = self.as_record();
+
+        let mut config = Config::default();
+
+        if let Ok(v) = v {
+            for (key, value) in v.0.iter().zip(v.1) {
+                match key.as_str() {
+                    "filesize_metric" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.filesize_metric = b;
+                        } else {
+                            eprintln!("$config.filesize_metric is not a bool")
+                        }
+                    }
+                    "table_mode" => {
+                        if let Ok(v) = value.as_string() {
+                            config.table_mode = v;
+                        } else {
+                            eprintln!("$config.table_mode is not a string")
+                        }
+                    }
+                    "use_ls_colors" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.use_ls_colors = b;
+                        } else {
+                            eprintln!("$config.use_ls_colors is not a bool")
+                        }
+                    }
+                    "color_config" => {
+                        if let Ok(map) = create_map(value, &config) {
+                            config.color_config = map;
+                        } else {
+                            eprintln!("$config.color_config is not a record")
+                        }
+                    }
+                    "use_grid_icons" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.use_grid_icons = b;
+                        } else {
+                            eprintln!("$config.use_grid_icons is not a bool")
+                        }
+                    }
+                    "footer_mode" => {
+                        if let Ok(b) = value.as_string() {
+                            let val_str = b.to_lowercase();
+                            config.footer_mode = match val_str.as_ref() {
+                                "auto" => FooterMode::Auto,
+                                "never" => FooterMode::Never,
+                                "always" => FooterMode::Always,
+                                _ => match &val_str.parse::<u64>() {
+                                    Ok(number) => FooterMode::RowCount(*number),
+                                    _ => FooterMode::Never,
+                                },
+                            };
+                        } else {
+                            eprintln!("$config.footer_mode is not a string")
+                        }
+                    }
+                    "animate_prompt" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.animate_prompt = b;
+                        } else {
+                            eprintln!("$config.animate_prompt is not a bool")
+                        }
+                    }
+                    "float_precision" => {
+                        if let Ok(i) = value.as_integer() {
+                            config.float_precision = i;
+                        } else {
+                            eprintln!("$config.float_precision is not an integer")
+                        }
+                    }
+                    "use_ansi_coloring" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.use_ansi_coloring = b;
+                        } else {
+                            eprintln!("$config.use_ansi_coloring is not a bool")
+                        }
+                    }
+                    "quick_completions" => {
+                        if let Ok(b) = value.as_bool() {
+                            config.quick_completions = b;
+                        } else {
+                            eprintln!("$config.quick_completions is not a bool")
+                        }
+                    }
+                    "filesize_format" => {
+                        if let Ok(v) = value.as_string() {
+                            config.filesize_format = v.to_lowercase();
+                        } else {
+                            eprintln!("$config.filesize_format is not a string")
+                        }
+                    }
+                    "env_conversions" => {
+                        if let Ok((env_vars, conversions)) = value.as_record() {
+                            let mut env_conversions = HashMap::new();
+
+                            for (env_var, record) in env_vars.iter().zip(conversions) {
+                                // println!("{}: {:?}", env_var, record);
+                                if let Ok(conversion) = EnvConversion::from_record(record) {
+                                    env_conversions.insert(env_var.into(), conversion);
+                                } else {
+                                    eprintln!("$config.env_conversions has incorrect conversion")
+                                }
+                            }
+
+                            config.env_conversions = env_conversions;
+                        } else {
+                            eprintln!("$config.env_conversions is not a record")
+                        }
+                    }
+                    "edit_mode" => {
+                        if let Ok(v) = value.as_string() {
+                            config.edit_mode = v.to_lowercase();
+                        } else {
+                            eprintln!("$config.edit_mode is not a string")
+                        }
+                    }
+                    "max_history_size" => {
+                        if let Ok(i) = value.as_i64() {
+                            config.max_history_size = i;
+                        } else {
+                            eprintln!("$config.max_history_size is not an integer")
+                        }
+                    }
+                    "log_level" => {
+                        if let Ok(v) = value.as_string() {
+                            config.log_level = v.to_lowercase();
+                        } else {
+                            eprintln!("$config.log_level is not a string")
+                        }
+                    }
+                    "menu_config" => {
+                        if let Ok(map) = create_map(value, &config) {
+                            config.menu_config = map;
+                        } else {
+                            eprintln!("$config.menu_config is not a record")
+                        }
+                    }
+                    "keybindings" => {
+                        if let Ok(keybindings) = create_keybindings(value, &config) {
+                            config.keybindings = keybindings;
+                        } else {
+                            eprintln!("$config.keybindings is not a valid keybindings list")
+                        }
+                    }
+                    "history_config" => {
+                        if let Ok(map) = create_map(value, &config) {
+                            config.history_config = map;
+                        } else {
+                            eprintln!("$config.history_config is not a record")
+                        }
+                    }
+                    x => {
+                        eprintln!("$config.{} is an unknown config setting", x)
+                    }
+                }
+            }
+        } else {
+            eprintln!("$config is not a record");
+        }
+
+        Ok(config)
+    }
+}
+
+fn create_map(value: &Value, config: &Config) -> Result<HashMap<String, Value>, ShellError> {
+    let (cols, inner_vals) = value.as_record()?;
+    let mut hm: HashMap<String, Value> = HashMap::new();
+
+    for (k, v) in cols.iter().zip(inner_vals) {
+        match &v {
+            Value::Record {
+                cols: inner_cols,
+                vals: inner_vals,
+                span,
+            } => {
+                let val = color_value_string(span, inner_cols, inner_vals, config);
+                hm.insert(k.to_string(), val);
+            }
+            _ => {
+                hm.insert(k.to_string(), v.clone());
+            }
+        }
+    }
+
+    Ok(hm)
+}
+
+fn color_value_string(
+    span: &Span,
+    inner_cols: &[String],
+    inner_vals: &[Value],
+    config: &Config,
+) -> Value {
+    // make a string from our config.color_config section that
+    // looks like this: { fg: "#rrggbb" bg: "#rrggbb" attr: "abc", }
+    // the real key here was to have quotes around the values but not
+    // require them around the keys.
+
+    // maybe there's a better way to generate this but i'm not sure
+    // what it is.
+    let val: String = inner_cols
+        .iter()
+        .zip(inner_vals)
+        .map(|(x, y)| format!("{}: \"{}\" ", x, y.into_string(", ", config)))
+        .collect();
+
+    // now insert the braces at the front and the back to fake the json string
+    Value::String {
+        val: format!("{{{}}}", val),
+        span: *span,
+    }
+}
+
+// Parses the config object to extract the strings that will compose a keybinding for reedline
+fn create_keybindings(value: &Value, config: &Config) -> Result<Vec<ParsedKeybinding>, ShellError> {
+    match value {
+        Value::Record { cols, vals, span } => {
+            // Finding the modifier value in the record
+            let modifier = extract_value("modifier", cols, vals, span)?;
+            let keycode = extract_value("keycode", cols, vals, span)?;
+            let mode = extract_value("mode", cols, vals, span)?;
+            let event = extract_value("event", cols, vals, span)?;
+
+            let keybinding = ParsedKeybinding {
+                modifier: modifier.clone(),
+                keycode: keycode.clone(),
+                mode: mode.clone(),
+                event: event.clone(),
+            };
+
+            Ok(vec![keybinding])
+        }
+        Value::List { vals, .. } => {
+            let res = vals
+                .iter()
+                .map(|inner_value| create_keybindings(inner_value, config))
+                .collect::<Result<Vec<Vec<ParsedKeybinding>>, ShellError>>();
+
+            let res = res?
+                .into_iter()
+                .flatten()
+                .collect::<Vec<ParsedKeybinding>>();
+
+            Ok(res)
+        }
+        _ => Ok(Vec::new()),
+    }
+}
+
+pub fn extract_value<'record>(
+    name: &str,
+    cols: &'record [String],
+    vals: &'record [Value],
+    span: &Span,
+) -> Result<&'record Value, ShellError> {
+    cols.iter()
+        .position(|col| col.as_str() == name)
+        .and_then(|index| vals.get(index))
+        .ok_or_else(|| ShellError::MissingConfigValue(name.to_string(), *span))
+}

crates/nu-protocol/src/config_path.rs

@@ -1,19 +0,0 @@
-use serde::{Deserialize, Serialize};
-use std::path::PathBuf;
-
-/// Specifies a path to a configuration file and its type
-#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
-pub enum ConfigPath {
-    /// Path to the global configuration file
-    Global(PathBuf),
-    /// Path to a local configuration file
-    Local(PathBuf),
-}
-
-impl ConfigPath {
-    pub fn get_path(&self) -> &PathBuf {
-        match self {
-            ConfigPath::Global(p) | ConfigPath::Local(p) => p,
-        }
-    }
-}

crates/nu-protocol/src/dataframe/compute_between.rs

@@ -1,880 +0,0 @@
-use bigdecimal::BigDecimal;
-use nu_errors::ShellError;
-use nu_source::Span;
-use num_traits::ToPrimitive;
-
-use super::{Axis, NuDataFrame};
-use crate::hir::Operator;
-use crate::{Primitive, ShellTypeName, UntaggedValue, Value};
-
-use polars::prelude::{
-    BooleanType, ChunkCompare, ChunkedArray, DataType, Float64Type, Int64Type, IntoSeries,
-    NumOpsDispatchChecked, PolarsError, Series,
-};
-use std::ops::{Add, BitAnd, BitOr, Div, Mul, Sub};
-
-pub fn compute_between_dataframes(
-    operator: Operator,
-    left: &Value,
-    right: &Value,
-) -> Result<UntaggedValue, (&'static str, &'static str)> {
-    if let (UntaggedValue::DataFrame(lhs), UntaggedValue::DataFrame(rhs)) =
-        (&left.value, &right.value)
-    {
-        let operation_span = right.tag.span.merge(left.tag.span);
-        match (lhs.is_series(), rhs.is_series()) {
-            (true, true) => {
-                let lhs = &lhs
-                    .as_series(&left.tag.span)
-                    .expect("Already checked that is a series");
-                let rhs = &rhs
-                    .as_series(&right.tag.span)
-                    .expect("Already checked that is a series");
-
-                if lhs.dtype() != rhs.dtype() {
-                    return Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Mixed datatypes",
-                            "this datatype does not match the right hand side datatype",
-                            &left.tag.span,
-                            format!(
-                                "Perhaps you want to change this datatype to '{}'",
-                                lhs.as_ref().dtype()
-                            ),
-                            &right.tag.span,
-                        ),
-                    ));
-                }
-
-                if lhs.len() != rhs.len() {
-                    return Ok(UntaggedValue::Error(ShellError::labeled_error(
-                        "Different length",
-                        "this column length does not match the right hand column length",
-                        &left.tag.span,
-                    )));
-                }
-
-                compute_between_series(operator, lhs, rhs, &operation_span)
-            }
-            _ => {
-                if lhs.as_ref().height() != rhs.as_ref().height() {
-                    return Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Mixed datatypes",
-                            "this datatype size does not match the right hand side datatype",
-                            &left.tag.span,
-                            "Perhaps you want to select another dataframe with same number of rows",
-                            &right.tag.span,
-                        ),
-                    ));
-                }
-
-                between_dataframes(operator, lhs, rhs, &operation_span)
-            }
-        }
-    } else {
-        Err((left.type_name(), right.type_name()))
-    }
-}
-
-pub fn between_dataframes(
-    operator: Operator,
-    lhs: &NuDataFrame,
-    rhs: &NuDataFrame,
-    operation_span: &Span,
-) -> Result<UntaggedValue, (&'static str, &'static str)> {
-    match operator {
-        Operator::Plus => match lhs.append_df(rhs, Axis::Row, operation_span) {
-            Ok(df) => Ok(df.into_untagged()),
-            Err(e) => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                "Appending error",
-                e.to_string(),
-                operation_span,
-            ))),
-        },
-        _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-            "Incorrect datatype",
-            "unable to use this datatype for this operation",
-            operation_span,
-        ))),
-    }
-}
-
-pub fn compute_between_series(
-    operator: Operator,
-    lhs: &Series,
-    rhs: &Series,
-    operation_span: &Span,
-) -> Result<UntaggedValue, (&'static str, &'static str)> {
-    match operator {
-        Operator::Plus => {
-            let mut res = lhs + rhs;
-            let name = format!("sum_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::Minus => {
-            let mut res = lhs - rhs;
-            let name = format!("sub_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::Multiply => {
-            let mut res = lhs * rhs;
-            let name = format!("mul_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::Divide => {
-            let res = lhs.checked_div(rhs);
-            match res {
-                Ok(mut res) => {
-                    let name = format!("div_{}_{}", lhs.name(), rhs.name());
-                    res.rename(&name);
-                    Ok(NuDataFrame::series_to_untagged(res, operation_span))
-                }
-                Err(e) => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                    "Division error",
-                    e.to_string(),
-                    operation_span,
-                ))),
-            }
-        }
-        Operator::Equal => {
-            let mut res = Series::eq(lhs, rhs).into_series();
-            let name = format!("eq_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::NotEqual => {
-            let mut res = Series::neq(lhs, rhs).into_series();
-            let name = format!("neq_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::LessThan => {
-            let mut res = Series::lt(lhs, rhs).into_series();
-            let name = format!("lt_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::LessThanOrEqual => {
-            let mut res = Series::lt_eq(lhs, rhs).into_series();
-            let name = format!("lte_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::GreaterThan => {
-            let mut res = Series::gt(lhs, rhs).into_series();
-            let name = format!("gt_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::GreaterThanOrEqual => {
-            let mut res = Series::gt_eq(lhs, rhs).into_series();
-            let name = format!("gte_{}_{}", lhs.name(), rhs.name());
-            res.rename(&name);
-            Ok(NuDataFrame::series_to_untagged(res, operation_span))
-        }
-        Operator::And => match lhs.dtype() {
-            DataType::Boolean => {
-                let lhs_cast = lhs.bool();
-                let rhs_cast = rhs.bool();
-
-                match (lhs_cast, rhs_cast) {
-                    (Ok(l), Ok(r)) => {
-                        let mut res = l.bitand(r).into_series();
-                        let name = format!("and_{}_{}", lhs.name(), rhs.name());
-                        res.rename(&name);
-                        Ok(NuDataFrame::series_to_untagged(res, operation_span))
-                    }
-                    _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                        "Casting error",
-                        "unable to cast to boolean",
-                        operation_span,
-                    ))),
-                }
-            }
-            _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                "Incorrect datatype",
-                "And operation can only be done with boolean values",
-                operation_span,
-            ))),
-        },
-        Operator::Or => match lhs.dtype() {
-            DataType::Boolean => {
-                let lhs_cast = lhs.bool();
-                let rhs_cast = rhs.bool();
-
-                match (lhs_cast, rhs_cast) {
-                    (Ok(l), Ok(r)) => {
-                        let mut res = l.bitor(r).into_series();
-                        let name = format!("or_{}_{}", lhs.name(), rhs.name());
-                        res.rename(&name);
-                        Ok(NuDataFrame::series_to_untagged(res, operation_span))
-                    }
-                    _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                        "Casting error",
-                        "unable to cast to boolean",
-                        operation_span,
-                    ))),
-                }
-            }
-            _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                "Incorrect datatype",
-                "And operation can only be done with boolean values",
-                operation_span,
-            ))),
-        },
-        _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-            "Incorrect datatype",
-            "unable to use this datatype for this operation",
-            operation_span,
-        ))),
-    }
-}
-
-pub fn compute_series_single_value(
-    operator: Operator,
-    left: &Value,
-    right: &Value,
-) -> Result<UntaggedValue, (&'static str, &'static str)> {
-    if let (UntaggedValue::DataFrame(lhs), UntaggedValue::Primitive(_)) =
-        (&left.value, &right.value)
-    {
-        let lhs = match lhs.as_series(&left.tag.span) {
-            Ok(series) => series,
-            Err(e) => return Ok(UntaggedValue::Error(e)),
-        };
-
-        match operator {
-            Operator::Plus => match &right.value {
-                UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Int64Type>>::add,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    &val.to_i64()
-                        .expect("Internal error: protocol did not use compatible decimal"),
-                    <ChunkedArray<Int64Type>>::add,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(compute_series_decimal(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Float64Type>>::add,
-                    &left.tag.span,
-                )),
-                _ => Ok(UntaggedValue::Error(
-                    ShellError::labeled_error_with_secondary(
-                        "Operation unavailable",
-                        "unable to sum this value to the series",
-                        &right.tag.span,
-                        "Only int, bigInt or decimal values are allowed",
-                        &right.tag.span,
-                    ),
-                )),
-            },
-            Operator::Minus => match &right.value {
-                UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Int64Type>>::sub,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    &val.to_i64()
-                        .expect("Internal error: protocol did not use compatible decimal"),
-                    <ChunkedArray<Int64Type>>::sub,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(compute_series_decimal(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Float64Type>>::sub,
-                    &left.tag.span,
-                )),
-                _ => Ok(UntaggedValue::Error(
-                    ShellError::labeled_error_with_secondary(
-                        "Operation unavailable",
-                        "unable to subtract this value to the series",
-                        &right.tag.span,
-                        "Only int, bigInt or decimal values are allowed",
-                        &right.tag.span,
-                    ),
-                )),
-            },
-            Operator::Multiply => match &right.value {
-                UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Int64Type>>::mul,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compute_series_i64(
-                    &lhs,
-                    &val.to_i64()
-                        .expect("Internal error: protocol did not use compatible decimal"),
-                    <ChunkedArray<Int64Type>>::mul,
-                    &left.tag.span,
-                )),
-                UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(compute_series_decimal(
-                    &lhs,
-                    val,
-                    <ChunkedArray<Float64Type>>::mul,
-                    &left.tag.span,
-                )),
-                _ => Ok(UntaggedValue::Error(
-                    ShellError::labeled_error_with_secondary(
-                        "Operation unavailable",
-                        "unable to multiply this value to the series",
-                        &right.tag.span,
-                        "Only int, bigInt or decimal values are allowed",
-                        &right.tag.span,
-                    ),
-                )),
-            },
-            Operator::Divide => match &right.value {
-                UntaggedValue::Primitive(Primitive::Int(val)) => {
-                    if *val == 0 {
-                        Ok(UntaggedValue::Error(ShellError::labeled_error(
-                            "Division by zero",
-                            "Zero value found",
-                            &right.tag.span,
-                        )))
-                    } else {
-                        Ok(compute_series_i64(
-                            &lhs,
-                            val,
-                            <ChunkedArray<Int64Type>>::div,
-                            &left.tag.span,
-                        ))
-                    }
-                }
-                UntaggedValue::Primitive(Primitive::BigInt(val)) => {
-                    if val.eq(&0.into()) {
-                        Ok(UntaggedValue::Error(ShellError::labeled_error(
-                            "Division by zero",
-                            "Zero value found",
-                            &right.tag.span,
-                        )))
-                    } else {
-                        Ok(compute_series_i64(
-                            &lhs,
-                            &val.to_i64()
-                                .expect("Internal error: protocol did not use compatible decimal"),
-                            <ChunkedArray<Int64Type>>::div,
-                            &left.tag.span,
-                        ))
-                    }
-                }
-                UntaggedValue::Primitive(Primitive::Decimal(val)) => {
-                    if val.eq(&0.into()) {
-                        Ok(UntaggedValue::Error(ShellError::labeled_error(
-                            "Division by zero",
-                            "Zero value found",
-                            &right.tag.span,
-                        )))
-                    } else {
-                        Ok(compute_series_decimal(
-                            &lhs,
-                            val,
-                            <ChunkedArray<Float64Type>>::div,
-                            &left.tag.span,
-                        ))
-                    }
-                }
-                _ => Ok(UntaggedValue::Error(
-                    ShellError::labeled_error_with_secondary(
-                        "Operation unavailable",
-                        "unable to divide this value to the series",
-                        &right.tag.span,
-                        "Only primary values are allowed",
-                        &right.tag.span,
-                    ),
-                )),
-            },
-            Operator::Equal => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::eq, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::NotEqual => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::neq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::neq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::neq, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::LessThan => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::lt,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::lt,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::lt, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::LessThanOrEqual => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::lt_eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::lt_eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::lt_eq, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::GreaterThan => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::gt,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::gt,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::gt, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::GreaterThanOrEqual => {
-                match &right.value {
-                    UntaggedValue::Primitive(Primitive::Int(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        val,
-                        ChunkedArray::gt_eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::BigInt(val)) => Ok(compare_series_i64(
-                        &lhs,
-                        &val.to_i64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        ChunkedArray::gt_eq,
-                        &left.tag.span,
-                    )),
-                    UntaggedValue::Primitive(Primitive::Decimal(val)) => Ok(
-                        compare_series_decimal(&lhs, val, ChunkedArray::gt_eq, &left.tag.span),
-                    ),
-                    _ => Ok(UntaggedValue::Error(
-                        ShellError::labeled_error_with_secondary(
-                            "Operation unavailable",
-                            "unable to compare this value to the series",
-                            &right.tag.span,
-                            "Only primary values are allowed",
-                            &right.tag.span,
-                        ),
-                    )),
-                }
-            }
-            Operator::Contains => match &right.value {
-                UntaggedValue::Primitive(Primitive::String(val)) => {
-                    Ok(contains_series_pat(&lhs, val, &left.tag.span))
-                }
-                _ => Ok(UntaggedValue::Error(
-                    ShellError::labeled_error_with_secondary(
-                        "Operation unavailable",
-                        "unable to perform this value to the series",
-                        &right.tag.span,
-                        "Only primary values are allowed",
-                        &right.tag.span,
-                    ),
-                )),
-            },
-            _ => Ok(UntaggedValue::Error(ShellError::labeled_error(
-                "Incorrect datatype",
-                "unable to use this value for this operation",
-                &left.tag.span,
-            ))),
-        }
-    } else {
-        Err((left.type_name(), right.type_name()))
-    }
-}
-
-fn compute_series_i64<F>(series: &Series, val: &i64, f: F, span: &Span) -> UntaggedValue
-where
-    F: Fn(ChunkedArray<Int64Type>, i64) -> ChunkedArray<Int64Type>,
-{
-    match series.dtype() {
-        DataType::UInt32 | DataType::Int32 | DataType::UInt64 => {
-            let to_i64 = series.cast(&DataType::Int64);
-
-            match to_i64 {
-                Ok(series) => {
-                    let casted = series.i64();
-                    compute_casted_i64(casted, *val, f, span)
-                }
-                Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                    "Casting error",
-                    e.to_string(),
-                    span,
-                )),
-            }
-        }
-        DataType::Int64 => {
-            let casted = series.i64();
-            compute_casted_i64(casted, *val, f, span)
-        }
-        _ => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            format!(
-                "Series of type {} can not be used for operations with an i64 value",
-                series.dtype()
-            ),
-            span,
-        )),
-    }
-}
-
-fn compute_casted_i64<F>(
-    casted: Result<&ChunkedArray<Int64Type>, PolarsError>,
-    val: i64,
-    f: F,
-    span: &Span,
-) -> UntaggedValue
-where
-    F: Fn(ChunkedArray<Int64Type>, i64) -> ChunkedArray<Int64Type>,
-{
-    match casted {
-        Ok(casted) => {
-            let res = f(casted.clone(), val);
-            let res = res.into_series();
-            NuDataFrame::series_to_untagged(res, span)
-        }
-        Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            e.to_string(),
-            span,
-        )),
-    }
-}
-
-fn compute_series_decimal<F>(series: &Series, val: &BigDecimal, f: F, span: &Span) -> UntaggedValue
-where
-    F: Fn(ChunkedArray<Float64Type>, f64) -> ChunkedArray<Float64Type>,
-{
-    match series.dtype() {
-        DataType::Float32 => {
-            let to_f64 = series.cast(&DataType::Float64);
-
-            match to_f64 {
-                Ok(series) => {
-                    let casted = series.f64();
-                    compute_casted_f64(
-                        casted,
-                        val.to_f64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        f,
-                        span,
-                    )
-                }
-                Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                    "Casting error",
-                    e.to_string(),
-                    span,
-                )),
-            }
-        }
-        DataType::Float64 => {
-            let casted = series.f64();
-            compute_casted_f64(
-                casted,
-                val.to_f64()
-                    .expect("Internal error: protocol did not use compatible decimal"),
-                f,
-                span,
-            )
-        }
-        _ => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            format!(
-                "Series of type {} can not be used for operations with a decimal value",
-                series.dtype()
-            ),
-            span,
-        )),
-    }
-}
-
-fn compute_casted_f64<F>(
-    casted: Result<&ChunkedArray<Float64Type>, PolarsError>,
-    val: f64,
-    f: F,
-    span: &Span,
-) -> UntaggedValue
-where
-    F: Fn(ChunkedArray<Float64Type>, f64) -> ChunkedArray<Float64Type>,
-{
-    match casted {
-        Ok(casted) => {
-            let res = f(casted.clone(), val);
-            let res = res.into_series();
-            NuDataFrame::series_to_untagged(res, span)
-        }
-        Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            e.to_string(),
-            span,
-        )),
-    }
-}
-
-fn compare_series_i64<F>(series: &Series, val: &i64, f: F, span: &Span) -> UntaggedValue
-where
-    F: Fn(&ChunkedArray<Int64Type>, i64) -> ChunkedArray<BooleanType>,
-{
-    match series.dtype() {
-        DataType::UInt32 | DataType::Int32 | DataType::UInt64 => {
-            let to_i64 = series.cast(&DataType::Int64);
-
-            match to_i64 {
-                Ok(series) => {
-                    let casted = series.i64();
-                    compare_casted_i64(casted, *val, f, span)
-                }
-                Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                    "Casting error",
-                    e.to_string(),
-                    span,
-                )),
-            }
-        }
-        DataType::Int64 => {
-            let casted = series.i64();
-            compare_casted_i64(casted, *val, f, span)
-        }
-        _ => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            format!(
-                "Series of type {} can not be used for operations with an i64 value",
-                series.dtype()
-            ),
-            span,
-        )),
-    }
-}
-
-fn compare_casted_i64<F>(
-    casted: Result<&ChunkedArray<Int64Type>, PolarsError>,
-    val: i64,
-    f: F,
-    span: &Span,
-) -> UntaggedValue
-where
-    F: Fn(&ChunkedArray<Int64Type>, i64) -> ChunkedArray<BooleanType>,
-{
-    match casted {
-        Ok(casted) => {
-            let res = f(casted, val);
-            let res = res.into_series();
-            NuDataFrame::series_to_untagged(res, span)
-        }
-        Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            e.to_string(),
-            span,
-        )),
-    }
-}
-
-fn compare_series_decimal<F>(series: &Series, val: &BigDecimal, f: F, span: &Span) -> UntaggedValue
-where
-    F: Fn(&ChunkedArray<Float64Type>, f64) -> ChunkedArray<BooleanType>,
-{
-    match series.dtype() {
-        DataType::Float32 => {
-            let to_f64 = series.cast(&DataType::Float64);
-
-            match to_f64 {
-                Ok(series) => {
-                    let casted = series.f64();
-                    compare_casted_f64(
-                        casted,
-                        val.to_f64()
-                            .expect("Internal error: protocol did not use compatible decimal"),
-                        f,
-                        span,
-                    )
-                }
-                Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                    "Casting error",
-                    e.to_string(),
-                    span,
-                )),
-            }
-        }
-        DataType::Float64 => {
-            let casted = series.f64();
-            compare_casted_f64(
-                casted,
-                val.to_f64()
-                    .expect("Internal error: protocol did not use compatible decimal"),
-                f,
-                span,
-            )
-        }
-        _ => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            format!(
-                "Series of type {} can not be used for operations with a decimal value",
-                series.dtype()
-            ),
-            span,
-        )),
-    }
-}
-
-fn compare_casted_f64<F>(
-    casted: Result<&ChunkedArray<Float64Type>, PolarsError>,
-    val: f64,
-    f: F,
-    span: &Span,
-) -> UntaggedValue
-where
-    F: Fn(&ChunkedArray<Float64Type>, f64) -> ChunkedArray<BooleanType>,
-{
-    match casted {
-        Ok(casted) => {
-            let res = f(casted, val);
-            let res = res.into_series();
-            NuDataFrame::series_to_untagged(res, span)
-        }
-        Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            e.to_string(),
-            span,
-        )),
-    }
-}
-
-fn contains_series_pat(series: &Series, pat: &str, span: &Span) -> UntaggedValue {
-    let casted = series.utf8();
-    match casted {
-        Ok(casted) => {
-            let res = casted.contains(pat);
-
-            match res {
-                Ok(res) => {
-                    let res = res.into_series();
-                    NuDataFrame::series_to_untagged(res, span)
-                }
-                Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                    "Search error",
-                    e.to_string(),
-                    span,
-                )),
-            }
-        }
-        Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-            "Casting error",
-            e.to_string(),
-            span,
-        )),
-    }
-}

crates/nu-protocol/src/dataframe/conversion.rs

@@ -1,661 +0,0 @@
-use indexmap::map::{Entry, IndexMap};
-use polars::chunked_array::object::builder::ObjectChunkedBuilder;
-use polars::chunked_array::ChunkedArray;
-
-use bigdecimal::{FromPrimitive, ToPrimitive};
-use chrono::{DateTime, FixedOffset, NaiveDateTime};
-use nu_errors::ShellError;
-use nu_source::{Span, Tag};
-use num_bigint::BigInt;
-use polars::prelude::{
-    DataFrame, DataType, DatetimeChunked, Int64Type, IntoSeries, NamedFrom, NewChunkedArray,
-    ObjectType, PolarsNumericType, Series,
-};
-use std::ops::{Deref, DerefMut};
-
-use super::NuDataFrame;
-use crate::{Dictionary, Primitive, UntaggedValue, Value};
-
-const SECS_PER_DAY: i64 = 86_400;
-
-#[derive(Debug)]
-pub struct Column {
-    name: String,
-    values: Vec<Value>,
-}
-
-impl Column {
-    pub fn new(name: String, values: Vec<Value>) -> Self {
-        Self { name, values }
-    }
-
-    pub fn new_empty(name: String) -> Self {
-        Self {
-            name,
-            values: Vec::new(),
-        }
-    }
-
-    pub fn name(&self) -> &str {
-        self.name.as_str()
-    }
-
-    pub fn iter(&self) -> impl Iterator<Item = &Value> {
-        self.values.iter()
-    }
-}
-
-impl IntoIterator for Column {
-    type Item = Value;
-    type IntoIter = std::vec::IntoIter<Self::Item>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        self.values.into_iter()
-    }
-}
-
-impl Deref for Column {
-    type Target = Vec<Value>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.values
-    }
-}
-
-impl DerefMut for Column {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.values
-    }
-}
-
-#[derive(Debug)]
-pub enum InputType {
-    Integer,
-    Decimal,
-    String,
-    Boolean,
-    Object,
-    Date,
-    Duration,
-}
-
-#[derive(Debug)]
-pub struct TypedColumn {
-    column: Column,
-    column_type: Option<InputType>,
-}
-
-impl TypedColumn {
-    fn new_empty(name: String) -> Self {
-        Self {
-            column: Column::new_empty(name),
-            column_type: None,
-        }
-    }
-}
-
-impl Deref for TypedColumn {
-    type Target = Column;
-
-    fn deref(&self) -> &Self::Target {
-        &self.column
-    }
-}
-
-impl DerefMut for TypedColumn {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.column
-    }
-}
-
-pub type ColumnMap = IndexMap<String, TypedColumn>;
-
-pub fn create_column(
-    series: &Series,
-    from_row: usize,
-    to_row: usize,
-) -> Result<Column, ShellError> {
-    let size = to_row - from_row;
-    match series.dtype() {
-        DataType::Null => {
-            let values = std::iter::repeat(Value {
-                value: UntaggedValue::Primitive(Primitive::Nothing),
-                tag: Tag::default(),
-            })
-            .take(size)
-            .collect::<Vec<Value>>();
-
-            Ok(Column::new(series.name().into(), values))
-        }
-        DataType::UInt8 => {
-            let casted = series.u8().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::UInt16 => {
-            let casted = series.u16().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::UInt32 => {
-            let casted = series.u32().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::UInt64 => {
-            let casted = series.u64().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Int8 => {
-            let casted = series.i8().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Int16 => {
-            let casted = series.i16().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Int32 => {
-            let casted = series.i32().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Int64 => {
-            let casted = series.i64().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Float32 => {
-            let casted = series.f32().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Float64 => {
-            let casted = series.f64().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-            Ok(column_from_casted(casted, from_row, size))
-        }
-        DataType::Boolean => {
-            let casted = series.bool().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-
-            let values = casted
-                .into_iter()
-                .skip(from_row)
-                .take(size)
-                .map(|v| match v {
-                    Some(a) => Value {
-                        value: UntaggedValue::Primitive((a).into()),
-                        tag: Tag::default(),
-                    },
-                    None => Value {
-                        value: UntaggedValue::Primitive(Primitive::Nothing),
-                        tag: Tag::default(),
-                    },
-                })
-                .collect::<Vec<Value>>();
-
-            Ok(Column::new(casted.name().into(), values))
-        }
-        DataType::Utf8 => {
-            let casted = series.utf8().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-
-            let values = casted
-                .into_iter()
-                .skip(from_row)
-                .take(size)
-                .map(|v| match v {
-                    Some(a) => Value {
-                        value: UntaggedValue::Primitive((a).into()),
-                        tag: Tag::default(),
-                    },
-                    None => Value {
-                        value: UntaggedValue::Primitive(Primitive::Nothing),
-                        tag: Tag::default(),
-                    },
-                })
-                .collect::<Vec<Value>>();
-
-            Ok(Column::new(casted.name().into(), values))
-        }
-        DataType::Object(_) => {
-            let casted = series
-                .as_any()
-                .downcast_ref::<ChunkedArray<ObjectType<Value>>>();
-
-            match casted {
-                None => Err(ShellError::labeled_error(
-                    "Format not supported",
-                    "Value not supported for conversion",
-                    Tag::unknown(),
-                )),
-                Some(ca) => {
-                    let values = ca
-                        .into_iter()
-                        .skip(from_row)
-                        .take(size)
-                        .map(|v| match v {
-                            Some(a) => a.clone(),
-                            None => Value {
-                                value: UntaggedValue::Primitive(Primitive::Nothing),
-                                tag: Tag::default(),
-                            },
-                        })
-                        .collect::<Vec<Value>>();
-
-                    Ok(Column::new(ca.name().into(), values))
-                }
-            }
-        }
-        DataType::Date => {
-            let casted = series.date().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-
-            let values = casted
-                .into_iter()
-                .skip(from_row)
-                .take(size)
-                .map(|v| match v {
-                    Some(a) => {
-                        // elapsed time in day since 1970-01-01
-                        let seconds = a as i64 * SECS_PER_DAY;
-                        let naive_datetime = NaiveDateTime::from_timestamp(seconds, 0);
-
-                        // Zero length offset
-                        let offset = FixedOffset::east(0);
-                        let datetime = DateTime::<FixedOffset>::from_utc(naive_datetime, offset);
-
-                        Value {
-                            value: UntaggedValue::Primitive(Primitive::Date(datetime)),
-                            tag: Tag::default(),
-                        }
-                    }
-                    None => Value {
-                        value: UntaggedValue::Primitive(Primitive::Nothing),
-                        tag: Tag::default(),
-                    },
-                })
-                .collect::<Vec<Value>>();
-
-            Ok(Column::new(casted.name().into(), values))
-        }
-        DataType::Datetime => {
-            let casted = series.datetime().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-
-            let values = casted
-                .into_iter()
-                .skip(from_row)
-                .take(size)
-                .map(|v| match v {
-                    Some(a) => {
-                        // elapsed time in milliseconds since 1970-01-01
-                        let seconds = a / 1000;
-                        let naive_datetime = NaiveDateTime::from_timestamp(seconds, 0);
-
-                        // Zero length offset
-                        let offset = FixedOffset::east(0);
-                        let datetime = DateTime::<FixedOffset>::from_utc(naive_datetime, offset);
-
-                        Value {
-                            value: UntaggedValue::Primitive(Primitive::Date(datetime)),
-                            tag: Tag::default(),
-                        }
-                    }
-                    None => Value {
-                        value: UntaggedValue::Primitive(Primitive::Nothing),
-                        tag: Tag::default(),
-                    },
-                })
-                .collect::<Vec<Value>>();
-
-            Ok(Column::new(casted.name().into(), values))
-        }
-        DataType::Time => {
-            let casted = series.time().map_err(|e| {
-                ShellError::labeled_error(
-                    "Casting error",
-                    format!("casting error: {}", e),
-                    Span::default(),
-                )
-            })?;
-
-            let values = casted
-                .into_iter()
-                .skip(from_row)
-                .take(size)
-                .map(|v| match v {
-                    Some(nanoseconds) => {
-                        let untagged = if let Some(bigint) = BigInt::from_i64(nanoseconds) {
-                            UntaggedValue::Primitive(Primitive::Duration(bigint))
-                        } else {
-                            unreachable!("Internal error: protocol did not use compatible decimal")
-                        };
-
-                        Value {
-                            value: untagged,
-                            tag: Tag::default(),
-                        }
-                    }
-                    None => Value {
-                        value: UntaggedValue::Primitive(Primitive::Nothing),
-                        tag: Tag::default(),
-                    },
-                })
-                .collect::<Vec<Value>>();
-
-            Ok(Column::new(casted.name().into(), values))
-        }
-        e => Err(ShellError::labeled_error(
-            "Format not supported",
-            format!("Value not supported for conversion: {}", e),
-            Tag::unknown(),
-        )),
-    }
-}
-
-fn column_from_casted<T>(casted: &ChunkedArray<T>, from_row: usize, size: usize) -> Column
-where
-    T: PolarsNumericType,
-    T::Native: Into<Primitive>,
-{
-    let values = casted
-        .into_iter()
-        .skip(from_row)
-        .take(size)
-        .map(|v| match v {
-            Some(a) => Value {
-                value: UntaggedValue::Primitive((a).into()),
-                tag: Tag::default(),
-            },
-            None => Value {
-                value: UntaggedValue::Primitive(Primitive::Nothing),
-                tag: Tag::default(),
-            },
-        })
-        .collect::<Vec<Value>>();
-
-    Column::new(casted.name().into(), values)
-}
-
-// Adds a separator to the vector of values using the column names from the
-// dataframe to create the Values Row
-pub fn add_separator(values: &mut Vec<Value>, df: &DataFrame) {
-    let column_names = df.get_column_names();
-
-    let mut dictionary = Dictionary::default();
-    for name in column_names {
-        let indicator = Value {
-            value: UntaggedValue::Primitive(Primitive::String("...".to_string())),
-            tag: Tag::unknown(),
-        };
-
-        dictionary.insert(name.to_string(), indicator);
-    }
-
-    let extra_column = Value {
-        value: UntaggedValue::Row(dictionary),
-        tag: Tag::unknown(),
-    };
-
-    values.push(extra_column);
-}
-
-// Inserting the values found in a UntaggedValue::Row
-// All the entries for the dictionary are checked in order to check if
-// the column values have the same type value.
-pub fn insert_row(column_values: &mut ColumnMap, dictionary: Dictionary) -> Result<(), ShellError> {
-    for (key, value) in dictionary.entries {
-        insert_value(value, key, column_values)?;
-    }
-
-    Ok(())
-}
-
-// Inserting the values found in a UntaggedValue::Table
-// All the entries for the table are checked in order to check if
-// the column values have the same type value.
-// The names for the columns are the enumerated numbers from the values
-pub fn insert_table(column_values: &mut ColumnMap, table: Vec<Value>) -> Result<(), ShellError> {
-    for (index, value) in table.into_iter().enumerate() {
-        let key = index.to_string();
-        insert_value(value, key, column_values)?;
-    }
-
-    Ok(())
-}
-
-pub fn insert_value(
-    value: Value,
-    key: String,
-    column_values: &mut ColumnMap,
-) -> Result<(), ShellError> {
-    let col_val = match column_values.entry(key.clone()) {
-        Entry::Vacant(entry) => entry.insert(TypedColumn::new_empty(key)),
-        Entry::Occupied(entry) => entry.into_mut(),
-    };
-
-    // Checking that the type for the value is the same
-    // for the previous value in the column
-    if col_val.values.is_empty() {
-        match &value.value {
-            UntaggedValue::Primitive(Primitive::Int(_)) => {
-                col_val.column_type = Some(InputType::Integer);
-            }
-            UntaggedValue::Primitive(Primitive::Decimal(_)) => {
-                col_val.column_type = Some(InputType::Decimal);
-            }
-            UntaggedValue::Primitive(Primitive::String(_)) => {
-                col_val.column_type = Some(InputType::String);
-            }
-            UntaggedValue::Primitive(Primitive::Boolean(_)) => {
-                col_val.column_type = Some(InputType::Boolean);
-            }
-            UntaggedValue::Primitive(Primitive::Date(_)) => {
-                col_val.column_type = Some(InputType::Date);
-            }
-            UntaggedValue::Primitive(Primitive::Duration(_)) => {
-                col_val.column_type = Some(InputType::Duration);
-            }
-            _ => col_val.column_type = Some(InputType::Object),
-        }
-        col_val.values.push(value);
-    } else {
-        let prev_value = &col_val.values[col_val.values.len() - 1];
-
-        match (&prev_value.value, &value.value) {
-            (
-                UntaggedValue::Primitive(Primitive::Int(_)),
-                UntaggedValue::Primitive(Primitive::Int(_)),
-            )
-            | (
-                UntaggedValue::Primitive(Primitive::Decimal(_)),
-                UntaggedValue::Primitive(Primitive::Decimal(_)),
-            )
-            | (
-                UntaggedValue::Primitive(Primitive::String(_)),
-                UntaggedValue::Primitive(Primitive::String(_)),
-            )
-            | (
-                UntaggedValue::Primitive(Primitive::Boolean(_)),
-                UntaggedValue::Primitive(Primitive::Boolean(_)),
-            )
-            | (
-                UntaggedValue::Primitive(Primitive::Date(_)),
-                UntaggedValue::Primitive(Primitive::Date(_)),
-            )
-            | (
-                UntaggedValue::Primitive(Primitive::Duration(_)),
-                UntaggedValue::Primitive(Primitive::Duration(_)),
-            ) => col_val.values.push(value),
-            _ => {
-                col_val.column_type = Some(InputType::Object);
-                col_val.values.push(value);
-            }
-        }
-    }
-
-    Ok(())
-}
-
-// The ColumnMap has the parsed data from the StreamInput
-// This data can be used to create a Series object that can initialize
-// the dataframe based on the type of data that is found
-pub fn from_parsed_columns(
-    column_values: ColumnMap,
-    span: &Span,
-) -> Result<NuDataFrame, ShellError> {
-    let mut df_series: Vec<Series> = Vec::new();
-    for (name, column) in column_values {
-        if let Some(column_type) = &column.column_type {
-            match column_type {
-                InputType::Decimal => {
-                    let series_values: Result<Vec<_>, _> =
-                        column.values.iter().map(|v| v.as_f64()).collect();
-                    let series = Series::new(&name, series_values?);
-                    df_series.push(series)
-                }
-                InputType::Integer => {
-                    let series_values: Result<Vec<_>, _> =
-                        column.values.iter().map(|v| v.as_i64()).collect();
-                    let series = Series::new(&name, series_values?);
-                    df_series.push(series)
-                }
-                InputType::String => {
-                    let series_values: Result<Vec<_>, _> =
-                        column.values.iter().map(|v| v.as_string()).collect();
-                    let series = Series::new(&name, series_values?);
-                    df_series.push(series)
-                }
-                InputType::Boolean => {
-                    let series_values: Result<Vec<_>, _> =
-                        column.values.iter().map(|v| v.as_bool()).collect();
-                    let series = Series::new(&name, series_values?);
-                    df_series.push(series)
-                }
-                InputType::Object => {
-                    let mut builder =
-                        ObjectChunkedBuilder::<Value>::new(&name, column.values.len());
-
-                    for v in &column.values {
-                        builder.append_value(v.clone());
-                    }
-
-                    let res = builder.finish();
-                    df_series.push(res.into_series())
-                }
-                InputType::Date => {
-                    let it = column.values.iter().map(|v| {
-                        if let UntaggedValue::Primitive(Primitive::Date(date)) = &v.value {
-                            Some(date.timestamp_millis())
-                        } else {
-                            None
-                        }
-                    });
-
-                    let res: DatetimeChunked =
-                        ChunkedArray::<Int64Type>::new_from_opt_iter(&name, it).into();
-
-                    df_series.push(res.into_series())
-                }
-                InputType::Duration => {
-                    let it = column.values.iter().map(|v| {
-                        if let UntaggedValue::Primitive(Primitive::Duration(duration)) = &v.value {
-                            Some(duration.to_i64().expect("Not expecting NAN in duration"))
-                        } else {
-                            None
-                        }
-                    });
-
-                    let res = ChunkedArray::<Int64Type>::new_from_opt_iter(&name, it);
-
-                    df_series.push(res.into_series())
-                }
-            }
-        }
-    }
-
-    let df = DataFrame::new(df_series);
-
-    match df {
-        Ok(df) => Ok(NuDataFrame::new(df)),
-        Err(e) => Err(ShellError::labeled_error(
-            "Error while creating dataframe",
-            e.to_string(),
-            span,
-        )),
-    }
-}

crates/nu-protocol/src/dataframe/mod.rs

@@ -1,17 +0,0 @@
-pub mod compute_between;
-pub mod conversion;
-pub mod nu_dataframe;
-pub mod nu_groupby;
-pub mod operations;
-
-pub use compute_between::{compute_between_dataframes, compute_series_single_value};
-pub use conversion::Column;
-pub use nu_dataframe::NuDataFrame;
-pub use nu_groupby::NuGroupBy;
-pub use operations::Axis;
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
-pub enum FrameStruct {
-    GroupBy(NuGroupBy),
-}

crates/nu-protocol/src/dataframe/nu_dataframe.rs

@@ -1,401 +0,0 @@
-use indexmap::IndexMap;
-use std::cmp::Ordering;
-use std::fmt::Display;
-use std::hash::{Hash, Hasher};
-
-use nu_errors::ShellError;
-use nu_source::{Span, Tag};
-use polars::prelude::{DataFrame, DataType, PolarsObject, Series};
-use serde::{Deserialize, Serialize};
-
-use super::conversion::{
-    add_separator, create_column, from_parsed_columns, insert_row, insert_table, insert_value,
-    Column, ColumnMap,
-};
-use crate::{Dictionary, Primitive, ShellTypeName, UntaggedValue, Value};
-
-impl Display for Value {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.type_name())
-    }
-}
-
-impl Default for Value {
-    fn default() -> Self {
-        Self {
-            value: UntaggedValue::Primitive(Primitive::Nothing),
-            tag: Tag::default(),
-        }
-    }
-}
-
-impl PolarsObject for Value {
-    fn type_name() -> &'static str {
-        "object"
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct NuDataFrame {
-    dataframe: DataFrame,
-}
-
-// Dataframes are considered equal if they have the same shape, column name
-// and values
-impl PartialEq for NuDataFrame {
-    fn eq(&self, other: &Self) -> bool {
-        if self.as_ref().width() == 0 {
-            // checking for empty dataframe
-            return false;
-        }
-
-        if self.as_ref().get_column_names() != other.as_ref().get_column_names() {
-            // checking both dataframes share the same names
-            return false;
-        }
-
-        if self.as_ref().height() != other.as_ref().height() {
-            // checking both dataframes have the same row size
-            return false;
-        }
-
-        // sorting dataframe by the first column
-        let column_names = self.as_ref().get_column_names();
-        let first_col = column_names
-            .get(0)
-            .expect("already checked that dataframe is different than 0");
-
-        // if unable to sort, then unable to compare
-        let lhs = match self.as_ref().sort(*first_col, false) {
-            Ok(df) => df,
-            Err(_) => return false,
-        };
-
-        let rhs = match other.as_ref().sort(*first_col, false) {
-            Ok(df) => df,
-            Err(_) => return false,
-        };
-
-        for name in self.as_ref().get_column_names() {
-            let self_series = lhs.column(name).expect("name from dataframe names");
-
-            let other_series = rhs
-                .column(name)
-                .expect("already checked that name in other");
-
-            let self_series = match self_series.dtype() {
-                // Casting needed to compare other numeric types with nushell numeric type.
-                // In nushell we only have i64 integer numeric types and any array created
-                // with nushell untagged primitives will be of type i64
-                DataType::UInt32 => match self_series.cast(&DataType::Int64) {
-                    Ok(series) => series,
-                    Err(_) => return false,
-                },
-                _ => self_series.clone(),
-            };
-
-            if !self_series.series_equal(other_series) {
-                return false;
-            }
-        }
-
-        true
-    }
-}
-
-impl Eq for NuDataFrame {}
-
-impl PartialOrd for NuDataFrame {
-    fn partial_cmp(&self, _: &Self) -> Option<Ordering> {
-        Some(Ordering::Equal)
-    }
-}
-
-impl Ord for NuDataFrame {
-    fn cmp(&self, _: &Self) -> Ordering {
-        Ordering::Equal
-    }
-}
-
-impl Hash for NuDataFrame {
-    fn hash<H: Hasher>(&self, _: &mut H) {}
-}
-
-impl AsRef<DataFrame> for NuDataFrame {
-    fn as_ref(&self) -> &polars::prelude::DataFrame {
-        &self.dataframe
-    }
-}
-
-impl AsMut<DataFrame> for NuDataFrame {
-    fn as_mut(&mut self) -> &mut polars::prelude::DataFrame {
-        &mut self.dataframe
-    }
-}
-
-impl NuDataFrame {
-    pub fn new(dataframe: polars::prelude::DataFrame) -> Self {
-        NuDataFrame { dataframe }
-    }
-
-    pub fn try_from_stream<T>(input: &mut T, span: &Span) -> Result<(Self, Tag), ShellError>
-    where
-        T: Iterator<Item = Value>,
-    {
-        input
-            .next()
-            .and_then(|value| match value.value {
-                UntaggedValue::DataFrame(df) => Some((df, value.tag)),
-                _ => None,
-            })
-            .ok_or_else(|| {
-                ShellError::labeled_error(
-                    "No dataframe in stream",
-                    "no dataframe found in input stream",
-                    span,
-                )
-            })
-    }
-
-    pub fn try_from_iter<T>(iter: T, tag: &Tag) -> Result<Self, ShellError>
-    where
-        T: Iterator<Item = Value>,
-    {
-        // Dictionary to store the columnar data extracted from
-        // the input. During the iteration we check if the values
-        // have different type
-        let mut column_values: ColumnMap = IndexMap::new();
-
-        for value in iter {
-            match value.value {
-                UntaggedValue::Row(dictionary) => insert_row(&mut column_values, dictionary)?,
-                UntaggedValue::Table(table) => insert_table(&mut column_values, table)?,
-                UntaggedValue::Primitive(Primitive::Int(_))
-                | UntaggedValue::Primitive(Primitive::Decimal(_))
-                | UntaggedValue::Primitive(Primitive::String(_))
-                | UntaggedValue::Primitive(Primitive::Boolean(_))
-                | UntaggedValue::Primitive(Primitive::Date(_))
-                | UntaggedValue::DataFrame(_) => {
-                    let key = "0".to_string();
-                    insert_value(value, key, &mut column_values)?
-                }
-                _ => {
-                    return Err(ShellError::labeled_error_with_secondary(
-                        "Format not supported",
-                        "Value not supported for conversion",
-                        &value.tag,
-                        "Perhaps you want to use a List, a List of Tables or a Dictionary",
-                        &value.tag,
-                    ));
-                }
-            }
-        }
-
-        from_parsed_columns(column_values, &tag.span)
-    }
-
-    pub fn try_from_series(columns: Vec<Series>, span: &Span) -> Result<Self, ShellError> {
-        let dataframe = DataFrame::new(columns).map_err(|e| {
-            ShellError::labeled_error(
-                "DataFrame Creation",
-                format!("Unable to create DataFrame: {}", e),
-                span,
-            )
-        })?;
-
-        Ok(Self { dataframe })
-    }
-
-    pub fn try_from_columns(columns: Vec<Column>, span: &Span) -> Result<Self, ShellError> {
-        let mut column_values: ColumnMap = IndexMap::new();
-
-        for column in columns {
-            let name = column.name().to_string();
-            for value in column {
-                insert_value(value, name.clone(), &mut column_values)?;
-            }
-        }
-
-        from_parsed_columns(column_values, span)
-    }
-
-    pub fn into_value(self, tag: Tag) -> Value {
-        Value {
-            value: Self::into_untagged(self),
-            tag,
-        }
-    }
-
-    pub fn into_untagged(self) -> UntaggedValue {
-        UntaggedValue::DataFrame(self)
-    }
-
-    pub fn dataframe_to_value(df: DataFrame, tag: Tag) -> Value {
-        Value {
-            value: Self::dataframe_to_untagged(df),
-            tag,
-        }
-    }
-
-    pub fn dataframe_to_untagged(df: DataFrame) -> UntaggedValue {
-        UntaggedValue::DataFrame(Self::new(df))
-    }
-
-    pub fn series_to_untagged(series: Series, span: &Span) -> UntaggedValue {
-        match DataFrame::new(vec![series]) {
-            Ok(dataframe) => UntaggedValue::DataFrame(Self { dataframe }),
-            Err(e) => UntaggedValue::Error(ShellError::labeled_error(
-                "DataFrame Creation",
-                format!("Unable to create DataFrame: {}", e),
-                span,
-            )),
-        }
-    }
-
-    pub fn column(&self, column: &str, tag: &Tag) -> Result<Self, ShellError> {
-        let s = self
-            .as_ref()
-            .column(column)
-            .map_err(|e| ShellError::labeled_error("Column not found", e.to_string(), tag.span))?;
-
-        let dataframe = DataFrame::new(vec![s.clone()])
-            .map_err(|e| ShellError::labeled_error("DataFrame error", e.to_string(), tag.span))?;
-
-        Ok(Self { dataframe })
-    }
-
-    pub fn is_series(&self) -> bool {
-        self.as_ref().width() == 1
-    }
-
-    pub fn as_series(&self, span: &Span) -> Result<Series, ShellError> {
-        if !self.is_series() {
-            return Err(ShellError::labeled_error_with_secondary(
-                "Not a Series",
-                "DataFrame cannot be used as Series",
-                span,
-                "Note that a Series is a DataFrame with one column",
-                span,
-            ));
-        }
-
-        let series = self
-            .as_ref()
-            .get_columns()
-            .get(0)
-            .expect("We have already checked that the width is 1");
-
-        Ok(series.clone())
-    }
-
-    pub fn get_value(&self, row: usize, span: Span) -> Result<Value, ShellError> {
-        let series = self.as_series(&Span::default())?;
-        let column = create_column(&series, row, row + 1)?;
-
-        if column.len() == 0 {
-            Err(ShellError::labeled_error_with_secondary(
-                "Not a valid row",
-                format!("No value found for index {}", row),
-                span,
-                format!("Note that the column size is {}", series.len()),
-                span,
-            ))
-        } else {
-            let value = column
-                .into_iter()
-                .next()
-                .expect("already checked there is a value");
-            Ok(value)
-        }
-    }
-
-    // Print is made out a head and if the dataframe is too large, then a tail
-    pub fn print(&self) -> Result<Vec<Value>, ShellError> {
-        let df = &self.as_ref();
-        let size: usize = 20;
-
-        if df.height() > size {
-            let sample_size = size / 2;
-            let mut values = self.head(Some(sample_size))?;
-            add_separator(&mut values, df);
-            let remaining = df.height() - sample_size;
-            let tail_size = remaining.min(sample_size);
-            let mut tail_values = self.tail(Some(tail_size))?;
-            values.append(&mut tail_values);
-
-            Ok(values)
-        } else {
-            Ok(self.head(Some(size))?)
-        }
-    }
-
-    pub fn head(&self, rows: Option<usize>) -> Result<Vec<Value>, ShellError> {
-        let to_row = rows.unwrap_or(5);
-        let values = self.to_rows(0, to_row)?;
-
-        Ok(values)
-    }
-
-    pub fn tail(&self, rows: Option<usize>) -> Result<Vec<Value>, ShellError> {
-        let df = &self.as_ref();
-        let to_row = df.height();
-        let size = rows.unwrap_or(5);
-        let from_row = to_row.saturating_sub(size);
-
-        let values = self.to_rows(from_row, to_row)?;
-
-        Ok(values)
-    }
-
-    pub fn to_rows(&self, from_row: usize, to_row: usize) -> Result<Vec<Value>, ShellError> {
-        let df = self.as_ref();
-        let upper_row = to_row.min(df.height());
-
-        let mut size: usize = 0;
-        let columns = self
-            .as_ref()
-            .get_columns()
-            .iter()
-            .map(|col| match create_column(col, from_row, upper_row) {
-                Ok(col) => {
-                    size = col.len();
-                    Ok(col)
-                }
-                Err(e) => Err(e),
-            })
-            .collect::<Result<Vec<Column>, ShellError>>()?;
-
-        let mut iterators = columns
-            .into_iter()
-            .map(|col| (col.name().to_string(), col.into_iter()))
-            .collect::<Vec<(String, std::vec::IntoIter<Value>)>>();
-
-        let values = (0..size)
-            .into_iter()
-            .map(|i| {
-                let mut dictionary_row = Dictionary::default();
-
-                for (name, col) in &mut iterators {
-                    let dict_val = match col.next() {
-                        Some(v) => v,
-                        None => {
-                            println!("index: {}", i);
-                            Value {
-                                value: UntaggedValue::Primitive(Primitive::Nothing),
-                                tag: Tag::default(),
-                            }
-                        }
-                    };
-                    dictionary_row.insert(name.clone(), dict_val);
-                }
-
-                Value {
-                    value: UntaggedValue::Row(dictionary_row),
-                    tag: Tag::unknown(),
-                }
-            })
-            .collect::<Vec<Value>>();
-
-        Ok(values)
-    }
-}

crates/nu-protocol/src/dataframe/nu_groupby.rs

@@ -1,75 +0,0 @@
-use nu_source::{Span, Tag};
-use polars::frame::groupby::{GroupBy, GroupTuples};
-use serde::{Deserialize, Serialize};
-
-use super::{FrameStruct, NuDataFrame};
-use nu_errors::ShellError;
-
-use crate::{TaggedDictBuilder, UntaggedValue, Value};
-
-#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
-pub struct NuGroupBy {
-    dataframe: NuDataFrame,
-    by: Vec<String>,
-    groups: GroupTuples,
-}
-
-impl NuGroupBy {
-    pub fn new(dataframe: NuDataFrame, by: Vec<String>, groups: GroupTuples) -> Self {
-        NuGroupBy {
-            dataframe,
-            by,
-            groups,
-        }
-    }
-
-    pub fn by(&self) -> &[String] {
-        &self.by
-    }
-
-    pub fn try_from_stream<T>(input: &mut T, span: &Span) -> Result<NuGroupBy, ShellError>
-    where
-        T: Iterator<Item = Value>,
-    {
-        input
-            .next()
-            .and_then(|value| match value.value {
-                UntaggedValue::FrameStruct(FrameStruct::GroupBy(group)) => Some(group),
-                _ => None,
-            })
-            .ok_or_else(|| {
-                ShellError::labeled_error(
-                    "No groupby object in stream",
-                    "no groupby object found in input stream",
-                    span,
-                )
-            })
-    }
-
-    pub fn to_groupby(&self) -> Result<GroupBy, ShellError> {
-        let df = self.dataframe.as_ref();
-
-        let by = df.select_series(&self.by).map_err(|e| {
-            ShellError::labeled_error("Error creating groupby", e.to_string(), Tag::unknown())
-        })?;
-
-        Ok(GroupBy::new(df, by, self.groups.clone(), None))
-    }
-
-    pub fn print(&self) -> Result<Vec<Value>, ShellError> {
-        let mut values: Vec<Value> = Vec::new();
-
-        let mut data = TaggedDictBuilder::new(Tag::unknown());
-        data.insert_value("property", "group by");
-        data.insert_value("value", self.by.join(", "));
-        values.push(data.into_value());
-
-        Ok(values)
-    }
-}
-
-impl AsRef<polars::prelude::DataFrame> for NuGroupBy {
-    fn as_ref(&self) -> &polars::prelude::DataFrame {
-        self.dataframe.as_ref()
-    }
-}

crates/nu-protocol/src/dataframe/operations.rs

@@ -1,120 +0,0 @@
-use nu_errors::ShellError;
-use nu_source::Span;
-use polars::prelude::{DataFrame, Series};
-
-use super::NuDataFrame;
-
-pub enum Axis {
-    Row,
-    Column,
-}
-
-impl Axis {
-    pub fn try_from_str(axis: &str, span: &Span) -> Result<Axis, ShellError> {
-        match axis {
-            "row" => Ok(Axis::Row),
-            "col" => Ok(Axis::Column),
-            _ => Err(ShellError::labeled_error_with_secondary(
-                "Wrong axis",
-                "The selected axis does not exist",
-                span,
-                "The only axis options are 'row' or 'col'",
-                span,
-            )),
-        }
-    }
-}
-
-impl NuDataFrame {
-    pub fn append_df(
-        &self,
-        other: &NuDataFrame,
-        axis: Axis,
-        span: &Span,
-    ) -> Result<Self, ShellError> {
-        match axis {
-            Axis::Row => {
-                let mut columns: Vec<&str> = Vec::new();
-
-                let new_cols = self
-                    .as_ref()
-                    .get_columns()
-                    .iter()
-                    .chain(other.as_ref().get_columns())
-                    .map(|s| {
-                        let name = if columns.contains(&s.name()) {
-                            format!("{}_{}", s.name(), "x")
-                        } else {
-                            columns.push(s.name());
-                            s.name().to_string()
-                        };
-
-                        let mut series = s.clone();
-                        series.rename(&name);
-                        series
-                    })
-                    .collect::<Vec<Series>>();
-
-                let df_new = DataFrame::new(new_cols).map_err(|e| {
-                    ShellError::labeled_error("Appending error", e.to_string(), span)
-                })?;
-
-                Ok(NuDataFrame::new(df_new))
-            }
-            Axis::Column => {
-                if self.as_ref().width() != other.as_ref().width() {
-                    return Err(ShellError::labeled_error(
-                        "Appending error",
-                        "Dataframes with different number of columns",
-                        span,
-                    ));
-                }
-
-                if !self
-                    .as_ref()
-                    .get_column_names()
-                    .iter()
-                    .all(|col| other.as_ref().get_column_names().contains(col))
-                {
-                    return Err(ShellError::labeled_error(
-                        "Appending error",
-                        "Dataframes with different columns names",
-                        span,
-                    ));
-                }
-
-                let new_cols = self
-                    .as_ref()
-                    .get_columns()
-                    .iter()
-                    .map(|s| {
-                        let other_col = other
-                            .as_ref()
-                            .column(s.name())
-                            .expect("Already checked that dataframes have same columns");
-
-                        let mut tmp = s.clone();
-                        let res = tmp.append(other_col);
-
-                        match res {
-                            Ok(s) => Ok(s.clone()),
-                            Err(e) => Err({
-                                ShellError::labeled_error(
-                                    "Appending error",
-                                    format!("Unable to append dataframes: {}", e),
-                                    span,
-                                )
-                            }),
-                        }
-                    })
-                    .collect::<Result<Vec<Series>, ShellError>>()?;
-
-                let df_new = DataFrame::new(new_cols).map_err(|e| {
-                    ShellError::labeled_error("Appending error", e.to_string(), span)
-                })?;
-
-                Ok(NuDataFrame::new(df_new))
-            }
-        }
-    }
-}

crates/nu-protocol/src/engine/call_info.rs

@@ -0,0 +1,8 @@
+use crate::ast::Call;
+use crate::Span;
+
+#[derive(Debug, Clone)]
+pub struct UnevaluatedCallInfo {
+    pub args: Call,
+    pub name_span: Span,
+}

crates/nu-protocol/src/engine/capture_block.rs

@@ -0,0 +1,9 @@
+use std::collections::HashMap;
+
+use crate::{BlockId, Value, VarId};
+
+#[derive(Clone, Debug)]
+pub struct CaptureBlock {
+    pub block_id: BlockId,
+    pub captures: HashMap<VarId, Value>,
+}

crates/nu-protocol/src/engine/command.rs

@@ -0,0 +1,78 @@
+use std::path::PathBuf;
+
+use crate::{ast::Call, BlockId, Example, PipelineData, ShellError, Signature};
+
+use super::{EngineState, Stack};
+
+pub trait Command: Send + Sync + CommandClone {
+    fn name(&self) -> &str;
+
+    fn signature(&self) -> Signature;
+
+    fn usage(&self) -> &str;
+
+    fn extra_usage(&self) -> &str {
+        ""
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError>;
+
+    fn is_binary(&self) -> bool {
+        false
+    }
+
+    // Commands that are not meant to be run by users
+    fn is_private(&self) -> bool {
+        false
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        Vec::new()
+    }
+
+    // This is a built-in command
+    fn is_builtin(&self) -> bool {
+        true
+    }
+
+    // Is a sub command
+    fn is_sub(&self) -> bool {
+        self.name().contains(' ')
+    }
+
+    // Is a plugin command (returns plugin's path, encoding and type of shell
+    // if the declaration is a plugin)
+    fn is_plugin(&self) -> Option<(&PathBuf, &str, &Option<PathBuf>)> {
+        None
+    }
+
+    // If command is a block i.e. def blah [] { }, get the block id
+    fn get_block_id(&self) -> Option<BlockId> {
+        None
+    }
+}
+
+pub trait CommandClone {
+    fn clone_box(&self) -> Box<dyn Command>;
+}
+
+impl<T> CommandClone for T
+where
+    T: 'static + Command + Clone,
+{
+    fn clone_box(&self) -> Box<dyn Command> {
+        Box::new(self.clone())
+    }
+}
+
+impl Clone for Box<dyn Command> {
+    fn clone(&self) -> Box<dyn Command> {
+        self.clone_box()
+    }
+}

crates/nu-protocol/src/engine/mod.rs

@@ -0,0 +1,11 @@
+mod call_info;
+mod capture_block;
+mod command;
+mod engine_state;
+mod stack;
+
+pub use call_info::*;
+pub use capture_block::*;
+pub use command::*;
+pub use engine_state::*;
+pub use stack::*;

crates/nu-protocol/src/engine/stack.rs

@@ -0,0 +1,249 @@
+use std::collections::{HashMap, HashSet};
+
+use crate::engine::EngineState;
+use crate::{Config, ShellError, Span, Value, VarId, CONFIG_VARIABLE_ID};
+
+/// A runtime value stack used during evaluation
+///
+/// A note on implementation:
+///
+/// We previously set up the stack in a traditional way, where stack frames had parents which would
+/// represent other frames that you might return to when exiting a function.
+///
+/// While experimenting with blocks, we found that we needed to have closure captures of variables
+/// seen outside of the blocks, so that they blocks could be run in a way that was both thread-safe
+/// and followed the restrictions for closures applied to iterators. The end result left us with
+/// closure-captured single stack frames that blocks could see.
+///
+/// Blocks make up the only scope and stack definition abstraction in Nushell. As a result, we were
+/// creating closure captures at any point we wanted to have a Block value we could safely evaluate
+/// in any context. This meant that the parents were going largely unused, with captured variables
+/// taking their place. The end result is this, where we no longer have separate frames, but instead
+/// use the Stack as a way of representing the local and closure-captured state.
+#[derive(Debug, Clone)]
+pub struct Stack {
+    /// Variables
+    pub vars: HashMap<VarId, Value>,
+    /// Environment variables arranged as a stack to be able to recover values from parent scopes
+    pub env_vars: Vec<HashMap<String, Value>>,
+    /// Tells which environment variables from engine state are hidden. We don't need to track the
+    /// env vars in the stack since we can just delete them.
+    pub env_hidden: HashSet<String>,
+}
+
+impl Default for Stack {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Stack {
+    pub fn new() -> Stack {
+        Stack {
+            vars: HashMap::new(),
+            env_vars: vec![],
+            env_hidden: HashSet::new(),
+        }
+    }
+
+    pub fn with_env(&mut self, env_vars: &[HashMap<String, Value>], env_hidden: &HashSet<String>) {
+        // Do not clone the environment if it hasn't changed
+        if self.env_vars.iter().any(|scope| !scope.is_empty()) {
+            self.env_vars = env_vars.to_owned();
+        }
+
+        if !self.env_hidden.is_empty() {
+            self.env_hidden = env_hidden.clone();
+        }
+    }
+
+    pub fn get_var(&self, var_id: VarId, span: Span) -> Result<Value, ShellError> {
+        if let Some(v) = self.vars.get(&var_id) {
+            return Ok(v.clone().with_span(span));
+        }
+
+        Err(ShellError::VariableNotFoundAtRuntime(span))
+    }
+
+    pub fn get_var_with_origin(&self, var_id: VarId, span: Span) -> Result<Value, ShellError> {
+        if let Some(v) = self.vars.get(&var_id) {
+            return Ok(v.clone());
+        }
+
+        Err(ShellError::VariableNotFoundAtRuntime(span))
+    }
+
+    pub fn add_var(&mut self, var_id: VarId, value: Value) {
+        self.vars.insert(var_id, value);
+    }
+
+    pub fn add_env_var(&mut self, var: String, value: Value) {
+        // if the env var was hidden, let's activate it again
+        self.env_hidden.remove(&var);
+
+        if let Some(scope) = self.env_vars.last_mut() {
+            scope.insert(var, value);
+        } else {
+            self.env_vars.push(HashMap::from([(var, value)]));
+        }
+    }
+
+    pub fn captures_to_stack(&self, captures: &HashMap<VarId, Value>) -> Stack {
+        let mut output = Stack::new();
+
+        output.vars = captures.clone();
+
+        // FIXME: this is probably slow
+        output.env_vars = self.env_vars.clone();
+        output.env_vars.push(HashMap::new());
+
+        let config = self
+            .get_var(CONFIG_VARIABLE_ID, Span::new(0, 0))
+            .expect("internal error: config is missing");
+        output.vars.insert(CONFIG_VARIABLE_ID, config);
+
+        output
+    }
+
+    pub fn gather_captures(&self, captures: &[VarId]) -> Stack {
+        let mut output = Stack::new();
+
+        let fake_span = Span::new(0, 0);
+
+        for capture in captures {
+            // Note: this assumes we have calculated captures correctly and that commands
+            // that take in a var decl will manually set this into scope when running the blocks
+            if let Ok(value) = self.get_var(*capture, fake_span) {
+                output.vars.insert(*capture, value);
+            }
+        }
+
+        // FIXME: this is probably slow
+        output.env_vars = self.env_vars.clone();
+        output.env_vars.push(HashMap::new());
+
+        let config = self
+            .get_var(CONFIG_VARIABLE_ID, fake_span)
+            .expect("internal error: config is missing");
+        output.vars.insert(CONFIG_VARIABLE_ID, config);
+
+        output
+    }
+
+    /// Flatten the env var scope frames into one frame
+    pub fn get_env_vars(&self, engine_state: &EngineState) -> HashMap<String, Value> {
+        // TODO: We're collecting im::HashMap to HashMap here. It might make sense to make these
+        // the same data structure.
+        let mut result: HashMap<String, Value> = engine_state
+            .env_vars
+            .iter()
+            .filter(|(k, _)| !self.env_hidden.contains(*k))
+            .map(|(k, v)| (k.clone(), v.clone()))
+            .collect();
+
+        for scope in &self.env_vars {
+            result.extend(scope.clone());
+        }
+
+        result
+    }
+
+    /// Same as get_env_vars, but returns only the names as a HashSet
+    pub fn get_env_var_names(&self, engine_state: &EngineState) -> HashSet<String> {
+        let mut result: HashSet<String> = engine_state
+            .env_vars
+            .keys()
+            .filter(|k| !self.env_hidden.contains(*k))
+            .cloned()
+            .collect();
+
+        for scope in &self.env_vars {
+            let scope_keys: HashSet<String> = scope.keys().cloned().collect();
+            result.extend(scope_keys);
+        }
+
+        result
+    }
+
+    pub fn get_env_var(&self, engine_state: &EngineState, name: &str) -> Option<Value> {
+        for scope in self.env_vars.iter().rev() {
+            if let Some(v) = scope.get(name) {
+                return Some(v.clone());
+            }
+        }
+
+        if self.env_hidden.contains(name) {
+            None
+        } else {
+            engine_state.env_vars.get(name).cloned()
+        }
+    }
+
+    pub fn has_env_var(&self, engine_state: &EngineState, name: &str) -> bool {
+        for scope in self.env_vars.iter().rev() {
+            if scope.contains_key(name) {
+                return true;
+            }
+        }
+
+        if self.env_hidden.contains(name) {
+            false
+        } else {
+            engine_state.env_vars.contains_key(name)
+        }
+    }
+
+    pub fn remove_env_var(&mut self, engine_state: &EngineState, name: &str) -> Option<Value> {
+        for scope in self.env_vars.iter_mut().rev() {
+            if let Some(v) = scope.remove(name) {
+                return Some(v);
+            }
+        }
+
+        if self.env_hidden.contains(name) {
+            // the environment variable is already hidden
+            None
+        } else if let Some(val) = engine_state.env_vars.get(name) {
+            // the environment variable was found in the engine state => mark it as hidden
+            self.env_hidden.insert(name.to_string());
+            Some(val.clone())
+        } else {
+            None
+        }
+    }
+
+    pub fn get_config(&self) -> Result<Config, ShellError> {
+        let config = self.get_var(CONFIG_VARIABLE_ID, Span::new(0, 0));
+
+        match config {
+            Ok(config) => config.into_config(),
+            Err(e) => Err(e),
+        }
+    }
+
+    pub fn update_config(&mut self, name: &str, value: Value) {
+        if let Some(Value::Record { cols, vals, .. }) = self.vars.get_mut(&CONFIG_VARIABLE_ID) {
+            for col_val in cols.iter().zip(vals.iter_mut()) {
+                if col_val.0 == name {
+                    *col_val.1 = value;
+                    return;
+                }
+            }
+            cols.push(name.to_string());
+            vals.push(value);
+        }
+    }
+
+    pub fn print_stack(&self) {
+        println!("vars:");
+        for (var, val) in &self.vars {
+            println!("  {}: {:?}", var, val);
+        }
+        for (i, scope) in self.env_vars.iter().rev().enumerate() {
+            println!("env vars, scope {} (from the last);", i);
+            for (var, val) in scope {
+                println!("  {}: {:?}", var, val.clone().debug_value());
+            }
+        }
+    }
+}

crates/nu-protocol/src/example.rs

@@ -0,0 +1,8 @@
+use crate::Value;
+
+#[derive(Debug)]
+pub struct Example {
+    pub example: &'static str,
+    pub description: &'static str,
+    pub result: Option<Value>,
+}

crates/nu-protocol/src/exportable.rs

@@ -0,0 +1,6 @@
+use crate::{BlockId, DeclId};
+
+pub enum Exportable {
+    Decl(DeclId),
+    EnvVar(BlockId),
+}

crates/nu-protocol/src/id.rs

@@ -0,0 +1,4 @@
+pub type VarId = usize;
+pub type DeclId = usize;
+pub type BlockId = usize;
+pub type OverlayId = usize;

crates/nu-protocol/src/lib.rs

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 #[macro_use]
 mod macros;
 
@@ -33,3 +34,37 @@ pub use crate::value::primitive::{format_date, format_duration, format_primitive
 pub use crate::value::range::{Range, RangeInclusion};
 pub use crate::value::value_structure::{ValueResource, ValueStructure};
 pub use crate::value::{merge_descriptors, UntaggedValue, Value};
+=======
+pub mod ast;
+mod config;
+pub mod engine;
+mod example;
+mod exportable;
+mod id;
+mod overlay;
+mod pipeline_data;
+mod shell_error;
+mod signature;
+mod span;
+mod syntax_shape;
+mod ty;
+mod value;
+pub use value::Value;
+
+pub use config::*;
+pub use engine::{
+    CONFIG_VARIABLE_ID, ENV_VARIABLE_ID, IN_VARIABLE_ID, NU_VARIABLE_ID, SCOPE_VARIABLE_ID,
+};
+pub use example::*;
+pub use exportable::*;
+pub use id::*;
+pub use overlay::*;
+pub use pipeline_data::*;
+pub use shell_error::*;
+pub use signature::*;
+pub use span::*;
+pub use syntax_shape::*;
+pub use ty::*;
+pub use value::CustomValue;
+pub use value::*;
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce

crates/nu-protocol/src/macros.rs

@@ -1,49 +0,0 @@
-/// Outputs to standard out
-///
-/// Note: this exists to differentiate between intentional writing to stdout
-/// and stray printlns left by accident
-#[macro_export]
-macro_rules! out {
-    ($($tokens:tt)*) => {
-        use std::io::Write;
-        write!(std::io::stdout(), $($tokens)*).unwrap_or(());
-        let _ = std::io::stdout().flush();
-    }
-}
-
-/// Outputs to standard out with a newline added
-///
-/// Note: this exists to differentiate between intentional writing to stdout
-/// and stray printlns left by accident
-#[macro_export]
-macro_rules! outln {
-    ($($tokens:tt)*) => {
-        {
-            use std::io::Write;
-            writeln!(std::io::stdout(), $($tokens)*).unwrap_or(())
-        }
-    }
-}
-
-/// Outputs to standard error
-///
-/// Note: this exists to differentiate between intentional writing to stdout
-/// and stray printlns left by accident
-#[macro_export]
-macro_rules! errln {
-    ($($tokens:tt)*) => {
-        {
-            use std::io::Write;
-            writeln!(std::io::stderr(), $($tokens)*).unwrap_or(())
-        }
-    }
-}
-
-#[macro_export]
-macro_rules! row {
-    ($( $key: expr => $val: expr ),*) => {{
-         let mut map = ::indexmap::IndexMap::new();
-         $( map.insert($key, $val); )*
-         ::nu_protocol::UntaggedValue::row(map).into_untagged_value()
-    }}
-}

crates/nu-protocol/src/maybe_owned.rs

@@ -1,18 +0,0 @@
-#![allow(clippy::should_implement_trait)]
-
-/// Helper type to allow passing something that may potentially be owned, but could also be borrowed
-#[derive(Debug)]
-pub enum MaybeOwned<'a, T> {
-    Owned(T),
-    Borrowed(&'a T),
-}
-
-impl<T> MaybeOwned<'_, T> {
-    /// Allows the borrowing of an owned value or passes out the borrowed value
-    pub fn borrow(&self) -> &T {
-        match self {
-            MaybeOwned::Owned(v) => v,
-            MaybeOwned::Borrowed(v) => v,
-        }
-    }
-}

crates/nu-protocol/src/overlay.rs

@@ -0,0 +1,131 @@
+use crate::{BlockId, DeclId, Span};
+
+use indexmap::IndexMap;
+
+// TODO: Move the import pattern matching logic here from use/hide commands and
+// parse_use/parse_hide
+
+/// Collection of definitions that can be exported from a module
+#[derive(Debug, Clone)]
+pub struct Overlay {
+    pub decls: IndexMap<Vec<u8>, DeclId>,
+    pub env_vars: IndexMap<Vec<u8>, BlockId>,
+    pub span: Option<Span>,
+}
+
+impl Overlay {
+    pub fn new() -> Self {
+        Overlay {
+            decls: IndexMap::new(),
+            env_vars: IndexMap::new(),
+            span: None,
+        }
+    }
+
+    pub fn from_span(span: Span) -> Self {
+        Overlay {
+            decls: IndexMap::new(),
+            env_vars: IndexMap::new(),
+            span: Some(span),
+        }
+    }
+
+    pub fn add_decl(&mut self, name: &[u8], decl_id: DeclId) -> Option<DeclId> {
+        self.decls.insert(name.to_vec(), decl_id)
+    }
+
+    pub fn add_env_var(&mut self, name: &[u8], block_id: BlockId) -> Option<BlockId> {
+        self.env_vars.insert(name.to_vec(), block_id)
+    }
+
+    pub fn extend(&mut self, other: &Overlay) {
+        self.decls.extend(other.decls.clone());
+        self.env_vars.extend(other.env_vars.clone());
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.decls.is_empty() && self.env_vars.is_empty()
+    }
+
+    pub fn get_decl_id(&self, name: &[u8]) -> Option<DeclId> {
+        self.decls.get(name).copied()
+    }
+
+    pub fn has_decl(&self, name: &[u8]) -> bool {
+        self.decls.contains_key(name)
+    }
+
+    pub fn decl_with_head(&self, name: &[u8], head: &[u8]) -> Option<(Vec<u8>, DeclId)> {
+        if let Some(id) = self.get_decl_id(name) {
+            let mut new_name = head.to_vec();
+            new_name.push(b' ');
+            new_name.extend(name);
+            Some((new_name, id))
+        } else {
+            None
+        }
+    }
+
+    pub fn decls_with_head(&self, head: &[u8]) -> Vec<(Vec<u8>, DeclId)> {
+        self.decls
+            .iter()
+            .map(|(name, id)| {
+                let mut new_name = head.to_vec();
+                new_name.push(b' ');
+                new_name.extend(name);
+                (new_name, *id)
+            })
+            .collect()
+    }
+
+    pub fn decls(&self) -> Vec<(Vec<u8>, DeclId)> {
+        self.decls
+            .iter()
+            .map(|(name, id)| (name.clone(), *id))
+            .collect()
+    }
+
+    pub fn get_env_var_id(&self, name: &[u8]) -> Option<BlockId> {
+        self.env_vars.get(name).copied()
+    }
+
+    pub fn has_env_var(&self, name: &[u8]) -> bool {
+        self.env_vars.contains_key(name)
+    }
+
+    pub fn env_var_with_head(&self, name: &[u8], head: &[u8]) -> Option<(Vec<u8>, BlockId)> {
+        if let Some(id) = self.get_env_var_id(name) {
+            let mut new_name = head.to_vec();
+            new_name.push(b' ');
+            new_name.extend(name);
+            Some((new_name, id))
+        } else {
+            None
+        }
+    }
+
+    pub fn env_vars_with_head(&self, head: &[u8]) -> Vec<(Vec<u8>, BlockId)> {
+        self.env_vars
+            .iter()
+            .map(|(name, id)| {
+                let mut new_name = head.to_vec();
+                new_name.push(b' ');
+                new_name.extend(name);
+                (new_name, *id)
+            })
+            .collect()
+    }
+
+    pub fn env_vars(&self) -> Vec<(Vec<u8>, BlockId)> {
+        self.env_vars
+            .iter()
+            .map(|(name, id)| (name.clone(), *id))
+            .collect()
+    }
+}
+
+impl Default for Overlay {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/nu-protocol/src/pipeline_data.rs

@@ -0,0 +1,470 @@
+use std::sync::{atomic::AtomicBool, Arc};
+
+use crate::{ast::PathMember, Config, ListStream, RawStream, ShellError, Span, Value};
+
+/// The foundational abstraction for input and output to commands
+///
+/// This represents either a single Value or a stream of values coming into the command or leaving a command.
+///
+/// A note on implementation:
+///
+/// We've tried a few variations of this structure. Listing these below so we have a record.
+///
+/// * We tried always assuming a stream in Nushell. This was a great 80% solution, but it had some rough edges.
+/// Namely, how do you know the difference between a single string and a list of one string. How do you know
+/// when to flatten the data given to you from a data source into the stream or to keep it as an unflattened
+/// list?
+///
+/// * We tried putting the stream into Value. This had some interesting properties as now commands "just worked
+/// on values", but lead to a few unfortunate issues.
+///
+/// The first is that you can't easily clone Values in a way that felt largely immutable. For example, if
+/// you cloned a Value which contained a stream, and in one variable drained some part of it, then the second
+/// variable would see different values based on what you did to the first.
+///
+/// To make this kind of mutation thread-safe, we would have had to produce a lock for the stream, which in
+/// practice would have meant always locking the stream before reading from it. But more fundamentally, it
+/// felt wrong in practice that observation of a value at runtime could affect other values which happen to
+/// alias the same stream. By separating these, we don't have this effect. Instead, variables could get
+/// concrete list values rather than streams, and be able to view them without non-local effects.
+///
+/// * A balance of the two approaches is what we've landed on: Values are thread-safe to pass, and we can stream
+/// them into any sources. Streams are still available to model the infinite streams approach of original
+/// Nushell.
+#[derive(Debug)]
+pub enum PipelineData {
+    Value(Value, Option<PipelineMetadata>),
+    ListStream(ListStream, Option<PipelineMetadata>),
+    RawStream(RawStream, Span, Option<PipelineMetadata>),
+}
+
+#[derive(Debug, Clone)]
+pub struct PipelineMetadata {
+    pub data_source: DataSource,
+}
+
+#[derive(Debug, Clone)]
+pub enum DataSource {
+    Ls,
+}
+
+impl PipelineData {
+    pub fn new(span: Span) -> PipelineData {
+        PipelineData::Value(Value::Nothing { span }, None)
+    }
+
+    pub fn new_with_metadata(metadata: Option<PipelineMetadata>, span: Span) -> PipelineData {
+        PipelineData::Value(Value::Nothing { span }, metadata)
+    }
+
+    pub fn metadata(&self) -> Option<PipelineMetadata> {
+        match self {
+            PipelineData::ListStream(_, x) => x.clone(),
+            PipelineData::RawStream(_, _, x) => x.clone(),
+            PipelineData::Value(_, x) => x.clone(),
+        }
+    }
+
+    pub fn set_metadata(mut self, metadata: Option<PipelineMetadata>) -> Self {
+        match &mut self {
+            PipelineData::ListStream(_, x) => *x = metadata,
+            PipelineData::RawStream(_, _, x) => *x = metadata,
+            PipelineData::Value(_, x) => *x = metadata,
+        }
+
+        self
+    }
+
+    pub fn is_nothing(&self) -> bool {
+        matches!(self, PipelineData::Value(Value::Nothing { .. }, ..))
+    }
+
+    pub fn into_value(self, span: Span) -> Value {
+        match self {
+            PipelineData::Value(Value::Nothing { .. }, ..) => Value::nothing(span),
+            PipelineData::Value(v, ..) => v,
+            PipelineData::ListStream(s, ..) => Value::List {
+                vals: s.collect(),
+                span, // FIXME?
+            },
+            PipelineData::RawStream(mut s, ..) => {
+                let mut items = vec![];
+
+                for val in &mut s {
+                    match val {
+                        Ok(val) => {
+                            items.push(val);
+                        }
+                        Err(e) => {
+                            return Value::Error { error: e };
+                        }
+                    }
+                }
+
+                if s.is_binary {
+                    let mut output = vec![];
+                    for item in items {
+                        match item.as_binary() {
+                            Ok(item) => {
+                                output.extend(item);
+                            }
+                            Err(err) => {
+                                return Value::Error { error: err };
+                            }
+                        }
+                    }
+
+                    Value::Binary {
+                        val: output,
+                        span, // FIXME?
+                    }
+                } else {
+                    let mut output = String::new();
+                    for item in items {
+                        match item.as_string() {
+                            Ok(s) => output.push_str(&s),
+                            Err(err) => {
+                                return Value::Error { error: err };
+                            }
+                        }
+                    }
+                    Value::String {
+                        val: output,
+                        span, // FIXME?
+                    }
+                }
+            }
+        }
+    }
+
+    pub fn into_interruptible_iter(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineIterator {
+        let mut iter = self.into_iter();
+
+        if let PipelineIterator(PipelineData::ListStream(s, ..)) = &mut iter {
+            s.ctrlc = ctrlc;
+        }
+
+        iter
+    }
+
+    pub fn collect_string(self, separator: &str, config: &Config) -> Result<String, ShellError> {
+        match self {
+            PipelineData::Value(v, ..) => Ok(v.into_string(separator, config)),
+            PipelineData::ListStream(s, ..) => Ok(s.into_string(separator, config)),
+            PipelineData::RawStream(s, ..) => {
+                let mut items = vec![];
+
+                for val in s {
+                    match val {
+                        Ok(val) => {
+                            items.push(val);
+                        }
+                        Err(e) => {
+                            return Err(e);
+                        }
+                    }
+                }
+
+                let mut output = String::new();
+                for item in items {
+                    match item.as_string() {
+                        Ok(s) => output.push_str(&s),
+                        Err(err) => {
+                            return Err(err);
+                        }
+                    }
+                }
+
+                Ok(output)
+            }
+        }
+    }
+
+    pub fn follow_cell_path(
+        self,
+        cell_path: &[PathMember],
+        head: Span,
+    ) -> Result<Value, ShellError> {
+        match self {
+            // FIXME: there are probably better ways of doing this
+            PipelineData::ListStream(stream, ..) => Value::List {
+                vals: stream.collect(),
+                span: head,
+            }
+            .follow_cell_path(cell_path),
+            PipelineData::Value(v, ..) => v.follow_cell_path(cell_path),
+            _ => Err(ShellError::IOError("can't follow stream paths".into())),
+        }
+    }
+
+    pub fn update_cell_path(
+        &mut self,
+        cell_path: &[PathMember],
+        callback: Box<dyn FnOnce(&Value) -> Value>,
+        head: Span,
+    ) -> Result<(), ShellError> {
+        match self {
+            // FIXME: there are probably better ways of doing this
+            PipelineData::ListStream(stream, ..) => Value::List {
+                vals: stream.collect(),
+                span: head,
+            }
+            .update_cell_path(cell_path, callback),
+            PipelineData::Value(v, ..) => v.update_cell_path(cell_path, callback),
+            _ => Ok(()),
+        }
+    }
+
+    /// Simplified mapper to help with simple values also. For full iterator support use `.into_iter()` instead
+    pub fn map<F>(
+        self,
+        mut f: F,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> Result<PipelineData, ShellError>
+    where
+        Self: Sized,
+        F: FnMut(Value) -> Value + 'static + Send,
+    {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }, ..) => {
+                Ok(vals.into_iter().map(f).into_pipeline_data(ctrlc))
+            }
+            PipelineData::ListStream(stream, ..) => Ok(stream.map(f).into_pipeline_data(ctrlc)),
+            PipelineData::RawStream(stream, ..) => {
+                let collected = stream.into_bytes()?;
+
+                if let Ok(st) = String::from_utf8(collected.clone().item) {
+                    Ok(f(Value::String {
+                        val: st,
+                        span: collected.span,
+                    })
+                    .into_pipeline_data())
+                } else {
+                    Ok(f(Value::Binary {
+                        val: collected.item,
+                        span: collected.span,
+                    })
+                    .into_pipeline_data())
+                }
+            }
+
+            PipelineData::Value(Value::Range { val, .. }, ..) => {
+                Ok(val.into_range_iter()?.map(f).into_pipeline_data(ctrlc))
+            }
+            PipelineData::Value(v, ..) => match f(v) {
+                Value::Error { error } => Err(error),
+                v => Ok(v.into_pipeline_data()),
+            },
+        }
+    }
+
+    /// Simplified flatmapper. For full iterator support use `.into_iter()` instead
+    pub fn flat_map<U, F>(
+        self,
+        mut f: F,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> Result<PipelineData, ShellError>
+    where
+        Self: Sized,
+        U: IntoIterator<Item = Value>,
+        <U as IntoIterator>::IntoIter: 'static + Send,
+        F: FnMut(Value) -> U + 'static + Send,
+    {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }, ..) => {
+                Ok(vals.into_iter().map(f).flatten().into_pipeline_data(ctrlc))
+            }
+            PipelineData::ListStream(stream, ..) => {
+                Ok(stream.map(f).flatten().into_pipeline_data(ctrlc))
+            }
+            PipelineData::RawStream(stream, ..) => {
+                let collected = stream.into_bytes()?;
+
+                if let Ok(st) = String::from_utf8(collected.clone().item) {
+                    Ok(f(Value::String {
+                        val: st,
+                        span: collected.span,
+                    })
+                    .into_iter()
+                    .into_pipeline_data(ctrlc))
+                } else {
+                    Ok(f(Value::Binary {
+                        val: collected.item,
+                        span: collected.span,
+                    })
+                    .into_iter()
+                    .into_pipeline_data(ctrlc))
+                }
+            }
+            PipelineData::Value(Value::Range { val, .. }, ..) => match val.into_range_iter() {
+                Ok(iter) => Ok(iter.map(f).flatten().into_pipeline_data(ctrlc)),
+                Err(error) => Err(error),
+            },
+            PipelineData::Value(v, ..) => Ok(f(v).into_iter().into_pipeline_data(ctrlc)),
+        }
+    }
+
+    pub fn filter<F>(
+        self,
+        mut f: F,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> Result<PipelineData, ShellError>
+    where
+        Self: Sized,
+        F: FnMut(&Value) -> bool + 'static + Send,
+    {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }, ..) => {
+                Ok(vals.into_iter().filter(f).into_pipeline_data(ctrlc))
+            }
+            PipelineData::ListStream(stream, ..) => Ok(stream.filter(f).into_pipeline_data(ctrlc)),
+            PipelineData::RawStream(stream, ..) => {
+                let collected = stream.into_bytes()?;
+
+                if let Ok(st) = String::from_utf8(collected.clone().item) {
+                    let v = Value::String {
+                        val: st,
+                        span: collected.span,
+                    };
+
+                    if f(&v) {
+                        Ok(v.into_pipeline_data())
+                    } else {
+                        Ok(PipelineData::new(collected.span))
+                    }
+                } else {
+                    let v = Value::Binary {
+                        val: collected.item,
+                        span: collected.span,
+                    };
+
+                    if f(&v) {
+                        Ok(v.into_pipeline_data())
+                    } else {
+                        Ok(PipelineData::new(collected.span))
+                    }
+                }
+            }
+            PipelineData::Value(Value::Range { val, .. }, ..) => {
+                Ok(val.into_range_iter()?.filter(f).into_pipeline_data(ctrlc))
+            }
+            PipelineData::Value(v, ..) => {
+                if f(&v) {
+                    Ok(v.into_pipeline_data())
+                } else {
+                    Ok(Value::Nothing { span: v.span()? }.into_pipeline_data())
+                }
+            }
+        }
+    }
+}
+
+pub struct PipelineIterator(PipelineData);
+
+impl IntoIterator for PipelineData {
+    type Item = Value;
+
+    type IntoIter = PipelineIterator;
+
+    fn into_iter(self) -> Self::IntoIter {
+        match self {
+            PipelineData::Value(Value::List { vals, .. }, metadata) => {
+                PipelineIterator(PipelineData::ListStream(
+                    ListStream {
+                        stream: Box::new(vals.into_iter()),
+                        ctrlc: None,
+                    },
+                    metadata,
+                ))
+            }
+            PipelineData::Value(Value::Range { val, .. }, metadata) => {
+                match val.into_range_iter() {
+                    Ok(iter) => PipelineIterator(PipelineData::ListStream(
+                        ListStream {
+                            stream: Box::new(iter),
+                            ctrlc: None,
+                        },
+                        metadata,
+                    )),
+                    Err(error) => PipelineIterator(PipelineData::ListStream(
+                        ListStream {
+                            stream: Box::new(std::iter::once(Value::Error { error })),
+                            ctrlc: None,
+                        },
+                        metadata,
+                    )),
+                }
+            }
+            x => PipelineIterator(x),
+        }
+    }
+}
+
+impl Iterator for PipelineIterator {
+    type Item = Value;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match &mut self.0 {
+            PipelineData::Value(Value::Nothing { .. }, ..) => None,
+            PipelineData::Value(v, ..) => Some(std::mem::take(v)),
+            PipelineData::ListStream(stream, ..) => stream.next(),
+            PipelineData::RawStream(stream, ..) => stream.next().map(|x| match x {
+                Ok(x) => x,
+                Err(err) => Value::Error { error: err },
+            }),
+        }
+    }
+}
+
+pub trait IntoPipelineData {
+    fn into_pipeline_data(self) -> PipelineData;
+}
+
+impl<V> IntoPipelineData for V
+where
+    V: Into<Value>,
+{
+    fn into_pipeline_data(self) -> PipelineData {
+        PipelineData::Value(self.into(), None)
+    }
+}
+
+pub trait IntoInterruptiblePipelineData {
+    fn into_pipeline_data(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineData;
+    fn into_pipeline_data_with_metadata(
+        self,
+        metadata: PipelineMetadata,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> PipelineData;
+}
+
+impl<I> IntoInterruptiblePipelineData for I
+where
+    I: IntoIterator + Send + 'static,
+    I::IntoIter: Send + 'static,
+    <I::IntoIter as Iterator>::Item: Into<Value>,
+{
+    fn into_pipeline_data(self, ctrlc: Option<Arc<AtomicBool>>) -> PipelineData {
+        PipelineData::ListStream(
+            ListStream {
+                stream: Box::new(self.into_iter().map(Into::into)),
+                ctrlc,
+            },
+            None,
+        )
+    }
+
+    fn into_pipeline_data_with_metadata(
+        self,
+        metadata: PipelineMetadata,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> PipelineData {
+        PipelineData::ListStream(
+            ListStream {
+                stream: Box::new(self.into_iter().map(Into::into)),
+                ctrlc,
+            },
+            Some(metadata),
+        )
+    }
+}

crates/nu-protocol/src/registry.rs

@@ -1,31 +0,0 @@
-use crate::{Signature, Value};
-use nu_source::Spanned;
-use std::fmt::Debug;
-
-pub trait VariableRegistry {
-    fn get_variable(&self, name: &Spanned<&str>) -> Option<Value>;
-    fn variables(&self) -> Vec<String>;
-}
-
-pub trait SignatureRegistry: Debug {
-    fn names(&self) -> Vec<String>;
-    fn has(&self, name: &str) -> bool;
-    fn get(&self, name: &str) -> Option<Signature>;
-    fn clone_box(&self) -> Box<dyn SignatureRegistry>;
-}
-
-impl SignatureRegistry for Box<dyn SignatureRegistry> {
-    fn names(&self) -> Vec<String> {
-        (&**self).names()
-    }
-
-    fn has(&self, name: &str) -> bool {
-        (&**self).has(name)
-    }
-    fn get(&self, name: &str) -> Option<Signature> {
-        (&**self).get(name)
-    }
-    fn clone_box(&self) -> Box<dyn SignatureRegistry> {
-        (&**self).clone_box()
-    }
-}

crates/nu-protocol/src/return_value.rs

@@ -1,140 +0,0 @@
-use crate::{value::Value, ConfigPath};
-use nu_errors::ShellError;
-use nu_source::{DbgDocBldr, DebugDocBuilder, PrettyDebug};
-use serde::{Deserialize, Serialize};
-
-/// The inner set of actions for the command processor. Each denotes a way to change state in the processor without changing it directly from the command itself.
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum CommandAction {
-    /// Change to a new directory or path (in non-filesystem situations)
-    ChangePath(String),
-    /// Exit out of Nu
-    Exit(i32),
-    /// Display an error
-    Error(ShellError),
-    /// Enter a new shell at the given path
-    EnterShell(String),
-    /// Convert the value given from one type to another
-    AutoConvert(Value, String),
-    /// Enter a value shell, one that allows exploring inside of a Value
-    EnterValueShell(Value),
-    /// Add plugins from path given
-    AddPlugins(String),
-    /// Unload the config specified by PathBuf if present
-    UnloadConfig(ConfigPath),
-    /// Load the config specified by PathBuf
-    LoadConfig(ConfigPath),
-    /// Go to the previous shell in the shell ring buffer
-    PreviousShell,
-    /// Go to the next shell in the shell ring buffer
-    NextShell,
-    /// Jump to the specified shell in the shell ring buffer
-    GotoShell(usize),
-    /// Leave the current shell. If it's the last shell, exit out of Nu
-    LeaveShell(i32),
-}
-
-impl PrettyDebug for CommandAction {
-    /// Get a command action ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            CommandAction::ChangePath(path) => {
-                DbgDocBldr::typed("change path", DbgDocBldr::description(path))
-            }
-            CommandAction::Exit(_) => DbgDocBldr::description("exit"),
-            CommandAction::Error(_) => DbgDocBldr::error("error"),
-            CommandAction::AutoConvert(_, extension) => {
-                DbgDocBldr::typed("auto convert", DbgDocBldr::description(extension))
-            }
-            CommandAction::EnterShell(s) => {
-                DbgDocBldr::typed("enter shell", DbgDocBldr::description(s))
-            }
-            CommandAction::EnterValueShell(v) => DbgDocBldr::typed("enter value shell", v.pretty()),
-            CommandAction::AddPlugins(..) => DbgDocBldr::description("add plugins"),
-            CommandAction::PreviousShell => DbgDocBldr::description("previous shell"),
-            CommandAction::NextShell => DbgDocBldr::description("next shell"),
-            CommandAction::GotoShell(_) => DbgDocBldr::description("goto shell"),
-            CommandAction::LeaveShell(_) => DbgDocBldr::description("leave shell"),
-            CommandAction::UnloadConfig(cfg) => {
-                DbgDocBldr::description(format!("unload config {:?}", cfg))
-            }
-            CommandAction::LoadConfig(cfg) => {
-                DbgDocBldr::description(format!("load config {:?}", cfg))
-            }
-        }
-    }
-}
-
-/// The fundamental success type in the pipeline. Commands return these values as their main responsibility
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum ReturnSuccess {
-    /// A value to be used or shown to the user
-    Value(Value),
-    /// A debug-enabled value to be used or shown to the user
-    DebugValue(Value),
-    /// An action to be performed as values pass out of the command. These are performed rather than passed to the next command in the pipeline
-    Action(CommandAction),
-}
-
-impl PrettyDebug for ReturnSuccess {
-    /// Get a return success ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            ReturnSuccess::Value(value) => DbgDocBldr::typed("value", value.pretty()),
-            ReturnSuccess::DebugValue(value) => DbgDocBldr::typed("debug value", value.pretty()),
-            ReturnSuccess::Action(action) => DbgDocBldr::typed("action", action.pretty()),
-        }
-    }
-}
-
-/// The core Result type for pipelines
-pub type ReturnValue = Result<ReturnSuccess, ShellError>;
-
-impl From<Value> for ReturnValue {
-    fn from(v: Value) -> Self {
-        Ok(ReturnSuccess::Value(v))
-    }
-}
-
-impl ReturnSuccess {
-    /// Get to the contained Value, if possible
-    pub fn raw_value(&self) -> Option<Value> {
-        match self {
-            ReturnSuccess::Value(raw) => Some(raw.clone()),
-            ReturnSuccess::DebugValue(raw) => Some(raw.clone()),
-            ReturnSuccess::Action(_) => None,
-        }
-    }
-
-    /// Helper function for an action to change the the path
-    pub fn change_cwd(path: String) -> ReturnValue {
-        Ok(ReturnSuccess::Action(CommandAction::ChangePath(path)))
-    }
-
-    /// Helper function to create simple values for returning
-    pub fn value(input: impl Into<Value>) -> ReturnValue {
-        Ok(ReturnSuccess::Value(input.into()))
-    }
-
-    /// Helper function to create simple debug-enabled values for returning
-    pub fn debug_value(input: impl Into<Value>) -> ReturnValue {
-        Ok(ReturnSuccess::DebugValue(input.into()))
-    }
-
-    /// Helper function for creating actions
-    pub fn action(input: CommandAction) -> ReturnValue {
-        Ok(ReturnSuccess::Action(input))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::{ReturnSuccess, ReturnValue, UntaggedValue};
-
-    #[test]
-    fn return_value_can_be_used_in_assert_eq() {
-        let v1: ReturnValue = ReturnSuccess::value(UntaggedValue::nothing());
-        let v2: ReturnValue = ReturnSuccess::value(UntaggedValue::nothing());
-        assert_eq!(v1, v2);
-    }
-}

crates/nu-protocol/src/shell_error.rs

@@ -0,0 +1,367 @@
+use miette::Diagnostic;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::{ast::Operator, Span, Type};
+
+/// The fundamental error type for the evaluation engine. These cases represent different kinds of errors
+/// the evaluator might face, along with helpful spans to label. An error renderer will take this error value
+/// and pass it into an error viewer to display to the user.
+#[derive(Debug, Clone, Error, Diagnostic, Serialize, Deserialize)]
+pub enum ShellError {
+    #[error("Type mismatch during operation.")]
+    #[diagnostic(code(nu::shell::type_mismatch), url(docsrs))]
+    OperatorMismatch {
+        #[label = "type mismatch for operator"]
+        op_span: Span,
+        lhs_ty: Type,
+        #[label("{lhs_ty}")]
+        lhs_span: Span,
+        rhs_ty: Type,
+        #[label("{rhs_ty}")]
+        rhs_span: Span,
+    },
+
+    #[error("Operator overflow.")]
+    #[diagnostic(code(nu::shell::operator_overflow), url(docsrs))]
+    OperatorOverflow(String, #[label = "{0}"] Span),
+
+    #[error("Pipeline mismatch.")]
+    #[diagnostic(code(nu::shell::pipeline_mismatch), url(docsrs))]
+    PipelineMismatch(
+        String,
+        #[label("expected: {0}")] Span,
+        #[label("value originates from here")] Span,
+    ),
+
+    #[error("Type mismatch")]
+    #[diagnostic(code(nu::shell::type_mismatch), url(docsrs))]
+    TypeMismatch(String, #[label = "{0}"] Span),
+
+    #[error("Unsupported operator: {0}.")]
+    #[diagnostic(code(nu::shell::unsupported_operator), url(docsrs))]
+    UnsupportedOperator(Operator, #[label = "unsupported operator"] Span),
+
+    #[error("Unsupported operator: {0}.")]
+    #[diagnostic(code(nu::shell::unknown_operator), url(docsrs))]
+    UnknownOperator(String, #[label = "unsupported operator"] Span),
+
+    #[error("Missing parameter: {0}.")]
+    #[diagnostic(code(nu::shell::missing_parameter), url(docsrs))]
+    MissingParameter(String, #[label = "missing parameter: {0}"] Span),
+
+    // Be cautious, as flags can share the same span, resulting in a panic (ex: `rm -pt`)
+    #[error("Incompatible parameters.")]
+    #[diagnostic(code(nu::shell::incompatible_parameters), url(docsrs))]
+    IncompatibleParameters {
+        left_message: String,
+        #[label("{left_message}")]
+        left_span: Span,
+        right_message: String,
+        #[label("{right_message}")]
+        right_span: Span,
+    },
+
+    #[error("Delimiter error")]
+    #[diagnostic(code(nu::shell::delimiter_error), url(docsrs))]
+    DelimiterError(String, #[label("{0}")] Span),
+
+    #[error("Incompatible parameters.")]
+    #[diagnostic(code(nu::shell::incompatible_parameters), url(docsrs))]
+    IncompatibleParametersSingle(String, #[label = "{0}"] Span),
+
+    #[error("Feature not enabled.")]
+    #[diagnostic(code(nu::shell::feature_not_enabled), url(docsrs))]
+    FeatureNotEnabled(#[label = "feature not enabled"] Span),
+
+    #[error("External commands not yet supported")]
+    #[diagnostic(code(nu::shell::external_commands), url(docsrs))]
+    ExternalNotSupported(#[label = "external not supported"] Span),
+
+    #[error("Invalid Probability.")]
+    #[diagnostic(code(nu::shell::invalid_probability), url(docsrs))]
+    InvalidProbability(#[label = "invalid probability"] Span),
+
+    #[error("Invalid range {0}..{1}")]
+    #[diagnostic(code(nu::shell::invalid_range), url(docsrs))]
+    InvalidRange(String, String, #[label = "expected a valid range"] Span),
+
+    // Only use this one if we Nushell completely falls over and hits a state that isn't possible or isn't recoverable
+    #[error("Nushell failed: {0}.")]
+    #[diagnostic(code(nu::shell::nushell_failed), url(docsrs))]
+    NushellFailed(String),
+
+    // Only use this one if we Nushell completely falls over and hits a state that isn't possible or isn't recoverable
+    #[error("Nushell failed: {0}.")]
+    #[diagnostic(code(nu::shell::nushell_failed), url(docsrs))]
+    NushellFailedSpanned(String, String, #[label = "{1}"] Span),
+
+    #[error("Variable not found")]
+    #[diagnostic(code(nu::shell::variable_not_found), url(docsrs))]
+    VariableNotFoundAtRuntime(#[label = "variable not found"] Span),
+
+    #[error("Environment variable '{0}' not found")]
+    #[diagnostic(code(nu::shell::env_variable_not_found), url(docsrs))]
+    EnvVarNotFoundAtRuntime(String, #[label = "environment variable not found"] Span),
+
+    #[error("Not found.")]
+    #[diagnostic(code(nu::parser::not_found), url(docsrs))]
+    NotFound(#[label = "did not find anything under this name"] Span),
+
+    #[error("Can't convert to {0}.")]
+    #[diagnostic(code(nu::shell::cant_convert), url(docsrs))]
+    CantConvert(String, String, #[label("can't convert {1} to {0}")] Span),
+
+    #[error("Division by zero.")]
+    #[diagnostic(code(nu::shell::division_by_zero), url(docsrs))]
+    DivisionByZero(#[label("division by zero")] Span),
+
+    #[error("Can't convert range to countable values")]
+    #[diagnostic(code(nu::shell::range_to_countable), url(docsrs))]
+    CannotCreateRange(#[label = "can't convert to countable values"] Span),
+
+    #[error("Row number too large (max: {0}).")]
+    #[diagnostic(code(nu::shell::access_beyond_end), url(docsrs))]
+    AccessBeyondEnd(usize, #[label = "too large"] Span),
+
+    #[error("Row number too large.")]
+    #[diagnostic(code(nu::shell::access_beyond_end_of_stream), url(docsrs))]
+    AccessBeyondEndOfStream(#[label = "too large"] Span),
+
+    #[error("Data cannot be accessed with a cell path")]
+    #[diagnostic(code(nu::shell::incompatible_path_access), url(docsrs))]
+    IncompatiblePathAccess(String, #[label("{0} doesn't support cell paths")] Span),
+
+    #[error("Cannot find column")]
+    #[diagnostic(code(nu::shell::column_not_found), url(docsrs))]
+    CantFindColumn(
+        #[label = "cannot find column"] Span,
+        #[label = "value originates here"] Span,
+    ),
+
+    #[error("Not a list value")]
+    #[diagnostic(code(nu::shell::not_a_list), url(docsrs))]
+    NotAList(
+        #[label = "value not a list"] Span,
+        #[label = "value originates here"] Span,
+    ),
+
+    #[error("External command")]
+    #[diagnostic(code(nu::shell::external_command), url(docsrs), help("{1}"))]
+    ExternalCommand(String, String, #[label("{0}")] Span),
+
+    #[error("Unsupported input")]
+    #[diagnostic(code(nu::shell::unsupported_input), url(docsrs))]
+    UnsupportedInput(String, #[label("{0}")] Span),
+
+    #[error("Network failure")]
+    #[diagnostic(code(nu::shell::network_failure), url(docsrs))]
+    NetworkFailure(String, #[label("{0}")] Span),
+
+    #[error("Command not found")]
+    #[diagnostic(code(nu::shell::command_not_found), url(docsrs))]
+    CommandNotFound(#[label("command not found")] Span),
+
+    #[error("Flag not found")]
+    #[diagnostic(code(nu::shell::flag_not_found), url(docsrs))]
+    FlagNotFound(String, #[label("{0} not found")] Span),
+
+    #[error("File not found")]
+    #[diagnostic(code(nu::shell::file_not_found), url(docsrs))]
+    FileNotFound(#[label("file not found")] Span),
+
+    #[error("File not found")]
+    #[diagnostic(code(nu::shell::file_not_found), url(docsrs))]
+    FileNotFoundCustom(String, #[label("{0}")] Span),
+
+    #[error("Plugin failed to load: {0}")]
+    #[diagnostic(code(nu::shell::plugin_failed_to_load), url(docsrs))]
+    PluginFailedToLoad(String),
+
+    #[error("Plugin failed to encode: {0}")]
+    #[diagnostic(code(nu::shell::plugin_failed_to_encode), url(docsrs))]
+    PluginFailedToEncode(String),
+
+    #[error("Plugin failed to decode: {0}")]
+    #[diagnostic(code(nu::shell::plugin_failed_to_decode), url(docsrs))]
+    PluginFailedToDecode(String),
+
+    #[error("I/O error")]
+    #[diagnostic(code(nu::shell::io_error), url(docsrs), help("{0}"))]
+    IOError(String),
+
+    #[error("Cannot change to directory")]
+    #[diagnostic(code(nu::shell::cannot_cd_to_directory), url(docsrs))]
+    NotADirectory(#[label("is not a directory")] Span),
+
+    #[error("Directory not found")]
+    #[diagnostic(code(nu::shell::directory_not_found), url(docsrs))]
+    DirectoryNotFound(#[label("directory not found")] Span),
+
+    #[error("Directory not found")]
+    #[diagnostic(code(nu::shell::directory_not_found_custom), url(docsrs))]
+    DirectoryNotFoundCustom(String, #[label("{0}")] Span),
+
+    #[error("Directory not found")]
+    #[diagnostic(code(nu::shell::directory_not_found_help), url(docsrs), help("{1}"))]
+    DirectoryNotFoundHelp(#[label("directory not found")] Span, String),
+
+    #[error("Move not possible")]
+    #[diagnostic(code(nu::shell::move_not_possible), url(docsrs))]
+    MoveNotPossible {
+        source_message: String,
+        #[label("{source_message}")]
+        source_span: Span,
+        destination_message: String,
+        #[label("{destination_message}")]
+        destination_span: Span,
+    },
+
+    #[error("Move not possible")]
+    #[diagnostic(code(nu::shell::move_not_possible_single), url(docsrs))]
+    MoveNotPossibleSingle(String, #[label("{0}")] Span),
+
+    #[error("Create not possible")]
+    #[diagnostic(code(nu::shell::create_not_possible), url(docsrs))]
+    CreateNotPossible(String, #[label("{0}")] Span),
+
+    #[error("Remove not possible")]
+    #[diagnostic(code(nu::shell::remove_not_possible), url(docsrs))]
+    RemoveNotPossible(String, #[label("{0}")] Span),
+
+    #[error("No file to be removed")]
+    NoFileToBeRemoved(),
+    #[error("No file to be moved")]
+    NoFileToBeMoved(),
+    #[error("No file to be copied")]
+    NoFileToBeCopied(),
+
+    #[error("Name not found")]
+    #[diagnostic(code(nu::shell::name_not_found), url(docsrs))]
+    DidYouMean(String, #[label("did you mean '{0}'?")] Span),
+
+    #[error("Non-UTF8 string")]
+    #[diagnostic(code(nu::parser::non_utf8), url(docsrs))]
+    NonUtf8(#[label = "non-UTF8 string"] Span),
+
+    #[error("Casting error")]
+    #[diagnostic(code(nu::shell::downcast_not_possible), url(docsrs))]
+    DowncastNotPossible(String, #[label("{0}")] Span),
+
+    #[error("Unsupported config value")]
+    #[diagnostic(code(nu::shell::unsupported_config_value), url(docsrs))]
+    UnsupportedConfigValue(String, String, #[label = "expected {0}, got {1}"] Span),
+
+    #[error("Missing config value")]
+    #[diagnostic(code(nu::shell::missing_config_value), url(docsrs))]
+    MissingConfigValue(String, #[label = "missing {0}"] Span),
+
+    #[error("{0}")]
+    #[diagnostic()]
+    SpannedLabeledError(String, String, #[label("{1}")] Span),
+
+    #[error("{0}")]
+    #[diagnostic(help("{3}"))]
+    SpannedLabeledErrorHelp(String, String, #[label("{1}")] Span, String),
+
+    #[error("{0}")]
+    #[diagnostic(help("{1}"))]
+    LabeledError(String, String),
+}
+
+impl From<std::io::Error> for ShellError {
+    fn from(input: std::io::Error) -> ShellError {
+        ShellError::IOError(format!("{:?}", input))
+    }
+}
+
+impl std::convert::From<Box<dyn std::error::Error>> for ShellError {
+    fn from(input: Box<dyn std::error::Error>) -> ShellError {
+        ShellError::IOError(input.to_string())
+    }
+}
+
+impl From<Box<dyn std::error::Error + Send + Sync>> for ShellError {
+    fn from(input: Box<dyn std::error::Error + Send + Sync>) -> ShellError {
+        ShellError::IOError(format!("{:?}", input))
+    }
+}
+
+pub fn did_you_mean(possibilities: &[String], tried: &str) -> Option<String> {
+    let mut possible_matches: Vec<_> = possibilities
+        .iter()
+        .map(|word| {
+            let edit_distance = levenshtein_distance(word, tried);
+            (edit_distance, word.to_owned())
+        })
+        .collect();
+
+    possible_matches.sort();
+
+    if let Some((_, first)) = possible_matches.into_iter().next() {
+        Some(first)
+    } else {
+        None
+    }
+}
+
+// Borrowed from here https://github.com/wooorm/levenshtein-rs
+pub fn levenshtein_distance(a: &str, b: &str) -> usize {
+    let mut result = 0;
+
+    /* Shortcut optimizations / degenerate cases. */
+    if a == b {
+        return result;
+    }
+
+    let length_a = a.chars().count();
+    let length_b = b.chars().count();
+
+    if length_a == 0 {
+        return length_b;
+    }
+
+    if length_b == 0 {
+        return length_a;
+    }
+
+    /* Initialize the vector.
+     *
+     * This is why it’s fast, normally a matrix is used,
+     * here we use a single vector. */
+    let mut cache: Vec<usize> = (1..).take(length_a).collect();
+    let mut distance_a;
+    let mut distance_b;
+
+    /* Loop. */
+    for (index_b, code_b) in b.chars().enumerate() {
+        result = index_b;
+        distance_a = index_b;
+
+        for (index_a, code_a) in a.chars().enumerate() {
+            distance_b = if code_a == code_b {
+                distance_a
+            } else {
+                distance_a + 1
+            };
+
+            distance_a = cache[index_a];
+
+            result = if distance_a > result {
+                if distance_b > result {
+                    result + 1
+                } else {
+                    distance_b
+                }
+            } else if distance_b > distance_a {
+                distance_a + 1
+            } else {
+                distance_b
+            };
+
+            cache[index_a] = result;
+        }
+    }
+
+    result
+}

crates/nu-protocol/src/signature.rs

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 use crate::syntax_shape::SyntaxShape;
 use crate::type_shape::Type;
 use indexmap::IndexMap;
@@ -156,13 +157,118 @@ pub struct Signature {
     pub input: Option<Type>,
     /// If the command is expected to filter data, or to consume it (as a sink)
     pub is_filter: bool,
+=======
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::ast::Call;
+use crate::engine::Command;
+use crate::engine::EngineState;
+use crate::engine::Stack;
+use crate::BlockId;
+use crate::PipelineData;
+use crate::SyntaxShape;
+use crate::VarId;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Flag {
+    pub long: String,
+    pub short: Option<char>,
+    pub arg: Option<SyntaxShape>,
+    pub required: bool,
+    pub desc: String,
+    // For custom commands
+    pub var_id: Option<VarId>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PositionalArg {
+    pub name: String,
+    pub desc: String,
+    pub shape: SyntaxShape,
+    // For custom commands
+    pub var_id: Option<VarId>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum Category {
+    Default,
+    Conversions,
+    Core,
+    Date,
+    Env,
+    Experimental,
+    FileSystem,
+    Filters,
+    Formats,
+    Math,
+    Network,
+    Random,
+    Platform,
+    Shells,
+    Strings,
+    System,
+    Viewers,
+    Hash,
+    Generators,
+    Custom(String),
+}
+
+impl std::fmt::Display for Category {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let msg = match self {
+            Category::Default => "default",
+            Category::Conversions => "conversions",
+            Category::Core => "core",
+            Category::Date => "date",
+            Category::Env => "env",
+            Category::Experimental => "experimental",
+            Category::FileSystem => "filesystem",
+            Category::Filters => "filters",
+            Category::Formats => "formats",
+            Category::Math => "math",
+            Category::Network => "network",
+            Category::Random => "random",
+            Category::Platform => "platform",
+            Category::Shells => "shells",
+            Category::Strings => "strings",
+            Category::System => "system",
+            Category::Viewers => "viewers",
+            Category::Hash => "hash",
+            Category::Generators => "generators",
+            Category::Custom(name) => name,
+        };
+
+        write!(f, "{}", msg)
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Signature {
+    pub name: String,
+    pub usage: String,
+    pub extra_usage: String,
+    pub required_positional: Vec<PositionalArg>,
+    pub optional_positional: Vec<PositionalArg>,
+    pub rest_positional: Option<PositionalArg>,
+    pub named: Vec<Flag>,
+    pub is_filter: bool,
+    pub creates_scope: bool,
+    // Signature category used to classify commands stored in the list of declarations
+    pub category: Category,
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
 }
 
 impl PartialEq for Signature {
     fn eq(&self, other: &Self) -> bool {
         self.name == other.name
             && self.usage == other.usage
+<<<<<<< HEAD
             && self.positional == other.positional
+=======
+            && self.required_positional == other.required_positional
+            && self.optional_positional == other.optional_positional
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
             && self.rest_positional == other.rest_positional
             && self.is_filter == other.is_filter
     }
@@ -171,6 +277,7 @@ impl PartialEq for Signature {
 impl Eq for Signature {}
 
 impl Signature {
+<<<<<<< HEAD
     pub fn shift_positional(&mut self) {
         self.positional = Vec::from(&self.positional[1..]);
     }
@@ -224,10 +331,24 @@ impl PrettyDebugWithSource for Signature {
 impl Signature {
     /// Create a new command signature with the given name
     pub fn new(name: impl Into<String>) -> Signature {
+=======
+    pub fn new(name: impl Into<String>) -> Signature {
+        // default help flag
+        let flag = Flag {
+            long: "help".into(),
+            short: Some('h'),
+            arg: None,
+            desc: "Display this help message".into(),
+            required: false,
+            var_id: None,
+        };
+
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
         Signature {
             name: name.into(),
             usage: String::new(),
             extra_usage: String::new(),
+<<<<<<< HEAD
             positional: vec![],
             rest_positional: None,
             named: indexmap::indexmap! {"help".into() => (NamedType::Switch(Some('h')), "Display this help message".into())},
@@ -238,6 +359,17 @@ impl Signature {
     }
 
     /// Create a new signature
+=======
+            required_positional: vec![],
+            optional_positional: vec![],
+            rest_positional: None,
+            named: vec![flag],
+            is_filter: false,
+            creates_scope: false,
+            category: Category::Default,
+        }
+    }
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
     pub fn build(name: impl Into<String>) -> Signature {
         Signature::new(name.into())
     }
@@ -252,6 +384,7 @@ impl Signature {
     pub fn required(
         mut self,
         name: impl Into<String>,
+<<<<<<< HEAD
         ty: impl Into<SyntaxShape>,
         desc: impl Into<String>,
     ) -> Signature {
@@ -259,10 +392,22 @@ impl Signature {
             PositionalType::Mandatory(name.into(), ty.into()),
             desc.into(),
         ));
+=======
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.required_positional.push(PositionalArg {
+            name: name.into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
 
         self
     }
 
+<<<<<<< HEAD
     /// Add an optional positional argument to the signature
     pub fn optional(
         mut self,
@@ -274,6 +419,37 @@ impl Signature {
             PositionalType::Optional(name.into(), ty.into()),
             desc.into(),
         ));
+=======
+    /// Add a required positional argument to the signature
+    pub fn optional(
+        mut self,
+        name: impl Into<String>,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.optional_positional.push(PositionalArg {
+            name: name.into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    pub fn rest(
+        mut self,
+        name: &str,
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+    ) -> Signature {
+        self.rest_positional = Some(PositionalArg {
+            name: name.into(),
+            desc: desc.into(),
+            shape: shape.into(),
+            var_id: None,
+        });
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
 
         self
     }
@@ -282,6 +458,7 @@ impl Signature {
     pub fn named(
         mut self,
         name: impl Into<String>,
+<<<<<<< HEAD
         ty: impl Into<SyntaxShape>,
         desc: impl Into<String>,
         short: Option<char>,
@@ -294,6 +471,22 @@ impl Signature {
             name.into(),
             (NamedType::Optional(s, ty.into()), desc.into()),
         );
+=======
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+        short: Option<char>,
+    ) -> Signature {
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name,
+            short: s,
+            arg: Some(shape.into()),
+            required: false,
+            desc: desc.into(),
+            var_id: None,
+        });
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
 
         self
     }
@@ -302,6 +495,7 @@ impl Signature {
     pub fn required_named(
         mut self,
         name: impl Into<String>,
+<<<<<<< HEAD
         ty: impl Into<SyntaxShape>,
         desc: impl Into<String>,
         short: Option<char>,
@@ -316,6 +510,23 @@ impl Signature {
             (NamedType::Mandatory(s, ty.into()), desc.into()),
         );
 
+=======
+        shape: impl Into<SyntaxShape>,
+        desc: impl Into<String>,
+        short: Option<char>,
+    ) -> Signature {
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name,
+            short: s,
+            arg: Some(shape.into()),
+            required: true,
+            desc: desc.into(),
+            var_id: None,
+        });
+
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
         self
     }
 
@@ -326,6 +537,80 @@ impl Signature {
         desc: impl Into<String>,
         short: Option<char>,
     ) -> Signature {
+<<<<<<< HEAD
+=======
+        let (name, s) = self.check_names(name, short);
+
+        self.named.push(Flag {
+            long: name,
+            short: s,
+            arg: None,
+            required: false,
+            desc: desc.into(),
+            var_id: None,
+        });
+
+        self
+    }
+
+    /// Changes the signature category
+    pub fn category(mut self, category: Category) -> Signature {
+        self.category = category;
+
+        self
+    }
+
+    /// Sets that signature will create a scope as it parses
+    pub fn creates_scope(mut self) -> Signature {
+        self.creates_scope = true;
+        self
+    }
+
+    pub fn call_signature(&self) -> String {
+        let mut one_liner = String::new();
+        one_liner.push_str(&self.name);
+        one_liner.push(' ');
+
+        // Note: the call signature needs flags first because on the nu commandline,
+        // flags will precede the script file name. Flags for internal commands can come
+        // either before or after (or around) positional parameters, so there isn't a strong
+        // preference, so we default to the more constrained example.
+        if self.named.len() > 1 {
+            one_liner.push_str("{flags} ");
+        }
+
+        for positional in &self.required_positional {
+            one_liner.push_str(&get_positional_short_name(positional, true));
+        }
+        for positional in &self.optional_positional {
+            one_liner.push_str(&get_positional_short_name(positional, false));
+        }
+
+        if let Some(rest) = &self.rest_positional {
+            one_liner.push_str(&format!("...{}", get_positional_short_name(rest, false)));
+        }
+
+        // if !self.subcommands.is_empty() {
+        //     one_liner.push_str("<subcommand> ");
+        // }
+
+        one_liner
+    }
+
+    /// Get list of the short-hand flags
+    pub fn get_shorts(&self) -> Vec<char> {
+        self.named.iter().filter_map(|f| f.short).collect()
+    }
+
+    /// Get list of the long-hand flags
+    pub fn get_names(&self) -> Vec<&str> {
+        self.named.iter().map(|f| f.long.as_str()).collect()
+    }
+
+    /// Checks if short or long are already present
+    /// Panics if one of them is found
+    fn check_names(&self, name: impl Into<String>, short: Option<char>) -> (String, Option<char>) {
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
         let s = short.map(|c| {
             debug_assert!(
                 !self.get_shorts().contains(&c),
@@ -334,9 +619,95 @@ impl Signature {
             c
         });
 
+<<<<<<< HEAD
         self.named
             .insert(name.into(), (NamedType::Switch(s), desc.into()));
         self
+=======
+        let name = {
+            let name: String = name.into();
+            debug_assert!(
+                !self.get_names().contains(&name.as_str()),
+                "There may be duplicate name flags, such as --help"
+            );
+            name
+        };
+
+        (name, s)
+    }
+
+    pub fn get_positional(&self, position: usize) -> Option<PositionalArg> {
+        if position < self.required_positional.len() {
+            self.required_positional.get(position).cloned()
+        } else if position < (self.required_positional.len() + self.optional_positional.len()) {
+            self.optional_positional
+                .get(position - self.required_positional.len())
+                .cloned()
+        } else {
+            self.rest_positional.clone()
+        }
+    }
+
+    pub fn num_positionals(&self) -> usize {
+        let mut total = self.required_positional.len() + self.optional_positional.len();
+
+        for positional in &self.required_positional {
+            if let SyntaxShape::Keyword(..) = positional.shape {
+                // Keywords have a required argument, so account for that
+                total += 1;
+            }
+        }
+        for positional in &self.optional_positional {
+            if let SyntaxShape::Keyword(..) = positional.shape {
+                // Keywords have a required argument, so account for that
+                total += 1;
+            }
+        }
+        total
+    }
+
+    pub fn num_positionals_after(&self, idx: usize) -> usize {
+        let mut total = 0;
+
+        for (curr, positional) in self.required_positional.iter().enumerate() {
+            match positional.shape {
+                SyntaxShape::Keyword(..) => {
+                    // Keywords have a required argument, so account for that
+                    if curr > idx {
+                        total += 2;
+                    }
+                }
+                _ => {
+                    if curr > idx {
+                        total += 1;
+                    }
+                }
+            }
+        }
+        total
+    }
+
+    /// Find the matching long flag
+    pub fn get_long_flag(&self, name: &str) -> Option<Flag> {
+        for flag in &self.named {
+            if flag.long == name {
+                return Some(flag.clone());
+            }
+        }
+        None
+    }
+
+    /// Find the matching long flag
+    pub fn get_short_flag(&self, short: char) -> Option<Flag> {
+        for flag in &self.named {
+            if let Some(short_flag) = &flag.short {
+                if *short_flag == short {
+                    return Some(flag.clone());
+                }
+            }
+        }
+        None
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
     }
 
     /// Set the filter flag for the signature
@@ -345,6 +716,7 @@ impl Signature {
         self
     }
 
+<<<<<<< HEAD
     /// Set the type for the "rest" of the positional arguments
     /// Note: Not naming the field in your struct holding the rest values "rest", can
     /// cause errors when deserializing
@@ -379,5 +751,102 @@ impl Signature {
             }
         }
         shorts
+=======
+    /// Create a placeholder implementation of Command as a way to predeclare a definition's
+    /// signature so other definitions can see it. This placeholder is later replaced with the
+    /// full definition in a second pass of the parser.
+    pub fn predeclare(self) -> Box<dyn Command> {
+        Box::new(Predeclaration { signature: self })
+    }
+
+    /// Combines a signature and a block into a runnable block
+    pub fn into_block_command(self, block_id: BlockId) -> Box<dyn Command> {
+        Box::new(BlockCommand {
+            signature: self,
+            block_id,
+        })
+    }
+}
+
+#[derive(Clone)]
+struct Predeclaration {
+    signature: Signature,
+}
+
+impl Command for Predeclaration {
+    fn name(&self) -> &str {
+        &self.signature.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.signature.usage
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        _call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, crate::ShellError> {
+        panic!("Internal error: can't run a predeclaration without a body")
+    }
+}
+
+fn get_positional_short_name(arg: &PositionalArg, is_required: bool) -> String {
+    match &arg.shape {
+        SyntaxShape::Keyword(name, ..) => {
+            if is_required {
+                format!("{} <{}> ", String::from_utf8_lossy(name), arg.name)
+            } else {
+                format!("({} <{}>) ", String::from_utf8_lossy(name), arg.name)
+            }
+        }
+        _ => {
+            if is_required {
+                format!("<{}> ", arg.name)
+            } else {
+                format!("({}) ", arg.name)
+            }
+        }
+    }
+}
+
+#[derive(Clone)]
+struct BlockCommand {
+    signature: Signature,
+    block_id: BlockId,
+}
+
+impl Command for BlockCommand {
+    fn name(&self) -> &str {
+        &self.signature.name
+    }
+
+    fn signature(&self) -> Signature {
+        self.signature.clone()
+    }
+
+    fn usage(&self) -> &str {
+        &self.signature.usage
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        _call: &Call,
+        _input: PipelineData,
+    ) -> Result<crate::PipelineData, crate::ShellError> {
+        panic!("Internal error: can't run custom command with 'run', use block_id");
+    }
+
+    fn get_block_id(&self) -> Option<BlockId> {
+        Some(self.block_id)
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
     }
 }

crates/nu-protocol/src/span.rs

@@ -0,0 +1,76 @@
+use miette::SourceSpan;
+use serde::{Deserialize, Serialize};
+
+/// A spanned area of interest, generic over what kind of thing is of interest
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Spanned<T>
+where
+    T: Clone + std::fmt::Debug,
+{
+    pub item: T,
+    pub span: Span,
+}
+
+/// Spans are a global offset across all seen files, which are cached in the engine's state. The start and
+/// end offset together make the inclusive start/exclusive end pair for where to underline to highlight
+/// a given point of interest.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+pub struct Span {
+    pub start: usize,
+    pub end: usize,
+}
+
+impl From<Span> for SourceSpan {
+    fn from(s: Span) -> Self {
+        Self::new(s.start.into(), (s.end - s.start).into())
+    }
+}
+
+impl Span {
+    pub fn new(start: usize, end: usize) -> Span {
+        Span { start, end }
+    }
+
+    /// Note: Only use this for test data, *not* live data, as it will point into unknown source
+    /// when used in errors.
+    pub fn test_data() -> Span {
+        Span { start: 0, end: 0 }
+    }
+
+    pub fn offset(&self, offset: usize) -> Span {
+        Span {
+            start: self.start - offset,
+            end: self.end - offset,
+        }
+    }
+
+    pub fn contains(&self, pos: usize) -> bool {
+        pos >= self.start && pos < self.end
+    }
+
+    /// Point to the space just past this span, useful for missing
+    /// values
+    pub fn past(&self) -> Span {
+        Span {
+            start: self.end,
+            end: self.end,
+        }
+    }
+}
+
+/// Used when you have a slice of spans of at least size 1
+pub fn span(spans: &[Span]) -> Span {
+    let length = spans.len();
+
+    if length == 0 {
+        // TODO: do this for now, but we might also want to protect against this case
+        Span { start: 0, end: 0 }
+    } else if length == 1 {
+        spans[0]
+    } else {
+        Span {
+            start: spans[0].start,
+            end: spans[length - 1].end,
+        }
+    }
+}

crates/nu-protocol/src/syntax_shape.rs

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 use nu_source::{DbgDocBldr, DebugDocBuilder, PrettyDebug};
 use serde::{Deserialize, Serialize};
 
@@ -58,13 +59,171 @@ impl SyntaxShape {
             SyntaxShape::Operator => "operator",
             SyntaxShape::RowCondition => "condition",
             SyntaxShape::MathExpression => "math expression",
+=======
+use std::fmt::Display;
+
+use serde::{Deserialize, Serialize};
+
+use crate::Type;
+
+/// The syntactic shapes that values must match to be passed into a command. You can think of this as the type-checking that occurs when you call a function.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum SyntaxShape {
+    /// A specific match to a word or symbol
+    Keyword(Vec<u8>, Box<SyntaxShape>),
+
+    /// Any syntactic form is allowed
+    Any,
+
+    /// Strings and string-like bare words are allowed
+    String,
+
+    /// A dotted path to navigate the table
+    CellPath,
+
+    /// A dotted path to navigate the table (including variable)
+    FullCellPath,
+
+    /// Only a numeric (integer or decimal) value is allowed
+    Number,
+
+    /// A range is allowed (eg, `1..3`)
+    Range,
+
+    /// Only an integer value is allowed
+    Int,
+
+    /// A filepath is allowed
+    Filepath,
+
+    /// A glob pattern is allowed, eg `foo*`
+    GlobPattern,
+
+    /// A module path pattern used for imports
+    ImportPattern,
+
+    /// A block is allowed, eg `{start this thing}`
+    Block(Option<Vec<SyntaxShape>>),
+
+    /// A table is allowed, eg `[[first, second]; [1, 2]]`
+    Table,
+
+    /// A table is allowed, eg `[first second]`
+    List(Box<SyntaxShape>),
+
+    /// A filesize value is allowed, eg `10kb`
+    Filesize,
+
+    /// A duration value is allowed, eg `19day`
+    Duration,
+
+    /// An operator
+    Operator,
+
+    /// A math expression which expands shorthand forms on the lefthand side, eg `foo > 1`
+    /// The shorthand allows us to more easily reach columns inside of the row being passed in
+    RowCondition,
+
+    /// A general math expression, eg `1 + 2`
+    MathExpression,
+
+    /// A variable name
+    Variable,
+
+    /// A variable with optional type, `x` or `x: int`
+    VarWithOptType,
+
+    /// A signature for a definition, `[x:int, --foo]`
+    Signature,
+
+    /// A general expression, eg `1 + 2` or `foo --bar`
+    Expression,
+
+    /// A boolean value
+    Boolean,
+
+    /// A record value
+    Record,
+
+    /// A custom shape with custom completion logic
+    Custom(Box<SyntaxShape>, String),
+}
+
+impl SyntaxShape {
+    pub fn to_type(&self) -> Type {
+        match self {
+            SyntaxShape::Any => Type::Unknown,
+            SyntaxShape::Block(_) => Type::Block,
+            SyntaxShape::CellPath => Type::Unknown,
+            SyntaxShape::Custom(custom, _) => custom.to_type(),
+            SyntaxShape::Duration => Type::Duration,
+            SyntaxShape::Expression => Type::Unknown,
+            SyntaxShape::Filepath => Type::String,
+            SyntaxShape::Filesize => Type::Filesize,
+            SyntaxShape::FullCellPath => Type::Unknown,
+            SyntaxShape::GlobPattern => Type::String,
+            SyntaxShape::ImportPattern => Type::Unknown,
+            SyntaxShape::Int => Type::Int,
+            SyntaxShape::List(x) => {
+                let contents = x.to_type();
+                Type::List(Box::new(contents))
+            }
+            SyntaxShape::Keyword(_, expr) => expr.to_type(),
+            SyntaxShape::MathExpression => Type::Unknown,
+            SyntaxShape::Number => Type::Number,
+            SyntaxShape::Operator => Type::Unknown,
+            SyntaxShape::Range => Type::Unknown,
+            SyntaxShape::Record => Type::Record(vec![]), // FIXME: Add actual record type
+            SyntaxShape::RowCondition => Type::Bool,
+            SyntaxShape::Boolean => Type::Bool,
+            SyntaxShape::Signature => Type::Signature,
+            SyntaxShape::String => Type::String,
+            SyntaxShape::Table => Type::List(Box::new(Type::Unknown)), // FIXME: Tables should have better types
+            SyntaxShape::VarWithOptType => Type::Unknown,
+            SyntaxShape::Variable => Type::Unknown,
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
         }
     }
 }
 
+<<<<<<< HEAD
 impl PrettyDebug for SyntaxShape {
     /// Prepare SyntaxShape for pretty-printing
     fn pretty(&self) -> DebugDocBuilder {
         DbgDocBldr::kind(self.syntax_shape_name().to_string())
+=======
+impl Display for SyntaxShape {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            SyntaxShape::Keyword(kw, shape) => {
+                write!(f, "\"{}\" {}", String::from_utf8_lossy(kw), shape)
+            }
+            SyntaxShape::Any => write!(f, "any"),
+            SyntaxShape::String => write!(f, "string"),
+            SyntaxShape::CellPath => write!(f, "cellpath"),
+            SyntaxShape::FullCellPath => write!(f, "cellpath"),
+            SyntaxShape::Number => write!(f, "number"),
+            SyntaxShape::Range => write!(f, "range"),
+            SyntaxShape::Int => write!(f, "int"),
+            SyntaxShape::Filepath => write!(f, "path"),
+            SyntaxShape::GlobPattern => write!(f, "glob"),
+            SyntaxShape::ImportPattern => write!(f, "import"),
+            SyntaxShape::Block(_) => write!(f, "block"),
+            SyntaxShape::Table => write!(f, "table"),
+            SyntaxShape::List(x) => write!(f, "list<{}>", x),
+            SyntaxShape::Record => write!(f, "record"),
+            SyntaxShape::Filesize => write!(f, "filesize"),
+            SyntaxShape::Duration => write!(f, "duration"),
+            SyntaxShape::Operator => write!(f, "operator"),
+            SyntaxShape::RowCondition => write!(f, "condition"),
+            SyntaxShape::MathExpression => write!(f, "variable"),
+            SyntaxShape::Variable => write!(f, "var"),
+            SyntaxShape::VarWithOptType => write!(f, "vardecl"),
+            SyntaxShape::Signature => write!(f, "signature"),
+            SyntaxShape::Expression => write!(f, "expression"),
+            SyntaxShape::Boolean => write!(f, "bool"),
+            SyntaxShape::Custom(x, _) => write!(f, "custom<{}>", x),
+        }
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
     }
 }

crates/nu-protocol/src/ty.rs

@@ -0,0 +1,64 @@
+use serde::{Deserialize, Serialize};
+
+use std::fmt::Display;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Type {
+    Int,
+    Float,
+    Range,
+    Bool,
+    String,
+    Block,
+    CellPath,
+    Duration,
+    Date,
+    Filesize,
+    List(Box<Type>),
+    Number,
+    Nothing,
+    Record(Vec<(String, Type)>),
+    Table,
+    ValueStream,
+    Unknown,
+    Error,
+    Binary,
+    Custom,
+    Signature,
+}
+
+impl Display for Type {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Type::Block => write!(f, "block"),
+            Type::Bool => write!(f, "bool"),
+            Type::CellPath => write!(f, "cell path"),
+            Type::Date => write!(f, "date"),
+            Type::Duration => write!(f, "duration"),
+            Type::Filesize => write!(f, "filesize"),
+            Type::Float => write!(f, "float"),
+            Type::Int => write!(f, "int"),
+            Type::Range => write!(f, "range"),
+            Type::Record(fields) => write!(
+                f,
+                "record<{}>",
+                fields
+                    .iter()
+                    .map(|(x, y)| format!("{}: {}", x, y))
+                    .collect::<Vec<String>>()
+                    .join(", "),
+            ),
+            Type::Table => write!(f, "table"),
+            Type::List(l) => write!(f, "list<{}>", l),
+            Type::Nothing => write!(f, "nothing"),
+            Type::Number => write!(f, "number"),
+            Type::String => write!(f, "string"),
+            Type::ValueStream => write!(f, "value stream"),
+            Type::Unknown => write!(f, "unknown"),
+            Type::Error => write!(f, "error"),
+            Type::Binary => write!(f, "binary"),
+            Type::Custom => write!(f, "custom"),
+            Type::Signature => write!(f, "signature"),
+        }
+    }
+}

crates/nu-protocol/src/type_name.rs

@@ -1,44 +0,0 @@
-use nu_source::{DebugDocBuilder, HasSpan, Spanned, SpannedItem, Tagged};
-
-/// A trait that allows structures to define a known .type_name() which pretty-prints the type
-pub trait ShellTypeName {
-    fn type_name(&self) -> &'static str;
-}
-
-impl<T: ShellTypeName> ShellTypeName for Spanned<T> {
-    /// Return the type_name of the spanned item
-    fn type_name(&self) -> &'static str {
-        self.item.type_name()
-    }
-}
-
-impl<T: ShellTypeName> ShellTypeName for &T {
-    /// Return the type_name for the borrowed reference
-    fn type_name(&self) -> &'static str {
-        (*self).type_name()
-    }
-}
-
-/// A trait that allows structures to define a known way to return a spanned type name
-pub trait SpannedTypeName {
-    fn spanned_type_name(&self) -> Spanned<&'static str>;
-}
-
-impl<T: ShellTypeName + HasSpan> SpannedTypeName for T {
-    /// Return the type name as a spanned string
-    fn spanned_type_name(&self) -> Spanned<&'static str> {
-        self.type_name().spanned(self.span())
-    }
-}
-
-impl<T: ShellTypeName> SpannedTypeName for Tagged<T> {
-    /// Return the spanned type name for a Tagged value
-    fn spanned_type_name(&self) -> Spanned<&'static str> {
-        self.item.type_name().spanned(self.tag.span)
-    }
-}
-
-/// A trait to enable pretty-printing of type information
-pub trait PrettyType {
-    fn pretty_type(&self) -> DebugDocBuilder;
-}

crates/nu-protocol/src/type_shape.rs

@@ -1,426 +0,0 @@
-///
-/// This file describes the structural types of the nushell system.
-///
-/// Its primary purpose today is to identify "equivalent" values for the purpose
-/// of merging rows into a single table or identify rows in a table that have the
-/// same shape for reflection.
-use crate::value::dict::Dictionary;
-use crate::value::primitive::Primitive;
-use crate::value::range::RangeInclusion;
-use crate::value::{UntaggedValue, Value};
-use derive_new::new;
-use indexmap::map::IndexMap;
-use nu_source::{DbgDocBldr, DebugDoc, DebugDocBuilder, PrettyDebug};
-use serde::{Deserialize, Serialize};
-use std::cmp::Ordering;
-use std::fmt::Debug;
-use std::hash::{Hash, Hasher};
-
-/// Representation of the type of ranges
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, new)]
-pub struct RangeType {
-    from: (Type, RangeInclusion),
-    to: (Type, RangeInclusion),
-}
-
-/// Representation of for the type of a value in Nu
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
-pub enum Type {
-    /// A value which has no value
-    Nothing,
-    /// An integer-based value
-    Int,
-    /// An big integer-based value
-    BigInt,
-    /// A range between two values
-    Range(Box<RangeType>),
-    /// A decimal (floating point) value
-    Decimal,
-    /// A filesize in bytes
-    Filesize,
-    /// A string of text
-    String,
-    /// A line of text (a string with trailing line ending)
-    Line,
-    /// A path through a table
-    ColumnPath,
-    /// A glob pattern (like foo*)
-    GlobPattern,
-    /// A boolean value
-    Boolean,
-    /// A date value (in UTC)
-    Date,
-    /// A data duration value
-    Duration,
-    /// A filepath value
-    FilePath,
-    /// A binary (non-text) buffer value
-    Binary,
-
-    /// A row of data
-    Row(Row),
-    /// A full table of data
-    Table(Vec<Type>),
-
-    /// A block of script (TODO)
-    Block,
-    /// An error value (TODO)
-    Error,
-
-    /// Beginning of stream marker (used as bookend markers rather than actual values)
-    BeginningOfStream,
-    /// End of stream marker (used as bookend markers rather than actual values)
-    EndOfStream,
-
-    /// Dataframe
-    #[cfg(feature = "dataframe")]
-    DataFrame,
-
-    /// Dataframe
-    #[cfg(feature = "dataframe")]
-    FrameStruct,
-}
-
-/// A shape representation of the type of a row
-#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Row {
-    map: IndexMap<Column, Type>,
-}
-
-#[allow(clippy::derive_hash_xor_eq)]
-impl Hash for Row {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let mut entries = self.map.clone();
-        entries.sort_keys();
-        entries.keys().collect::<Vec<&Column>>().hash(state);
-        entries.values().collect::<Vec<&Type>>().hash(state);
-    }
-}
-
-impl PartialOrd for Row {
-    /// Compare two dictionaries for sort ordering
-    fn partial_cmp(&self, other: &Row) -> Option<Ordering> {
-        let this: Vec<&Column> = self.map.keys().collect();
-        let that: Vec<&Column> = other.map.keys().collect();
-
-        if this != that {
-            return this.partial_cmp(&that);
-        }
-
-        let this: Vec<&Type> = self.map.values().collect();
-        let that: Vec<&Type> = self.map.values().collect();
-
-        this.partial_cmp(&that)
-    }
-}
-
-impl Ord for Row {
-    /// Compare two dictionaries for ordering
-    fn cmp(&self, other: &Row) -> Ordering {
-        let this: Vec<&Column> = self.map.keys().collect();
-        let that: Vec<&Column> = other.map.keys().collect();
-
-        if this != that {
-            return this.cmp(&that);
-        }
-
-        let this: Vec<&Type> = self.map.values().collect();
-        let that: Vec<&Type> = self.map.values().collect();
-
-        this.cmp(&that)
-    }
-}
-
-impl Type {
-    /// Convert a Primitive into its corresponding Type
-    pub fn from_primitive(primitive: &Primitive) -> Type {
-        match primitive {
-            Primitive::Nothing => Type::Nothing,
-            Primitive::Int(_) => Type::Int,
-            Primitive::BigInt(_) => Type::BigInt,
-            Primitive::Range(range) => {
-                let (left_value, left_inclusion) = &range.from;
-                let (right_value, right_inclusion) = &range.to;
-
-                let left_type = (Type::from_primitive(left_value), *left_inclusion);
-                let right_type = (Type::from_primitive(right_value), *right_inclusion);
-
-                let range = RangeType::new(left_type, right_type);
-                Type::Range(Box::new(range))
-            }
-            Primitive::Decimal(_) => Type::Decimal,
-            Primitive::Filesize(_) => Type::Filesize,
-            Primitive::String(_) => Type::String,
-            Primitive::ColumnPath(_) => Type::ColumnPath,
-            Primitive::GlobPattern(_) => Type::GlobPattern,
-            Primitive::Boolean(_) => Type::Boolean,
-            Primitive::Date(_) => Type::Date,
-            Primitive::Duration(_) => Type::Duration,
-            Primitive::FilePath(_) => Type::FilePath,
-            Primitive::Binary(_) => Type::Binary,
-            Primitive::BeginningOfStream => Type::BeginningOfStream,
-            Primitive::EndOfStream => Type::EndOfStream,
-        }
-    }
-
-    /// Convert a dictionary into its corresponding row Type
-    pub fn from_dictionary(dictionary: &Dictionary) -> Type {
-        let mut map = IndexMap::new();
-
-        for (key, value) in &dictionary.entries {
-            let column = Column::String(key.clone());
-            map.insert(column, Type::from_value(value));
-        }
-
-        Type::Row(Row { map })
-    }
-
-    /// Convert a table into its corresponding Type
-    pub fn from_table<'a>(table: impl IntoIterator<Item = &'a Value>) -> Type {
-        let mut vec = vec![];
-
-        for item in table {
-            vec.push(Type::from_value(item))
-        }
-
-        Type::Table(vec)
-    }
-
-    /// Convert a value into its corresponding Type
-    pub fn from_value<'a>(value: impl Into<&'a UntaggedValue>) -> Type {
-        match value.into() {
-            UntaggedValue::Primitive(p) => Type::from_primitive(p),
-            UntaggedValue::Row(row) => Type::from_dictionary(row),
-            UntaggedValue::Table(table) => Type::from_table(table),
-            UntaggedValue::Error(_) => Type::Error,
-            UntaggedValue::Block(_) => Type::Block,
-            #[cfg(feature = "dataframe")]
-            UntaggedValue::DataFrame(_) => Type::DataFrame,
-            #[cfg(feature = "dataframe")]
-            UntaggedValue::FrameStruct(_) => Type::DataFrame,
-        }
-    }
-}
-
-impl PrettyDebug for Type {
-    /// Prepare Type for pretty-printing
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            Type::Nothing => ty("nothing"),
-            Type::Int => ty("integer"),
-            Type::BigInt => ty("big integer"),
-            Type::Range(range) => {
-                let (left, left_inclusion) = &range.from;
-                let (right, right_inclusion) = &range.to;
-
-                let left_bracket = DbgDocBldr::delimiter(match left_inclusion {
-                    RangeInclusion::Exclusive => "(",
-                    RangeInclusion::Inclusive => "[",
-                });
-
-                let right_bracket = DbgDocBldr::delimiter(match right_inclusion {
-                    RangeInclusion::Exclusive => ")",
-                    RangeInclusion::Inclusive => "]",
-                });
-
-                DbgDocBldr::typed(
-                    "range",
-                    (left_bracket
-                        + left.pretty()
-                        + DbgDocBldr::operator(",")
-                        + DbgDocBldr::space()
-                        + right.pretty()
-                        + right_bracket)
-                        .group(),
-                )
-            }
-            Type::Decimal => ty("decimal"),
-            Type::Filesize => ty("filesize"),
-            Type::String => ty("string"),
-            Type::Line => ty("line"),
-            Type::ColumnPath => ty("column-path"),
-            Type::GlobPattern => ty("pattern"),
-            Type::Boolean => ty("boolean"),
-            Type::Date => ty("date"),
-            Type::Duration => ty("duration"),
-            Type::FilePath => ty("path"),
-            Type::Binary => ty("binary"),
-            Type::Error => DbgDocBldr::error("error"),
-            Type::BeginningOfStream => DbgDocBldr::keyword("beginning-of-stream"),
-            Type::EndOfStream => DbgDocBldr::keyword("end-of-stream"),
-            Type::Row(row) => (DbgDocBldr::kind("row")
-                + DbgDocBldr::space()
-                + DbgDocBldr::intersperse(
-                    row.map.iter().map(|(key, ty)| {
-                        (DbgDocBldr::key(match key {
-                            Column::String(string) => string.clone(),
-                            Column::Value => "".to_string(),
-                        }) + DbgDocBldr::delimit("(", ty.pretty(), ")").into_kind())
-                        .nest()
-                    }),
-                    DbgDocBldr::space(),
-                )
-                .nest())
-            .nest(),
-
-            Type::Table(table) => {
-                let mut group: Group<DebugDoc, Vec<(usize, usize)>> = Group::new();
-
-                for (i, item) in table.iter().enumerate() {
-                    group.add(item.to_doc(), i);
-                }
-
-                (DbgDocBldr::kind("table") + DbgDocBldr::space() + DbgDocBldr::keyword("of"))
-                    .group()
-                    + DbgDocBldr::space()
-                    + (if group.len() == 1 {
-                        let (doc, _) = group.into_iter().collect::<Vec<_>>()[0].clone();
-                        DebugDocBuilder::from_doc(doc)
-                    } else {
-                        DbgDocBldr::intersperse(
-                            group.into_iter().map(|(doc, rows)| {
-                                (DbgDocBldr::intersperse(
-                                    rows.iter().map(|(from, to)| {
-                                        if from == to {
-                                            DbgDocBldr::description(from)
-                                        } else {
-                                            (DbgDocBldr::description(from)
-                                                + DbgDocBldr::space()
-                                                + DbgDocBldr::keyword("to")
-                                                + DbgDocBldr::space()
-                                                + DbgDocBldr::description(to))
-                                            .group()
-                                        }
-                                    }),
-                                    DbgDocBldr::description(", "),
-                                ) + DbgDocBldr::description(":")
-                                    + DbgDocBldr::space()
-                                    + DebugDocBuilder::from_doc(doc))
-                                .nest()
-                            }),
-                            DbgDocBldr::space(),
-                        )
-                    })
-            }
-            Type::Block => ty("block"),
-            #[cfg(feature = "dataframe")]
-            Type::DataFrame | Type::FrameStruct => ty("data_type_formatter"),
-        }
-    }
-}
-
-/// A view into dictionaries for debug purposes
-#[derive(Debug, new)]
-struct DebugEntry<'a> {
-    key: &'a Column,
-    value: &'a Type,
-}
-
-impl<'a> PrettyDebug for DebugEntry<'a> {
-    /// Prepare debug entries for pretty-printing
-    fn pretty(&self) -> DebugDocBuilder {
-        DbgDocBldr::key(match self.key {
-            Column::String(string) => string.clone(),
-            Column::Value => "".to_string(),
-        }) + DbgDocBldr::delimit("(", self.value.pretty(), ")").into_kind()
-    }
-}
-
-/// Helper to create a pretty-print for the type
-fn ty(name: impl std::fmt::Display) -> DebugDocBuilder {
-    DbgDocBldr::kind(name)
-}
-
-pub trait GroupedValue: Debug + Clone {
-    type Item;
-
-    fn new() -> Self;
-    fn merge(&mut self, value: Self::Item);
-}
-
-impl GroupedValue for Vec<(usize, usize)> {
-    type Item = usize;
-
-    fn new() -> Vec<(usize, usize)> {
-        vec![]
-    }
-
-    fn merge(&mut self, new_value: usize) {
-        match self.last_mut() {
-            Some(value) if value.1 == new_value - 1 => {
-                value.1 += 1;
-            }
-
-            _ => self.push((new_value, new_value)),
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct Group<K: Debug + Eq + Hash, V: GroupedValue> {
-    values: indexmap::IndexMap<K, V>,
-}
-
-impl<K, G> Group<K, G>
-where
-    K: Debug + Eq + Hash,
-    G: GroupedValue,
-{
-    pub fn new() -> Group<K, G> {
-        Group {
-            values: indexmap::IndexMap::default(),
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        self.values.len()
-    }
-
-    pub fn into_iter(self) -> impl Iterator<Item = (K, G)> {
-        self.values.into_iter()
-    }
-
-    pub fn add(&mut self, key: impl Into<K>, value: impl Into<G::Item>) {
-        let key = key.into();
-        let value = value.into();
-
-        let group = self.values.get_mut(&key);
-
-        match group {
-            None => {
-                self.values.insert(key, {
-                    let mut group = G::new();
-                    group.merge(value);
-                    group
-                });
-            }
-            Some(group) => {
-                group.merge(value);
-            }
-        }
-    }
-}
-
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
-pub enum Column {
-    String(String),
-    Value,
-}
-
-impl From<String> for Column {
-    fn from(x: String) -> Self {
-        Column::String(x)
-    }
-}
-
-impl From<&String> for Column {
-    fn from(x: &String) -> Self {
-        Column::String(x.clone())
-    }
-}
-
-impl From<&str> for Column {
-    fn from(x: &str) -> Self {
-        Column::String(x.to_string())
-    }
-}

crates/nu-protocol/src/value/column_path.rs

@@ -1,363 +0,0 @@
-use derive_new::new;
-use getset::Getters;
-use nu_source::{
-    span_for_spanned_list, DbgDocBldr, DebugDocBuilder, HasFallibleSpan, PrettyDebug, Span,
-    Spanned, SpannedItem,
-};
-use serde::{Deserialize, Serialize};
-
-use crate::hir::{Expression, Literal, Member, SpannedExpression};
-use nu_errors::ParseError;
-
-/// A PathMember that has yet to be spanned so that it can be used in later processing
-#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
-pub enum UnspannedPathMember {
-    String(String),
-    Int(i64),
-}
-
-impl UnspannedPathMember {
-    /// Add the span information and get a full PathMember
-    pub fn into_path_member(self, span: impl Into<Span>) -> PathMember {
-        PathMember {
-            unspanned: self,
-            span: span.into(),
-        }
-    }
-}
-
-/// A basic piece of a ColumnPath, which describes the steps to take through a table to arrive a cell, row, or inner table
-#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
-pub struct PathMember {
-    pub unspanned: UnspannedPathMember,
-    pub span: Span,
-}
-
-impl PrettyDebug for &PathMember {
-    /// Gets the PathMember ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        match &self.unspanned {
-            UnspannedPathMember::String(string) => DbgDocBldr::primitive(format!("{:?}", string)),
-            UnspannedPathMember::Int(int) => DbgDocBldr::primitive(int),
-        }
-    }
-}
-
-/// The fundamental path primitive to describe how to navigate through a table to get to a sub-item. A path member can be either a word or a number. Words/strings are taken to mean
-/// a column name, and numbers are the row number. Taken together they describe which column or row to narrow to in order to get data.
-///
-/// Rows must follow column names, they can't come first. eg) `foo.1` is valid where `1.foo` is not.
-#[derive(
-    Debug, Hash, Serialize, Deserialize, Ord, PartialOrd, Eq, PartialEq, Getters, Clone, new,
-)]
-pub struct ColumnPath {
-    #[get = "pub"]
-    members: Vec<PathMember>,
-}
-
-impl ColumnPath {
-    /// Iterate over the members of the column path
-    pub fn iter(&self) -> impl Iterator<Item = &PathMember> {
-        self.members.iter()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.members.is_empty()
-    }
-
-    /// Returns the last member and a slice of the remaining members
-    pub fn split_last(&self) -> Option<(&PathMember, &[PathMember])> {
-        self.members.split_last()
-    }
-
-    /// Returns the last member
-    pub fn last(&self) -> Option<&PathMember> {
-        self.iter().last()
-    }
-
-    pub fn path(&self) -> String {
-        let sep = std::path::MAIN_SEPARATOR;
-        let mut members = self.iter();
-        let mut f = String::from(sep);
-
-        if let Some(member) = members.next() {
-            f.push_str(&member.as_string());
-        }
-
-        for member in members {
-            f.push(sep);
-            f.push_str(&member.as_string());
-        }
-
-        f
-    }
-
-    pub fn build(text: &Spanned<String>) -> ColumnPath {
-        if let (
-            SpannedExpression {
-                expr: Expression::Literal(Literal::ColumnPath(path)),
-                span: _,
-            },
-            _,
-        ) = parse(text)
-        {
-            ColumnPath {
-                members: path.iter().map(|member| member.to_path_member()).collect(),
-            }
-        } else {
-            ColumnPath { members: vec![] }
-        }
-    }
-
-    pub fn with_head(text: &Spanned<String>) -> Option<(String, ColumnPath)> {
-        match parse_full_column_path(text) {
-            (
-                SpannedExpression {
-                    expr: Expression::FullColumnPath(path),
-                    ..
-                },
-                _,
-            ) => {
-                if let crate::hir::FullColumnPath {
-                    head:
-                        SpannedExpression {
-                            expr: Expression::Variable(name, _),
-                            span: _,
-                        },
-                    tail,
-                } = *path
-                {
-                    Some((
-                        name,
-                        ColumnPath {
-                            members: tail.to_vec(),
-                        },
-                    ))
-                } else {
-                    None
-                }
-            }
-            _ => None,
-        }
-    }
-}
-
-impl PrettyDebug for ColumnPath {
-    /// Gets the ColumnPath ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        let members: Vec<DebugDocBuilder> =
-            self.members.iter().map(|member| member.pretty()).collect();
-
-        DbgDocBldr::delimit(
-            "(",
-            DbgDocBldr::description("path")
-                + DbgDocBldr::equals()
-                + DbgDocBldr::intersperse(members, DbgDocBldr::space()),
-            ")",
-        )
-        .nest()
-    }
-}
-
-impl HasFallibleSpan for ColumnPath {
-    /// Creates a span that will cover the column path, if possible
-    fn maybe_span(&self) -> Option<Span> {
-        if self.members.is_empty() {
-            None
-        } else {
-            Some(span_for_spanned_list(self.members.iter().map(|m| m.span)))
-        }
-    }
-}
-
-impl<'a> IntoIterator for &'a ColumnPath {
-    type Item = &'a PathMember;
-
-    type IntoIter = std::slice::Iter<'a, PathMember>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        self.members.iter()
-    }
-}
-
-impl PathMember {
-    /// Create a string path member
-    pub fn string(string: impl Into<String>, span: impl Into<Span>) -> PathMember {
-        UnspannedPathMember::String(string.into()).into_path_member(span)
-    }
-
-    /// Create a numeric path member
-    pub fn int(int: i64, span: impl Into<Span>) -> PathMember {
-        UnspannedPathMember::Int(int).into_path_member(span)
-    }
-
-    pub fn as_string(&self) -> String {
-        match &self.unspanned {
-            UnspannedPathMember::String(string) => string.clone(),
-            UnspannedPathMember::Int(int) => int.to_string(),
-        }
-    }
-}
-
-fn parse_full_column_path(
-    raw_column_path: &Spanned<String>,
-) -> (SpannedExpression, Option<ParseError>) {
-    let mut inside_delimiter = vec![];
-    let mut output = vec![];
-    let mut current_part = String::new();
-    let mut start_index = 0;
-    let mut last_index = 0;
-    let error = None;
-
-    let mut head = None;
-
-    for (idx, c) in raw_column_path.item.char_indices() {
-        last_index = idx;
-        if c == '(' {
-            inside_delimiter.push(')');
-        } else if let Some(delimiter) = inside_delimiter.last() {
-            if c == *delimiter {
-                inside_delimiter.pop();
-            }
-        } else if c == '\'' || c == '"' {
-            inside_delimiter.push(c);
-        } else if c == '.' {
-            let part_span = Span::new(
-                raw_column_path.span.start() + start_index,
-                raw_column_path.span.start() + idx,
-            );
-
-            if head.is_none() && current_part.starts_with('$') {
-                // We have the variable head
-                head = Some(Expression::variable(current_part.clone(), part_span))
-            } else if let Ok(row_number) = current_part.parse::<i64>() {
-                output.push(UnspannedPathMember::Int(row_number).into_path_member(part_span));
-            } else {
-                let current_part = trim_quotes(&current_part);
-                output.push(
-                    UnspannedPathMember::String(current_part.clone()).into_path_member(part_span),
-                );
-            }
-            current_part.clear();
-            // Note: I believe this is safe because of the delimiter we're using,
-            // but if we get fancy with Unicode we'll need to change this.
-            start_index = idx + '.'.len_utf8();
-            continue;
-        }
-        current_part.push(c);
-    }
-
-    if !current_part.is_empty() {
-        let part_span = Span::new(
-            raw_column_path.span.start() + start_index,
-            raw_column_path.span.start() + last_index + 1,
-        );
-
-        if head.is_none() {
-            if current_part.starts_with('$') {
-                head = Some(Expression::variable(current_part, raw_column_path.span));
-            } else if let Ok(row_number) = current_part.parse::<i64>() {
-                output.push(UnspannedPathMember::Int(row_number).into_path_member(part_span));
-            } else {
-                let current_part = trim_quotes(&current_part);
-                output.push(UnspannedPathMember::String(current_part).into_path_member(part_span));
-            }
-        } else if let Ok(row_number) = current_part.parse::<i64>() {
-            output.push(UnspannedPathMember::Int(row_number).into_path_member(part_span));
-        } else {
-            let current_part = trim_quotes(&current_part);
-            output.push(UnspannedPathMember::String(current_part).into_path_member(part_span));
-        }
-    }
-
-    if let Some(head) = head {
-        (
-            SpannedExpression::new(
-                Expression::path(SpannedExpression::new(head, raw_column_path.span), output),
-                raw_column_path.span,
-            ),
-            error,
-        )
-    } else {
-        (
-            SpannedExpression::new(
-                Expression::path(
-                    SpannedExpression::new(
-                        Expression::variable("$it".into(), raw_column_path.span),
-                        raw_column_path.span,
-                    ),
-                    output,
-                ),
-                raw_column_path.span,
-            ),
-            error,
-        )
-    }
-}
-
-fn parse(raw_column_path: &Spanned<String>) -> (SpannedExpression, Option<ParseError>) {
-    let mut delimiter = '.';
-    let mut inside_delimiter = false;
-    let mut output = vec![];
-    let mut current_part = String::new();
-    let mut start_index = 0;
-    let mut last_index = 0;
-
-    for (idx, c) in raw_column_path.item.char_indices() {
-        last_index = idx;
-        if inside_delimiter {
-            if c == delimiter {
-                inside_delimiter = false;
-            }
-        } else if c == '\'' || c == '"' || c == '`' {
-            inside_delimiter = true;
-            delimiter = c;
-        } else if c == '.' {
-            let part_span = Span::new(
-                raw_column_path.span.start() + start_index,
-                raw_column_path.span.start() + idx,
-            );
-
-            if let Ok(row_number) = current_part.parse::<i64>() {
-                output.push(Member::Int(row_number, part_span));
-            } else {
-                let trimmed = trim_quotes(&current_part);
-                output.push(Member::Bare(trimmed.clone().spanned(part_span)));
-            }
-            current_part.clear();
-            // Note: I believe this is safe because of the delimiter we're using,
-            // but if we get fancy with Unicode we'll need to change this.
-            start_index = idx + '.'.len_utf8();
-            continue;
-        }
-        current_part.push(c);
-    }
-
-    if !current_part.is_empty() {
-        let part_span = Span::new(
-            raw_column_path.span.start() + start_index,
-            raw_column_path.span.start() + last_index + 1,
-        );
-        if let Ok(row_number) = current_part.parse::<i64>() {
-            output.push(Member::Int(row_number, part_span));
-        } else {
-            let current_part = trim_quotes(&current_part);
-            output.push(Member::Bare(current_part.spanned(part_span)));
-        }
-    }
-
-    (
-        SpannedExpression::new(Expression::simple_column_path(output), raw_column_path.span),
-        None,
-    )
-}
-
-fn trim_quotes(input: &str) -> String {
-    let mut chars = input.chars();
-
-    match (chars.next(), chars.next_back()) {
-        (Some('\''), Some('\'')) => chars.collect(),
-        (Some('"'), Some('"')) => chars.collect(),
-        _ => input.to_string(),
-    }
-}

crates/nu-protocol/src/value/convert.rs

@@ -1,60 +0,0 @@
-use crate::type_name::SpannedTypeName;
-use crate::value::dict::Dictionary;
-use crate::value::primitive::Primitive;
-use crate::value::{UntaggedValue, Value};
-use nu_errors::{CoerceInto, ShellError};
-use nu_source::TaggedItem;
-
-impl std::convert::TryFrom<&Value> for i64 {
-    type Error = ShellError;
-
-    /// Convert to an i64 integer, if possible
-    fn try_from(value: &Value) -> Result<i64, ShellError> {
-        match &value.value {
-            UntaggedValue::Primitive(Primitive::Int(int)) => Ok(*int),
-            UntaggedValue::Primitive(Primitive::BigInt(int)) => {
-                int.tagged(&value.tag).coerce_into("converting to i64")
-            }
-            _ => Err(ShellError::type_error("Integer", value.spanned_type_name())),
-        }
-    }
-}
-
-impl std::convert::TryFrom<&Value> for String {
-    type Error = ShellError;
-
-    /// Convert to a string, if possible
-    fn try_from(value: &Value) -> Result<String, ShellError> {
-        match &value.value {
-            UntaggedValue::Primitive(Primitive::String(s)) => Ok(s.clone()),
-            _ => Err(ShellError::type_error("String", value.spanned_type_name())),
-        }
-    }
-}
-
-impl std::convert::TryFrom<&Value> for Vec<u8> {
-    type Error = ShellError;
-
-    /// Convert to a u8 vec, if possible
-    fn try_from(value: &Value) -> Result<Vec<u8>, ShellError> {
-        match &value.value {
-            UntaggedValue::Primitive(Primitive::Binary(b)) => Ok(b.clone()),
-            _ => Err(ShellError::type_error("Binary", value.spanned_type_name())),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Value> for &'a Dictionary {
-    type Error = ShellError;
-
-    /// Convert to a dictionary, if possible
-    fn try_from(value: &'a Value) -> Result<&'a Dictionary, ShellError> {
-        match &value.value {
-            UntaggedValue::Row(d) => Ok(d),
-            _ => Err(ShellError::type_error(
-                "Dictionary",
-                value.spanned_type_name(),
-            )),
-        }
-    }
-}

crates/nu-protocol/src/value/custom_value.rs

@@ -0,0 +1,60 @@
+use std::{cmp::Ordering, fmt};
+
+use crate::{ast::Operator, ShellError, Span, Value};
+
+// Trait definition for a custom value
+#[typetag::serde(tag = "type")]
+pub trait CustomValue: fmt::Debug + Send + Sync {
+    fn clone_value(&self, span: Span) -> Value;
+
+    //fn category(&self) -> Category;
+
+    // Define string representation of the custom value
+    fn value_string(&self) -> String;
+
+    // Converts the custom value to a base nushell value
+    // This is used to represent the custom value using the table representations
+    // That already exist in nushell
+    fn to_base_value(&self, span: Span) -> Result<Value, ShellError>;
+
+    // Json representation of custom value
+    fn to_json(&self) -> nu_json::Value {
+        nu_json::Value::Null
+    }
+
+    // Any representation used to downcast object to its original type
+    fn as_any(&self) -> &dyn std::any::Any;
+
+    // Follow cell path functions
+    fn follow_path_int(&self, _count: usize, span: Span) -> Result<Value, ShellError> {
+        Err(ShellError::IncompatiblePathAccess(
+            format!("{} does't support path access", self.value_string()),
+            span,
+        ))
+    }
+
+    fn follow_path_string(&self, _column_name: String, span: Span) -> Result<Value, ShellError> {
+        Err(ShellError::IncompatiblePathAccess(
+            format!("{} does't support path access", self.value_string()),
+            span,
+        ))
+    }
+
+    // ordering with other value
+    fn partial_cmp(&self, _other: &Value) -> Option<Ordering> {
+        None
+    }
+
+    // Definition of an operation between the object that implements the trait
+    // and another Value.
+    // The Operator enum is used to indicate the expected operation
+    fn operation(
+        &self,
+        _lhs_span: Span,
+        operator: Operator,
+        op: Span,
+        _right: &Value,
+    ) -> Result<Value, ShellError> {
+        Err(ShellError::UnsupportedOperator(operator, op))
+    }
+}

crates/nu-protocol/src/value/debug.rs

@@ -1,111 +0,0 @@
-use crate::type_name::PrettyType;
-use crate::value::primitive::Primitive;
-use crate::value::{UntaggedValue, Value};
-use nu_source::{DbgDocBldr, DebugDocBuilder, PrettyDebug};
-
-impl PrettyDebug for &Value {
-    /// Get a borrowed Value ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        PrettyDebug::pretty(*self)
-    }
-}
-
-impl PrettyDebug for Value {
-    /// Get a Value ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        match &self.value {
-            UntaggedValue::Primitive(p) => p.pretty(),
-            UntaggedValue::Row(row) => row.pretty_builder().nest(1).group().into(),
-            UntaggedValue::Table(table) => DbgDocBldr::delimit(
-                "[",
-                DbgDocBldr::intersperse(table, DbgDocBldr::space()),
-                "]",
-            )
-            .nest(),
-            UntaggedValue::Error(_) => DbgDocBldr::error("error"),
-            UntaggedValue::Block(_) => DbgDocBldr::opaque("block"),
-            #[cfg(feature = "dataframe")]
-            UntaggedValue::DataFrame(_) | UntaggedValue::FrameStruct(_) => {
-                DbgDocBldr::opaque("dataframe")
-            }
-        }
-    }
-}
-
-impl PrettyType for Primitive {
-    /// Find the type of the Value and prepare it for pretty-printing
-    fn pretty_type(&self) -> DebugDocBuilder {
-        match self {
-            Primitive::Nothing => ty("nothing"),
-            Primitive::Int(_) => ty("integer"),
-            Primitive::BigInt(_) => ty("big-integer"),
-            Primitive::Range(_) => ty("range"),
-            Primitive::Decimal(_) => ty("decimal"),
-            Primitive::Filesize(_) => ty("filesize"),
-            Primitive::String(_) => ty("string"),
-            Primitive::ColumnPath(_) => ty("column-path"),
-            Primitive::GlobPattern(_) => ty("pattern"),
-            Primitive::Boolean(_) => ty("boolean"),
-            Primitive::Date(_) => ty("date"),
-            Primitive::Duration(_) => ty("duration"),
-            Primitive::FilePath(_) => ty("path"),
-            Primitive::Binary(_) => ty("binary"),
-            Primitive::BeginningOfStream => DbgDocBldr::keyword("beginning-of-stream"),
-            Primitive::EndOfStream => DbgDocBldr::keyword("end-of-stream"),
-        }
-    }
-}
-
-impl PrettyDebug for Primitive {
-    /// Get a Primitive value ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        match self {
-            Primitive::Nothing => DbgDocBldr::primitive("nothing"),
-            Primitive::Int(int) => prim(format_args!("{}", int)),
-            Primitive::BigInt(int) => prim(format_args!("{}", int)),
-            Primitive::Decimal(decimal) => prim(format_args!("{}", decimal)),
-            Primitive::Range(range) => {
-                let (left, left_inclusion) = &range.from;
-                let (right, right_inclusion) = &range.to;
-
-                DbgDocBldr::typed(
-                    "range",
-                    (left_inclusion.debug_left_bracket()
-                        + left.pretty()
-                        + DbgDocBldr::operator(",")
-                        + DbgDocBldr::space()
-                        + right.pretty()
-                        + right_inclusion.debug_right_bracket())
-                    .group(),
-                )
-            }
-            Primitive::Filesize(bytes) => primitive_doc(bytes, "filesize"),
-            Primitive::String(string) => prim(string),
-            Primitive::ColumnPath(path) => path.pretty(),
-            Primitive::GlobPattern(pattern) => primitive_doc(pattern, "pattern"),
-            Primitive::Boolean(boolean) => match boolean {
-                true => DbgDocBldr::primitive("$yes"),
-                false => DbgDocBldr::primitive("$no"),
-            },
-            Primitive::Date(date) => primitive_doc(date, "date"),
-            Primitive::Duration(duration) => primitive_doc(duration, "nanoseconds"),
-            Primitive::FilePath(path) => primitive_doc(path, "path"),
-            Primitive::Binary(_) => DbgDocBldr::opaque("binary"),
-            Primitive::BeginningOfStream => DbgDocBldr::keyword("beginning-of-stream"),
-            Primitive::EndOfStream => DbgDocBldr::keyword("end-of-stream"),
-        }
-    }
-}
-
-fn prim(name: impl std::fmt::Debug) -> DebugDocBuilder {
-    DbgDocBldr::primitive(format!("{:?}", name))
-}
-
-fn primitive_doc(name: impl std::fmt::Debug, ty: impl Into<String>) -> DebugDocBuilder {
-    DbgDocBldr::primitive(format!("{:?}", name))
-        + DbgDocBldr::delimit("(", DbgDocBldr::kind(ty.into()), ")")
-}
-
-fn ty(name: impl std::fmt::Debug) -> DebugDocBuilder {
-    DbgDocBldr::kind(format!("{:?}", name))
-}

crates/nu-protocol/src/value/dict.rs

@@ -1,270 +0,0 @@
-use crate::maybe_owned::MaybeOwned;
-use crate::value::primitive::Primitive;
-use crate::value::{UntaggedValue, Value};
-use derive_new::new;
-use getset::Getters;
-use indexmap::IndexMap;
-use nu_source::{DbgDocBldr, DebugDocBuilder, PrettyDebug, Spanned, SpannedItem, Tag};
-use serde::{Deserialize, Serialize};
-use std::cmp::{Ord, Ordering, PartialOrd};
-use std::hash::{Hash, Hasher};
-
-/// A dictionary that can hold a mapping from names to Values
-#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, Clone, Getters, new)]
-pub struct Dictionary {
-    #[get = "pub"]
-    pub entries: IndexMap<String, Value>,
-}
-
-#[allow(clippy::derive_hash_xor_eq)]
-impl Hash for Dictionary {
-    /// Create the hash function to allow the Hash trait for dictionaries
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let mut entries = self.entries.clone();
-        entries.sort_keys();
-        entries.keys().collect::<Vec<&String>>().hash(state);
-        entries.values().collect::<Vec<&Value>>().hash(state);
-    }
-}
-
-impl PartialOrd for Dictionary {
-    /// Compare two dictionaries for sort ordering
-    fn partial_cmp(&self, other: &Dictionary) -> Option<Ordering> {
-        let this: Vec<&String> = self.entries.keys().collect();
-        let that: Vec<&String> = other.entries.keys().collect();
-
-        if this != that {
-            return this.partial_cmp(&that);
-        }
-
-        let this: Vec<&Value> = self.entries.values().collect();
-        let that: Vec<&Value> = self.entries.values().collect();
-
-        this.partial_cmp(&that)
-    }
-}
-
-impl Ord for Dictionary {
-    /// Compare two dictionaries for ordering
-    fn cmp(&self, other: &Dictionary) -> Ordering {
-        let this: Vec<&String> = self.entries.keys().collect();
-        let that: Vec<&String> = other.entries.keys().collect();
-
-        if this != that {
-            return this.cmp(&that);
-        }
-
-        let this: Vec<&Value> = self.entries.values().collect();
-        let that: Vec<&Value> = self.entries.values().collect();
-
-        this.cmp(&that)
-    }
-}
-
-impl PartialEq<Value> for Dictionary {
-    /// Test a dictionary against a Value for equality
-    fn eq(&self, other: &Value) -> bool {
-        matches!(&other.value, UntaggedValue::Row(d) if self == d)
-    }
-}
-
-/// A key-value pair specifically meant to be used in debug and pretty-printing
-#[derive(Debug, new)]
-struct DebugEntry<'a> {
-    key: &'a str,
-    value: &'a Value,
-}
-
-impl<'a> PrettyDebug for DebugEntry<'a> {
-    /// Build the the information to pretty-print the DebugEntry
-    fn pretty(&self) -> DebugDocBuilder {
-        (DbgDocBldr::key(self.key.to_string())
-            + DbgDocBldr::equals()
-            + self.value.pretty().into_value())
-        .group()
-    }
-}
-
-impl PrettyDebug for Dictionary {
-    /// Get a Dictionary ready to be pretty-printed
-    fn pretty(&self) -> DebugDocBuilder {
-        DbgDocBldr::delimit(
-            "(",
-            DbgDocBldr::intersperse(
-                self.entries()
-                    .iter()
-                    .map(|(key, value)| DebugEntry::new(key, value)),
-                DbgDocBldr::space(),
-            ),
-            ")",
-        )
-    }
-}
-
-impl From<IndexMap<String, Value>> for Dictionary {
-    /// Create a dictionary from a map of strings to Values
-    fn from(input: IndexMap<String, Value>) -> Dictionary {
-        let mut out = IndexMap::default();
-
-        for (key, value) in input {
-            out.insert(key, value);
-        }
-
-        Dictionary::new(out)
-    }
-}
-
-impl Dictionary {
-    /// Find the matching Value for a given key, if possible. If not, return a Primitive::Nothing
-    pub fn get_data(&self, desc: &str) -> MaybeOwned<'_, Value> {
-        match self.entries.get(desc) {
-            Some(v) => MaybeOwned::Borrowed(v),
-            None => MaybeOwned::Owned(
-                UntaggedValue::Primitive(Primitive::Nothing).into_untagged_value(),
-            ),
-        }
-    }
-
-    pub fn insert(&mut self, key: String, value: Value) -> Option<Value> {
-        self.entries.insert_full(key, value).1
-    }
-
-    pub fn merge_from(&self, other: &Dictionary) -> Dictionary {
-        let mut obj = self.clone();
-
-        for column in other.keys() {
-            let key = column.clone();
-            let value_key = key.as_str();
-            let value_spanned_key = value_key.spanned_unknown();
-
-            let other_column = match other.get_data_by_key(value_spanned_key) {
-                Some(value) => value,
-                None => UntaggedValue::Primitive(Primitive::Nothing).into_untagged_value(),
-            };
-
-            obj.entries.insert(key, other_column);
-        }
-
-        obj
-    }
-
-    /// Iterate the keys in the Dictionary
-    pub fn keys(&self) -> impl Iterator<Item = &String> {
-        self.entries.keys()
-    }
-
-    /// Iterate the values in the Dictionary
-    pub fn values(&self) -> impl Iterator<Item = &Value> {
-        self.entries.values()
-    }
-
-    /// Checks if given key exists
-    pub fn contains_key(&self, key: &str) -> bool {
-        self.entries.contains_key(key)
-    }
-
-    /// Find the matching Value for a key, if possible
-    pub fn get_data_by_key(&self, name: Spanned<&str>) -> Option<Value> {
-        let result = self
-            .entries
-            .iter()
-            .find(|(desc_name, _)| *desc_name == name.item)?
-            .1;
-
-        Some(
-            result
-                .value
-                .clone()
-                .into_value(Tag::new(result.tag.anchor(), name.span)),
-        )
-    }
-
-    /// Get a mutable entry that matches a key, if possible
-    pub fn get_mut_data_by_key(&mut self, name: &str) -> Option<&mut Value> {
-        self.entries
-            .iter_mut()
-            .find(|(desc_name, _)| *desc_name == name)
-            .map(|(_, v)| v)
-    }
-
-    /// Insert a new key/value pair into the dictionary
-    pub fn insert_data_at_key(&mut self, name: &str, value: Value) {
-        self.entries.insert(name.to_string(), value);
-    }
-
-    /// Return size of dictionary
-    pub fn length(&self) -> usize {
-        self.entries.len()
-    }
-}
-
-/// A helper to help create dictionaries for you. It has the ability to insert values into the dictionary while maintaining the tags that need to be applied to the individual members
-#[derive(Debug, Clone)]
-pub struct TaggedDictBuilder {
-    tag: Tag,
-    dict: IndexMap<String, Value>,
-}
-
-impl TaggedDictBuilder {
-    /// Create a new builder
-    pub fn new(tag: impl Into<Tag>) -> TaggedDictBuilder {
-        TaggedDictBuilder {
-            tag: tag.into(),
-            dict: IndexMap::default(),
-        }
-    }
-
-    /// Build the contents of the builder into a Value
-    pub fn build(tag: impl Into<Tag>, block: impl FnOnce(&mut TaggedDictBuilder)) -> Value {
-        let mut builder = TaggedDictBuilder::new(tag);
-        block(&mut builder);
-        builder.into_value()
-    }
-
-    /// Create a new builder with a pre-defined capacity
-    pub fn with_capacity(tag: impl Into<Tag>, n: usize) -> TaggedDictBuilder {
-        TaggedDictBuilder {
-            tag: tag.into(),
-            dict: IndexMap::with_capacity(n),
-        }
-    }
-
-    /// Insert an untagged key/value pair into the dictionary, to later be tagged when built
-    pub fn insert_untagged(&mut self, key: impl Into<String>, value: impl Into<UntaggedValue>) {
-        self.dict
-            .insert(key.into(), value.into().into_value(&self.tag));
-    }
-
-    ///  Insert a key/value pair into the dictionary
-    pub fn insert_value(&mut self, key: impl Into<String>, value: impl Into<Value>) {
-        self.dict.insert(key.into(), value.into());
-    }
-
-    /// Convert the dictionary into a tagged Value using the original tag
-    pub fn into_value(self) -> Value {
-        let tag = self.tag.clone();
-        self.into_untagged_value().into_value(tag)
-    }
-
-    /// Convert the dictionary into an UntaggedValue
-    pub fn into_untagged_value(self) -> UntaggedValue {
-        UntaggedValue::Row(Dictionary { entries: self.dict })
-    }
-
-    /// Returns true if the dictionary is empty, false otherwise
-    pub fn is_empty(&self) -> bool {
-        self.dict.is_empty()
-    }
-
-    /// Checks if given key exists
-    pub fn contains_key(&self, key: &str) -> bool {
-        self.dict.contains_key(key)
-    }
-}
-
-impl From<TaggedDictBuilder> for Value {
-    /// Convert a builder into a tagged Value
-    fn from(input: TaggedDictBuilder) -> Value {
-        input.into_value()
-    }
-}

crates/nu-protocol/src/value/did_you_mean.rs

@@ -1,132 +0,0 @@
-use crate::Value;
-
-/// Prepares a list of "sounds like" matches (using edit distance) for the string you're trying to find
-pub fn did_you_mean(obj_source: &Value, field_tried: String) -> Option<Vec<String>> {
-    let possibilities = obj_source.data_descriptors();
-
-    let mut possible_matches: Vec<_> = possibilities
-        .into_iter()
-        .map(|word| {
-            let edit_distance = levenshtein_distance(&word, &field_tried);
-            (edit_distance, word)
-        })
-        .collect();
-
-    if !possible_matches.is_empty() {
-        possible_matches.sort();
-        let words_matched: Vec<String> = possible_matches.into_iter().map(|m| m.1).collect();
-        Some(words_matched)
-    } else {
-        None
-    }
-}
-
-// Borrowed from here https://github.com/wooorm/levenshtein-rs
-pub fn levenshtein_distance(a: &str, b: &str) -> usize {
-    let mut result = 0;
-
-    /* Shortcut optimizations / degenerate cases. */
-    if a == b {
-        return result;
-    }
-
-    let length_a = a.chars().count();
-    let length_b = b.chars().count();
-
-    if length_a == 0 {
-        return length_b;
-    }
-
-    if length_b == 0 {
-        return length_a;
-    }
-
-    /* Initialize the vector.
-     *
-     * This is why it’s fast, normally a matrix is used,
-     * here we use a single vector. */
-    let mut cache: Vec<usize> = (1..).take(length_a).collect();
-    let mut distance_a;
-    let mut distance_b;
-
-    /* Loop. */
-    for (index_b, code_b) in b.chars().enumerate() {
-        result = index_b;
-        distance_a = index_b;
-
-        for (index_a, code_a) in a.chars().enumerate() {
-            distance_b = if code_a == code_b {
-                distance_a
-            } else {
-                distance_a + 1
-            };
-
-            distance_a = cache[index_a];
-
-            result = if distance_a > result {
-                if distance_b > result {
-                    result + 1
-                } else {
-                    distance_b
-                }
-            } else if distance_b > distance_a {
-                distance_a + 1
-            } else {
-                distance_b
-            };
-
-            cache[index_a] = result;
-        }
-    }
-
-    result
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::UntaggedValue;
-    use indexmap::indexmap;
-    use nu_source::Tag;
-
-    #[test]
-    fn did_you_mean_returns_possible_column_matches() {
-        let value = UntaggedValue::row(indexmap! {
-           "dog".to_string() => UntaggedValue::int(1).into(),
-           "cat".to_string() => UntaggedValue::int(1).into(),
-           "alt".to_string() => UntaggedValue::int(1).into(),
-        });
-
-        let source = Value {
-            tag: Tag::unknown(),
-            value,
-        };
-
-        assert_eq!(
-            Some(vec![
-                "cat".to_string(),
-                "alt".to_string(),
-                "dog".to_string()
-            ]),
-            did_you_mean(&source, "hat".to_string())
-        )
-    }
-
-    #[test]
-    fn did_you_mean_returns_no_matches_when_empty() {
-        let empty_source = Value {
-            tag: Tag::unknown(),
-            value: UntaggedValue::row(indexmap! {}),
-        };
-
-        assert_eq!(None, did_you_mean(&empty_source, "hat".to_string()))
-    }
-
-    #[test]
-    fn test_levenshtein_distance() {
-        assert_eq!(super::levenshtein_distance("hello world", "hello world"), 0);
-        assert_eq!(super::levenshtein_distance("hello", "hello world"), 6);
-        assert_eq!(super::levenshtein_distance("°C", "°C"), 0);
-        assert_eq!(super::levenshtein_distance("°", "°C"), 1);
-    }
-}

crates/nu-protocol/src/value/from.rs

@@ -0,0 +1,25 @@
+use crate::{ShellError, Value};
+
+impl Value {
+    pub fn as_f64(&self) -> Result<f64, ShellError> {
+        match self {
+            Value::Float { val, .. } => Ok(*val),
+            x => Err(ShellError::CantConvert(
+                "f64".into(),
+                x.get_type().to_string(),
+                self.span()?,
+            )),
+        }
+    }
+
+    pub fn as_i64(&self) -> Result<i64, ShellError> {
+        match self {
+            Value::Int { val, .. } => Ok(*val),
+            x => Err(ShellError::CantConvert(
+                "rf64".into(),
+                x.get_type().to_string(),
+                self.span()?,
+            )),
+        }
+    }
+}

crates/nu-protocol/src/value/from_value.rs

@@ -0,0 +1,404 @@
+use std::path::PathBuf;
+use std::str::FromStr;
+
+use crate::ast::{CellPath, PathMember};
+use crate::engine::CaptureBlock;
+use crate::ShellError;
+use crate::{Range, Spanned, Value};
+use chrono::{DateTime, FixedOffset};
+
+pub trait FromValue: Sized {
+    fn from_value(v: &Value) -> Result<Self, ShellError>;
+}
+
+impl FromValue for Value {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        Ok(v.clone())
+    }
+}
+
+impl FromValue for Spanned<i64> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Int { val, span } => Ok(Spanned {
+                item: *val,
+                span: *span,
+            }),
+            Value::Filesize { val, span } => Ok(Spanned {
+                item: *val as i64,
+                span: *span,
+            }),
+            Value::Duration { val, span } => Ok(Spanned {
+                item: *val as i64,
+                span: *span,
+            }),
+
+            v => Err(ShellError::CantConvert(
+                "integer".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for i64 {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Int { val, .. } => Ok(*val),
+            Value::Filesize { val, .. } => Ok(*val as i64),
+            Value::Duration { val, .. } => Ok(*val as i64),
+
+            v => Err(ShellError::CantConvert(
+                "integer".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<f64> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Int { val, span } => Ok(Spanned {
+                item: *val as f64,
+                span: *span,
+            }),
+            Value::Float { val, span } => Ok(Spanned {
+                item: *val,
+                span: *span,
+            }),
+
+            v => Err(ShellError::CantConvert(
+                "float".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for f64 {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Float { val, .. } => Ok(*val),
+            Value::Int { val, .. } => Ok(*val as f64),
+            v => Err(ShellError::CantConvert(
+                "float".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<usize> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Int { val, span } => Ok(Spanned {
+                item: *val as usize,
+                span: *span,
+            }),
+            Value::Filesize { val, span } => Ok(Spanned {
+                item: *val as usize,
+                span: *span,
+            }),
+            Value::Duration { val, span } => Ok(Spanned {
+                item: *val as usize,
+                span: *span,
+            }),
+
+            v => Err(ShellError::CantConvert(
+                "integer".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for usize {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Int { val, .. } => Ok(*val as usize),
+            Value::Filesize { val, .. } => Ok(*val as usize),
+            Value::Duration { val, .. } => Ok(*val as usize),
+
+            v => Err(ShellError::CantConvert(
+                "integer".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for String {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        // FIXME: we may want to fail a little nicer here
+        match v {
+            Value::CellPath { val, .. } => Ok(val.into_string()),
+            Value::String { val, .. } => Ok(val.clone()),
+            v => Err(ShellError::CantConvert(
+                "string".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<String> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        Ok(Spanned {
+            item: match v {
+                Value::CellPath { val, .. } => val.into_string(),
+                Value::String { val, .. } => val.clone(),
+                v => {
+                    return Err(ShellError::CantConvert(
+                        "string".into(),
+                        v.get_type().to_string(),
+                        v.span()?,
+                    ))
+                }
+            },
+            span: v.span()?,
+        })
+    }
+}
+
+impl FromValue for Vec<String> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        // FIXME: we may want to fail a little nicer here
+        match v {
+            Value::List { vals, .. } => vals
+                .iter()
+                .map(|val| match val {
+                    Value::String { val, .. } => Ok(val.clone()),
+                    c => Err(ShellError::CantConvert(
+                        "string".into(),
+                        c.get_type().to_string(),
+                        c.span()?,
+                    )),
+                })
+                .collect::<Result<Vec<String>, ShellError>>(),
+            v => Err(ShellError::CantConvert(
+                "string".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for CellPath {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        let span = v.span()?;
+        match v {
+            Value::CellPath { val, .. } => Ok(val.clone()),
+            Value::String { val, .. } => Ok(CellPath {
+                members: vec![PathMember::String {
+                    val: val.clone(),
+                    span,
+                }],
+            }),
+            Value::Int { val, .. } => Ok(CellPath {
+                members: vec![PathMember::Int {
+                    val: *val as usize,
+                    span,
+                }],
+            }),
+            x => Err(ShellError::CantConvert(
+                "cell path".into(),
+                x.get_type().to_string(),
+                span,
+            )),
+        }
+    }
+}
+
+impl FromValue for bool {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Bool { val, .. } => Ok(*val),
+            v => Err(ShellError::CantConvert(
+                "bool".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<bool> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Bool { val, span } => Ok(Spanned {
+                item: *val,
+                span: *span,
+            }),
+            v => Err(ShellError::CantConvert(
+                "bool".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for DateTime<FixedOffset> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Date { val, .. } => Ok(*val),
+            v => Err(ShellError::CantConvert(
+                "date".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<DateTime<FixedOffset>> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Date { val, span } => Ok(Spanned {
+                item: *val,
+                span: *span,
+            }),
+            v => Err(ShellError::CantConvert(
+                "date".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Range {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Range { val, .. } => Ok((**val).clone()),
+            v => Err(ShellError::CantConvert(
+                "range".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<Range> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Range { val, span } => Ok(Spanned {
+                item: (**val).clone(),
+                span: *span,
+            }),
+            v => Err(ShellError::CantConvert(
+                "range".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Vec<u8> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Binary { val, .. } => Ok(val.clone()),
+            Value::String { val, .. } => Ok(val.bytes().collect()),
+            v => Err(ShellError::CantConvert(
+                "binary data".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<PathBuf> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::String { val, span } => Ok(Spanned {
+                item: PathBuf::from_str(val)
+                    .map_err(|err| ShellError::FileNotFoundCustom(err.to_string(), *span))?,
+                span: *span,
+            }),
+            v => Err(ShellError::CantConvert(
+                "range".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Vec<Value> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        // FIXME: we may want to fail a little nicer here
+        match v {
+            Value::List { vals, .. } => Ok(vals.clone()),
+            v => Err(ShellError::CantConvert(
+                "Vector of values".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+// A record
+impl FromValue for (Vec<String>, Vec<Value>) {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Record { cols, vals, .. } => Ok((cols.clone(), vals.clone())),
+            v => Err(ShellError::CantConvert(
+                "Record".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for CaptureBlock {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Block { val, captures, .. } => Ok(CaptureBlock {
+                block_id: *val,
+                captures: captures.clone(),
+            }),
+            v => Err(ShellError::CantConvert(
+                "Block".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}
+
+impl FromValue for Spanned<CaptureBlock> {
+    fn from_value(v: &Value) -> Result<Self, ShellError> {
+        match v {
+            Value::Block {
+                val,
+                captures,
+                span,
+            } => Ok(Spanned {
+                item: CaptureBlock {
+                    block_id: *val,
+                    captures: captures.clone(),
+                },
+                span: *span,
+            }),
+            v => Err(ShellError::CantConvert(
+                "Block".into(),
+                v.get_type().to_string(),
+                v.span()?,
+            )),
+        }
+    }
+}

crates/nu-protocol/src/value/iter.rs

@@ -1,52 +0,0 @@
-use crate::value::{UntaggedValue, Value};
-
-#[derive(Debug)]
-pub enum RowValueIter<'a> {
-    Empty,
-    Entries(indexmap::map::Iter<'a, String, Value>),
-}
-
-#[derive(Debug)]
-pub enum TableValueIter<'a> {
-    Empty,
-    Entries(std::slice::Iter<'a, Value>),
-}
-
-impl<'a> Iterator for RowValueIter<'a> {
-    type Item = (&'a String, &'a Value);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self {
-            RowValueIter::Empty => None,
-            RowValueIter::Entries(iter) => iter.next(),
-        }
-    }
-}
-
-impl<'a> Iterator for TableValueIter<'a> {
-    type Item = &'a Value;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self {
-            TableValueIter::Empty => None,
-            TableValueIter::Entries(iter) => iter.next(),
-        }
-    }
-}
-
-pub fn table_entries(value: &Value) -> TableValueIter<'_> {
-    match &value.value {
-        UntaggedValue::Table(t) => TableValueIter::Entries(t.iter()),
-        _ => TableValueIter::Empty,
-    }
-}
-
-pub fn row_entries(value: &Value) -> RowValueIter<'_> {
-    match &value.value {
-        UntaggedValue::Row(o) => {
-            let iter = o.entries.iter();
-            RowValueIter::Entries(iter)
-        }
-        _ => RowValueIter::Empty,
-    }
-}

crates/nu-protocol/src/value/primitive.rs

@@ -1,599 +0,0 @@
-use crate::type_name::ShellTypeName;
-use crate::value::column_path::ColumnPath;
-use crate::value::range::{Range, RangeInclusion};
-use crate::value::{serde_bigdecimal, serde_bigint};
-use bigdecimal::BigDecimal;
-use chrono::{DateTime, FixedOffset};
-use chrono_humanize::HumanTime;
-use nu_errors::{ExpectedRange, ShellError};
-use nu_source::{PrettyDebug, Span, SpannedItem};
-use num_bigint::BigInt;
-use num_integer::Integer;
-use num_traits::cast::{FromPrimitive, ToPrimitive};
-use num_traits::identities::Zero;
-use num_traits::sign::Signed;
-use serde::{Deserialize, Serialize};
-use std::path::PathBuf;
-
-const NANOS_PER_SEC: u32 = 1_000_000_000;
-
-/// The most fundamental of structured values in Nu are the Primitive values. These values represent types like integers, strings, booleans, dates, etc
-/// that are then used as the building blocks of more complex structures.
-///
-/// Primitives also include marker values BeginningOfStream and EndOfStream which denote a change of condition in the stream
-#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Deserialize, Serialize)]
-pub enum Primitive {
-    /// An empty value
-    Nothing,
-    /// A common integer
-    Int(i64),
-    /// A "big int", an integer with arbitrarily large size (aka not limited to 64-bit)
-    #[serde(with = "serde_bigint")]
-    BigInt(BigInt),
-    /// A "big decimal", an decimal number with arbitrarily large size (aka not limited to 64-bit)
-    #[serde(with = "serde_bigdecimal")]
-    Decimal(BigDecimal),
-    /// A count in the number of bytes, used as a filesize
-    Filesize(u64),
-    /// A string value
-    String(String),
-    /// A path to travel to reach a value in a table
-    ColumnPath(ColumnPath),
-    /// A glob pattern, eg foo*
-    GlobPattern(String),
-    /// A boolean value
-    Boolean(bool),
-    /// A date value
-    Date(DateTime<FixedOffset>),
-    /// A count in the number of nanoseconds
-    #[serde(with = "serde_bigint")]
-    Duration(BigInt),
-    /// A range of values
-    Range(Box<Range>),
-    /// A file path
-    FilePath(PathBuf),
-    /// A vector of raw binary data
-    #[serde(with = "serde_bytes")]
-    Binary(Vec<u8>),
-
-    /// Beginning of stream marker, a pseudo-value not intended for tables
-    BeginningOfStream,
-    /// End of stream marker, a pseudo-value not intended for tables
-    EndOfStream,
-}
-
-impl Primitive {
-    /// Converts a primitive value to a char, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_char(&self, span: Span) -> Result<char, ShellError> {
-        match self {
-            Primitive::String(s) => {
-                if s.len() > 1 {
-                    return Err(ShellError::type_error(
-                        "char",
-                        self.type_name().spanned(span),
-                    ));
-                }
-                s.chars()
-                    .next()
-                    .ok_or_else(|| ShellError::type_error("char", self.type_name().spanned(span)))
-            }
-            other => Err(ShellError::type_error(
-                "char",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Converts a primitive value to a u64, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_usize(&self, span: Span) -> Result<usize, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_usize().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U64,
-                    &int.to_string().spanned(span),
-                    "converting an integer into an unsigned 64-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_usize().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U64,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into an unsigned 64-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Converts a primitive value to a u64, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_u64(&self, span: Span) -> Result<u64, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_u64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U64,
-                    &int.to_string().spanned(span),
-                    "converting an integer into an unsigned 64-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_u64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U64,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into an unsigned 64-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Converts a primitive value to a f64, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_f64(&self, span: Span) -> Result<f64, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_f64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::F64,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a 64-bit floating point",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_f64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::F64,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a 64-bit floating point",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Converts a primitive value to a i64, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_i64(&self, span: Span) -> Result<i64, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_i64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I64,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a signed 64-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_i64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I64,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a signed 64-bit integer",
-                )
-            }),
-            Primitive::Duration(duration) => duration.to_i64().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I64,
-                    &duration.to_string().spanned(span),
-                    "converting a duration into a signed 64-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Converts a primitive value to a u32, if possible. Uses a span to build an error if the conversion isn't possible.
-    pub fn as_u32(&self, span: Span) -> Result<u32, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_u32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U32,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a unsigned 32-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_u32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::U32,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a unsigned 32-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    pub fn as_i32(&self, span: Span) -> Result<i32, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_i32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I32,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a signed 32-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_i32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I32,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a signed 32-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    pub fn as_i16(&self, span: Span) -> Result<i16, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_i16().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I16,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a signed 16-bit integer",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_i16().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::I16,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a signed 16-bit integer",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    pub fn as_f32(&self, span: Span) -> Result<f32, ShellError> {
-        match self {
-            Primitive::Int(int) => int.to_f32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::F32,
-                    &int.to_string().spanned(span),
-                    "converting an integer into a signed 32-bit float",
-                )
-            }),
-            Primitive::Decimal(decimal) => decimal.to_f32().ok_or_else(|| {
-                ShellError::range_error(
-                    ExpectedRange::F32,
-                    &decimal.to_string().spanned(span),
-                    "converting a decimal into a signed 32-bit float",
-                )
-            }),
-            other => Err(ShellError::type_error(
-                "number",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    // FIXME: This is a bad name, but no other way to differentiate with our own Duration.
-    pub fn into_chrono_duration(self, span: Span) -> Result<chrono::Duration, ShellError> {
-        match self {
-            Primitive::Duration(duration) => {
-                // Divide into seconds because BigInt can be larger than i64
-                let (secs, nanos) = duration.div_rem(
-                    &BigInt::from_u32(NANOS_PER_SEC)
-                        .expect("Internal error: conversion from u32 failed"),
-                );
-                let secs = match secs.to_i64() {
-                    //The duration crate doesn't accept seconds bigger than i64::MAX / 1000
-                    Some(secs) => match secs.checked_mul(1000) {
-                        Some(_) => secs,
-                        None => {
-                            return Err(ShellError::labeled_error(
-                                "Internal duration conversion overflow.",
-                                "duration overflow",
-                                span,
-                            ))
-                        }
-                    },
-                    None => {
-                        return Err(ShellError::labeled_error(
-                            "Internal duration conversion overflow.",
-                            "duration overflow",
-                            span,
-                        ))
-                    }
-                };
-                // This should never fail since NANOS_PER_SEC won't overflow
-                let nanos = nanos.to_i64().expect("Unexpected i64 overflow");
-                // This should also never fail since we are adding less than NANOS_PER_SEC.
-                chrono::Duration::seconds(secs)
-                    .checked_add(&chrono::Duration::nanoseconds(nanos))
-                    .ok_or_else(|| ShellError::unexpected("Unexpected duration overflow"))
-            }
-            other => Err(ShellError::type_error(
-                "duration",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    pub fn into_string(self, span: Span) -> Result<String, ShellError> {
-        match self {
-            Primitive::String(s) => Ok(s),
-            other => Err(ShellError::type_error(
-                "string",
-                other.type_name().spanned(span),
-            )),
-        }
-    }
-
-    /// Returns true if the value is empty
-    pub fn is_empty(&self) -> bool {
-        match self {
-            Primitive::Nothing => true,
-            Primitive::String(s) => s.is_empty(),
-            _ => false,
-        }
-    }
-}
-
-impl From<bool> for Primitive {
-    /// Helper to convert from boolean to a primitive
-    fn from(b: bool) -> Primitive {
-        Primitive::Boolean(b)
-    }
-}
-
-impl From<&str> for Primitive {
-    /// Helper to convert from string slices to a primitive
-    fn from(s: &str) -> Primitive {
-        Primitive::String(s.to_string())
-    }
-}
-
-impl From<String> for Primitive {
-    /// Helper to convert from Strings to a primitive
-    fn from(s: String) -> Primitive {
-        Primitive::String(s)
-    }
-}
-
-impl From<BigDecimal> for Primitive {
-    /// Helper to convert from decimals to a Primitive value
-    fn from(decimal: BigDecimal) -> Primitive {
-        Primitive::Decimal(decimal)
-    }
-}
-
-impl From<BigInt> for Primitive {
-    /// Helper to convert from integers to a Primitive value
-    fn from(int: BigInt) -> Primitive {
-        Primitive::BigInt(int)
-    }
-}
-
-// Macro to define the From trait for native types to primitives
-// The from trait requires a converter that will be applied to the
-// native type.
-macro_rules! from_native_to_primitive {
-    ($native_type:ty, $primitive_type:expr, $converter: expr) => {
-        // e.g. from u32 -> Primitive
-        impl From<$native_type> for Primitive {
-            fn from(value: $native_type) -> Primitive {
-                if let Some(i) = $converter(value) {
-                    $primitive_type(i)
-                } else {
-                    unreachable!("Internal error: protocol did not use compatible decimal")
-                }
-            }
-        }
-    };
-}
-
-from_native_to_primitive!(i8, Primitive::Int, i64::from_i8);
-from_native_to_primitive!(i16, Primitive::Int, i64::from_i16);
-from_native_to_primitive!(i32, Primitive::Int, i64::from_i32);
-from_native_to_primitive!(i64, Primitive::Int, i64::from_i64);
-from_native_to_primitive!(u8, Primitive::Int, i64::from_u8);
-from_native_to_primitive!(u16, Primitive::Int, i64::from_u16);
-from_native_to_primitive!(u32, Primitive::Int, i64::from_u32);
-from_native_to_primitive!(u64, Primitive::BigInt, BigInt::from_u64);
-from_native_to_primitive!(f32, Primitive::Decimal, BigDecimal::from_f32);
-from_native_to_primitive!(f64, Primitive::Decimal, BigDecimal::from_f64);
-
-impl From<chrono::Duration> for Primitive {
-    fn from(duration: chrono::Duration) -> Primitive {
-        // FIXME: This is a hack since chrono::Duration does not give access to its 'nanos' field.
-        let secs: i64 = duration.num_seconds();
-        // This will never fail.
-        let nanos: u32 = duration
-            .checked_sub(&chrono::Duration::seconds(secs))
-            .expect("Unexpected overflow")
-            .num_nanoseconds()
-            .expect("Unexpected overflow") as u32;
-        Primitive::Duration(
-            BigInt::from_i64(secs * NANOS_PER_SEC as i64 + nanos as i64)
-                .expect("Internal error: can't convert from i64"),
-        )
-    }
-}
-
-impl std::fmt::Display for Primitive {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "{:?}", self)
-    }
-}
-
-impl ShellTypeName for Primitive {
-    /// Get the name of the type of a Primitive value
-    fn type_name(&self) -> &'static str {
-        match self {
-            Primitive::Nothing => "nothing",
-            Primitive::Int(_) => "integer",
-            Primitive::BigInt(_) => "big integer",
-            Primitive::Range(_) => "range",
-            Primitive::Decimal(_) => "decimal",
-            Primitive::Filesize(_) => "filesize(in bytes)",
-            Primitive::String(_) => "string",
-            Primitive::ColumnPath(_) => "column path",
-            Primitive::GlobPattern(_) => "pattern",
-            Primitive::Boolean(_) => "boolean",
-            Primitive::Date(_) => "date",
-            Primitive::Duration(_) => "duration",
-            Primitive::FilePath(_) => "file path",
-            Primitive::Binary(_) => "binary",
-            Primitive::BeginningOfStream => "marker<beginning of stream>",
-            Primitive::EndOfStream => "marker<end of stream>",
-        }
-    }
-}
-
-/// Format a Primitive value into a string
-pub fn format_primitive(primitive: &Primitive, field_name: Option<&String>) -> String {
-    match primitive {
-        Primitive::Nothing => String::new(),
-        Primitive::BeginningOfStream => String::new(),
-        Primitive::EndOfStream => String::new(),
-        Primitive::FilePath(p) => p.display().to_string(),
-        Primitive::Filesize(num_bytes) => {
-            if let Some(value) = num_bytes.to_u128() {
-                let byte = byte_unit::Byte::from_bytes(value);
-
-                if byte.get_bytes() == 0u128 {
-                    return "—".to_string();
-                }
-
-                let byte = byte.get_appropriate_unit(false);
-
-                match byte.get_unit() {
-                    byte_unit::ByteUnit::B => format!("{} B ", byte.get_value()),
-                    _ => byte.format(1),
-                }
-            } else {
-                format!("{} B", num_bytes)
-            }
-        }
-        Primitive::Duration(duration) => format_duration(duration),
-        Primitive::Int(i) => i.to_string(),
-        Primitive::BigInt(i) => i.to_string(),
-        Primitive::Decimal(decimal) => {
-            // TODO: We should really pass the precision in here instead of hard coding it
-            let decimal_string = decimal.to_string();
-            let decimal_places: Vec<&str> = decimal_string.split('.').collect();
-            if decimal_places.len() == 2 && decimal_places[1].len() > 4 {
-                format!("{:.4}", decimal)
-            } else {
-                decimal.to_string()
-            }
-        }
-        Primitive::Range(range) => format!(
-            "{}..{}{}",
-            format_primitive(&range.from.0.item, None),
-            if range.to.1 == RangeInclusion::Exclusive {
-                "<"
-            } else {
-                ""
-            },
-            format_primitive(&range.to.0.item, None)
-        ),
-        Primitive::GlobPattern(s) => s.to_string(),
-        Primitive::String(s) => s.to_owned(),
-        Primitive::ColumnPath(p) => {
-            let mut members = p.iter();
-            let mut f = String::new();
-
-            f.push_str(
-                &members
-                    .next()
-                    .expect("BUG: column path with zero members")
-                    .display(),
-            );
-
-            for member in members {
-                f.push('.');
-                f.push_str(&member.display())
-            }
-
-            f
-        }
-        Primitive::Boolean(b) => match (b, field_name) {
-            (true, None) => "true",
-            (false, None) => "false",
-            (true, Some(s)) if !s.is_empty() => s,
-            (false, Some(s)) if !s.is_empty() => "",
-            (true, Some(_)) => "true",
-            (false, Some(_)) => "false",
-        }
-        .to_owned(),
-        Primitive::Binary(_) => "<binary>".to_owned(),
-        Primitive::Date(d) => format_date(d),
-    }
-}
-
-/// Format a duration in nanoseconds into a string
-pub fn format_duration(duration: &BigInt) -> String {
-    let is_zero = duration.is_zero();
-    // FIXME: This involves a lot of allocation, but it seems inevitable with BigInt.
-    let big_int_1000 = BigInt::from(1000);
-    let big_int_60 = BigInt::from(60);
-    let big_int_24 = BigInt::from(24);
-    // We only want the biggest subdivision to have the negative sign.
-    let (sign, duration) = if duration.is_zero() || duration.is_positive() {
-        (1, duration.clone())
-    } else {
-        (-1, -duration)
-    };
-    let (micros, nanos): (BigInt, BigInt) = duration.div_rem(&big_int_1000);
-    let (millis, micros): (BigInt, BigInt) = micros.div_rem(&big_int_1000);
-    let (secs, millis): (BigInt, BigInt) = millis.div_rem(&big_int_1000);
-    let (mins, secs): (BigInt, BigInt) = secs.div_rem(&big_int_60);
-    let (hours, mins): (BigInt, BigInt) = mins.div_rem(&big_int_60);
-    let (days, hours): (BigInt, BigInt) = hours.div_rem(&big_int_24);
-
-    let mut output_prep = vec![];
-
-    if !days.is_zero() {
-        output_prep.push(format!("{}day", days));
-    }
-
-    if !hours.is_zero() {
-        output_prep.push(format!("{}hr", hours));
-    }
-
-    if !mins.is_zero() {
-        output_prep.push(format!("{}min", mins));
-    }
-    // output 0sec for zero duration
-    if is_zero || !secs.is_zero() {
-        output_prep.push(format!("{}sec", secs));
-    }
-
-    if !millis.is_zero() {
-        output_prep.push(format!("{}ms", millis));
-    }
-
-    if !micros.is_zero() {
-        output_prep.push(format!("{}us", micros));
-    }
-
-    if !nanos.is_zero() {
-        output_prep.push(format!("{}ns", nanos));
-    }
-
-    format!(
-        "{}{}",
-        if sign == -1 { "-" } else { "" },
-        output_prep.join(" ")
-    )
-}
-
-/// Format a date value into a humanized string (eg "1 week ago" instead of a formal date string)
-pub fn format_date(d: &DateTime<FixedOffset>) -> String {
-    HumanTime::from(*d).to_string()
-}

crates/nu-protocol/src/value/range.rs

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 use crate::value::Primitive;
 use derive_new::new;
 use nu_errors::ShellError;
@@ -150,5 +151,225 @@ impl Range {
         }
 
         // How would inclusive vs. exclusive range work here?
+=======
+use serde::{Deserialize, Serialize};
+use std::cmp::Ordering;
+
+/// A Range is an iterator over integers.
+use crate::{
+    ast::{RangeInclusion, RangeOperator},
+    *,
+};
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Range {
+    pub from: Value,
+    pub incr: Value,
+    pub to: Value,
+    pub inclusion: RangeInclusion,
+}
+
+impl Range {
+    pub fn new(
+        expr_span: Span,
+        from: Value,
+        next: Value,
+        to: Value,
+        operator: &RangeOperator,
+    ) -> Result<Range, ShellError> {
+        // Select from & to values if they're not specified
+        // TODO: Replace the placeholder values with proper min/max for range based on data type
+        let from = if let Value::Nothing { .. } = from {
+            Value::Int {
+                val: 0i64,
+                span: expr_span,
+            }
+        } else {
+            from
+        };
+
+        let to = if let Value::Nothing { .. } = to {
+            if let Ok(Value::Bool { val: true, .. }) = next.lt(expr_span, &from) {
+                Value::Int {
+                    val: -100i64,
+                    span: expr_span,
+                }
+            } else {
+                Value::Int {
+                    val: 100i64,
+                    span: expr_span,
+                }
+            }
+        } else {
+            to
+        };
+
+        // Check if the range counts up or down
+        let moves_up = matches!(from.lte(expr_span, &to), Ok(Value::Bool { val: true, .. }));
+
+        // Convert the next value into the inctement
+        let incr = if let Value::Nothing { .. } = next {
+            if moves_up {
+                Value::Int {
+                    val: 1i64,
+                    span: expr_span,
+                }
+            } else {
+                Value::Int {
+                    val: -1i64,
+                    span: expr_span,
+                }
+            }
+        } else {
+            next.sub(operator.next_op_span, &from)?
+        };
+
+        let zero = Value::Int {
+            val: 0i64,
+            span: expr_span,
+        };
+
+        // Increment must be non-zero, otherwise we iterate forever
+        if matches!(incr.eq(expr_span, &zero), Ok(Value::Bool { val: true, .. })) {
+            return Err(ShellError::CannotCreateRange(expr_span));
+        }
+
+        // If to > from, then incr > 0, otherwise we iterate forever
+        if let (Value::Bool { val: true, .. }, Value::Bool { val: false, .. }) = (
+            to.gt(operator.span, &from)?,
+            incr.gt(operator.next_op_span, &zero)?,
+        ) {
+            return Err(ShellError::CannotCreateRange(expr_span));
+        }
+
+        // If to < from, then incr < 0, otherwise we iterate forever
+        if let (Value::Bool { val: true, .. }, Value::Bool { val: false, .. }) = (
+            to.lt(operator.span, &from)?,
+            incr.lt(operator.next_op_span, &zero)?,
+        ) {
+            return Err(ShellError::CannotCreateRange(expr_span));
+        }
+
+        Ok(Range {
+            from,
+            incr,
+            to,
+            inclusion: operator.inclusion,
+        })
+    }
+
+    #[inline]
+    fn moves_up(&self) -> bool {
+        self.from <= self.to
+    }
+
+    #[inline]
+    fn is_end_inclusive(&self) -> bool {
+        matches!(self.inclusion, RangeInclusion::Inclusive)
+    }
+
+    pub fn contains(&self, item: &Value) -> bool {
+        match (item.partial_cmp(&self.from), item.partial_cmp(&self.to)) {
+            (Some(Ordering::Greater | Ordering::Equal), Some(Ordering::Less)) => self.moves_up(),
+            (Some(Ordering::Less | Ordering::Equal), Some(Ordering::Greater)) => !self.moves_up(),
+            (Some(_), Some(Ordering::Equal)) => self.is_end_inclusive(),
+            (_, _) => false,
+        }
+    }
+
+    pub fn into_range_iter(self) -> Result<RangeIterator, ShellError> {
+        let span = self.from.span()?;
+
+        Ok(RangeIterator::new(self, span))
+    }
+}
+
+pub struct RangeIterator {
+    curr: Value,
+    end: Value,
+    span: Span,
+    is_end_inclusive: bool,
+    moves_up: bool,
+    incr: Value,
+    done: bool,
+}
+
+impl RangeIterator {
+    pub fn new(range: Range, span: Span) -> RangeIterator {
+        let moves_up = range.moves_up();
+        let is_end_inclusive = range.is_end_inclusive();
+
+        let start = match range.from {
+            Value::Nothing { .. } => Value::Int { val: 0, span },
+            x => x,
+        };
+
+        let end = match range.to {
+            Value::Nothing { .. } => Value::Int {
+                val: i64::MAX,
+                span,
+            },
+            x => x,
+        };
+
+        RangeIterator {
+            moves_up,
+            curr: start,
+            end,
+            span,
+            is_end_inclusive,
+            done: false,
+            incr: range.incr,
+        }
+    }
+}
+
+impl Iterator for RangeIterator {
+    type Item = Value;
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.done {
+            return None;
+        }
+
+        let ordering = if matches!(self.end, Value::Nothing { .. }) {
+            Some(Ordering::Less)
+        } else {
+            self.curr.partial_cmp(&self.end)
+        };
+
+        let ordering = if let Some(ord) = ordering {
+            ord
+        } else {
+            self.done = true;
+            return Some(Value::Error {
+                error: ShellError::CannotCreateRange(self.span),
+            });
+        };
+
+        let desired_ordering = if self.moves_up {
+            Ordering::Less
+        } else {
+            Ordering::Greater
+        };
+
+        if (ordering == desired_ordering) || (self.is_end_inclusive && ordering == Ordering::Equal)
+        {
+            let next_value = self.curr.add(self.span, &self.incr);
+
+            let mut next = match next_value {
+                Ok(result) => result,
+
+                Err(error) => {
+                    self.done = true;
+                    return Some(Value::Error { error });
+                }
+            };
+            std::mem::swap(&mut self.curr, &mut next);
+
+            Some(next)
+        } else {
+            None
+        }
+>>>>>>> 9259a56a28f1dd3a4b720ad815aa19c6eaf6adce
     }
 }

crates/nu-protocol/src/value/serde_bigdecimal.rs

@@ -1,19 +0,0 @@
-use bigdecimal::BigDecimal;
-
-/// Enable big decimal serialization by providing a `serialize` function
-pub fn serialize<S>(big_decimal: &BigDecimal, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: serde::Serializer,
-{
-    serde::Serialize::serialize(&big_decimal.to_string(), serializer)
-}
-
-/// Enable big decimal deserialization by providing a `deserialize` function
-pub fn deserialize<'de, D>(deserializer: D) -> Result<BigDecimal, D::Error>
-where
-    D: serde::Deserializer<'de>,
-{
-    let x: String = serde::Deserialize::deserialize(deserializer)?;
-    BigDecimal::parse_bytes(x.as_bytes(), 10)
-        .ok_or_else(|| serde::de::Error::custom("expected a bigdecimal"))
-}

crates/nu-protocol/src/value/serde_bigint.rs

@@ -1,20 +0,0 @@
-use num_bigint::BigInt;
-
-/// Enable big int serialization by providing a `serialize` function
-pub fn serialize<S>(big_int: &BigInt, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: serde::Serializer,
-{
-    serde::Serialize::serialize(&big_int.to_string(), serializer)
-}
-
-/// Enable big int deserialization by providing a `deserialize` function
-pub fn deserialize<'de, D>(deserializer: D) -> Result<BigInt, D::Error>
-where
-    D: serde::Deserializer<'de>,
-{
-    let x: String = serde::Deserialize::deserialize(deserializer)?;
-
-    BigInt::parse_bytes(x.as_bytes(), 10)
-        .ok_or_else(|| serde::de::Error::custom("expected a bignum"))
-}

crates/nu-protocol/src/value/stream.rs

@@ -0,0 +1,204 @@
+use crate::*;
+use std::{
+    fmt::Debug,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+};
+
+pub struct RawStream {
+    pub stream: Box<dyn Iterator<Item = Result<Vec<u8>, ShellError>> + Send + 'static>,
+    pub leftover: Vec<u8>,
+    pub ctrlc: Option<Arc<AtomicBool>>,
+    pub is_binary: bool,
+    pub span: Span,
+}
+
+impl RawStream {
+    pub fn new(
+        stream: Box<dyn Iterator<Item = Result<Vec<u8>, ShellError>> + Send + 'static>,
+        ctrlc: Option<Arc<AtomicBool>>,
+        span: Span,
+    ) -> Self {
+        Self {
+            stream,
+            leftover: vec![],
+            ctrlc,
+            is_binary: false,
+            span,
+        }
+    }
+
+    pub fn into_bytes(self) -> Result<Spanned<Vec<u8>>, ShellError> {
+        let mut output = vec![];
+
+        for item in self.stream {
+            output.extend(item?);
+        }
+
+        Ok(Spanned {
+            item: output,
+            span: self.span,
+        })
+    }
+
+    pub fn into_string(self) -> Result<Spanned<String>, ShellError> {
+        let mut output = String::new();
+        let span = self.span;
+
+        for item in self {
+            output.push_str(&item?.as_string()?);
+        }
+
+        Ok(Spanned { item: output, span })
+    }
+}
+impl Debug for RawStream {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RawStream").finish()
+    }
+}
+impl Iterator for RawStream {
+    type Item = Result<Value, ShellError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // If we know we're already binary, just output that
+        if self.is_binary {
+            match self.stream.next() {
+                Some(buffer) => match buffer {
+                    Ok(mut v) => {
+                        if !self.leftover.is_empty() {
+                            while let Some(b) = self.leftover.pop() {
+                                v.insert(0, b);
+                            }
+                        }
+                        Some(Ok(Value::Binary {
+                            val: v,
+                            span: self.span,
+                        }))
+                    }
+                    Err(e) => Some(Err(e)),
+                },
+                None => None,
+            }
+        } else {
+            // We *may* be text. We're only going to try utf-8. Other decodings
+            // needs to be taken as binary first, then passed through `decode`.
+            match self.stream.next() {
+                Some(buffer) => match buffer {
+                    Ok(mut v) => {
+                        if !self.leftover.is_empty() {
+                            while let Some(b) = self.leftover.pop() {
+                                v.insert(0, b);
+                            }
+                        }
+
+                        match String::from_utf8(v.clone()) {
+                            Ok(s) => {
+                                // Great, we have a complete string, let's output it
+                                Some(Ok(Value::String {
+                                    val: s,
+                                    span: self.span,
+                                }))
+                            }
+                            Err(err) => {
+                                // Okay, we *might* have a string but we've also got some errors
+                                if v.is_empty() {
+                                    // We can just end here
+                                    None
+                                } else if v.len() > 3
+                                    && (v.len() - err.utf8_error().valid_up_to() > 3)
+                                {
+                                    // As UTF-8 characters are max 4 bytes, if we have more than that in error we know
+                                    // that it's not just a character spanning two frames.
+                                    // We now know we are definitely binary, so switch to binary and stay there.
+                                    self.is_binary = true;
+                                    Some(Ok(Value::Binary {
+                                        val: v,
+                                        span: self.span,
+                                    }))
+                                } else {
+                                    // Okay, we have a tiny bit of error at the end of the buffer. This could very well be
+                                    // a character that spans two frames. Since this is the case, remove the error from
+                                    // the current frame an dput it in the leftover buffer.
+                                    self.leftover = v[err.utf8_error().valid_up_to()..].to_vec();
+
+                                    let buf = v[0..err.utf8_error().valid_up_to()].to_vec();
+
+                                    match String::from_utf8(buf) {
+                                        Ok(s) => Some(Ok(Value::String {
+                                            val: s,
+                                            span: self.span,
+                                        })),
+                                        Err(_) => {
+                                            // Something is definitely wrong. Switch to binary, and stay there
+                                            self.is_binary = true;
+                                            Some(Ok(Value::Binary {
+                                                val: v,
+                                                span: self.span,
+                                            }))
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    Err(e) => Some(Err(e)),
+                },
+                None => None,
+            }
+        }
+    }
+}
+
+/// A potentially infinite stream of values, optinally with a mean to send a Ctrl-C signal to stop
+/// the stream from continuing.
+///
+/// In practice, a "stream" here means anything which can be iterated and produce Values as it iterates.
+/// Like other iterators in Rust, observing values from this stream will drain the items as you view them
+/// and the stream cannot be replayed.
+pub struct ListStream {
+    pub stream: Box<dyn Iterator<Item = Value> + Send + 'static>,
+    pub ctrlc: Option<Arc<AtomicBool>>,
+}
+
+impl ListStream {
+    pub fn into_string(self, separator: &str, config: &Config) -> String {
+        self.map(|x: Value| x.into_string(", ", config))
+            .collect::<Vec<String>>()
+            .join(separator)
+    }
+
+    pub fn from_stream(
+        input: impl Iterator<Item = Value> + Send + 'static,
+        ctrlc: Option<Arc<AtomicBool>>,
+    ) -> ListStream {
+        ListStream {
+            stream: Box::new(input),
+            ctrlc,
+        }
+    }
+}
+
+impl Debug for ListStream {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ValueStream").finish()
+    }
+}
+
+impl Iterator for ListStream {
+    type Item = Value;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(ctrlc) = &self.ctrlc {
+            if ctrlc.load(Ordering::SeqCst) {
+                None
+            } else {
+                self.stream.next()
+            }
+        } else {
+            self.stream.next()
+        }
+    }
+}

crates/nu-protocol/src/value/unit.rs

@@ -0,0 +1,27 @@
+#[derive(Debug, Clone, Copy)]
+pub enum Unit {
+    // Filesize units: metric
+    Byte,
+    Kilobyte,
+    Megabyte,
+    Gigabyte,
+    Terabyte,
+    Petabyte,
+
+    // Filesize units: ISO/IEC 80000
+    Kibibyte,
+    Mebibyte,
+    Gibibyte,
+    Tebibyte,
+    Pebibyte,
+
+    // Duration units
+    Nanosecond,
+    Microsecond,
+    Millisecond,
+    Second,
+    Minute,
+    Hour,
+    Day,
+    Week,
+}

crates/nu-protocol/src/value/value_structure.rs

@@ -1,83 +0,0 @@
-use crate::{UntaggedValue, Value};
-use nu_errors::ShellError;
-use std::path::{Component, Path, PathBuf};
-
-fn is_value_tagged_dir(value: &Value) -> bool {
-    matches!(
-        &value.value,
-        UntaggedValue::Row(_) | UntaggedValue::Table(_)
-    )
-}
-
-#[derive(Debug, Eq, Ord, PartialEq, PartialOrd)]
-pub struct ValueResource {
-    pub at: usize,
-    pub loc: PathBuf,
-}
-
-impl ValueResource {}
-
-#[derive(Default, Debug)]
-pub struct ValueStructure {
-    pub resources: Vec<ValueResource>,
-}
-
-impl ValueStructure {
-    pub fn new() -> ValueStructure {
-        ValueStructure {
-            resources: Vec::<ValueResource>::new(),
-        }
-    }
-
-    pub fn exists(&self, path: &Path) -> bool {
-        if path == Path::new("/") {
-            return true;
-        }
-
-        let path = if path.starts_with("/") {
-            path.strip_prefix("/").unwrap_or(path)
-        } else {
-            path
-        };
-
-        let comps: Vec<_> = path.components().map(Component::as_os_str).collect();
-
-        let mut is_there = true;
-
-        for (at, fragment) in comps.iter().enumerate() {
-            is_there = is_there
-                && self
-                    .resources
-                    .iter()
-                    .any(|resource| at == resource.at && *fragment == resource.loc.as_os_str());
-        }
-
-        is_there
-    }
-
-    pub fn walk_decorate(&mut self, start: &Value) -> Result<(), ShellError> {
-        self.resources = Vec::<ValueResource>::new();
-        self.build(start, 0)?;
-        self.resources.sort();
-
-        Ok(())
-    }
-
-    fn build(&mut self, src: &Value, lvl: usize) -> Result<(), ShellError> {
-        for entry in src.row_entries() {
-            let value = entry.1;
-            let path = entry.0;
-
-            self.resources.push(ValueResource {
-                at: lvl,
-                loc: PathBuf::from(path),
-            });
-
-            if is_value_tagged_dir(value) {
-                self.build(value, lvl + 1)?;
-            }
-        }
-
-        Ok(())
-    }
-}