holm/nushell
1
1
Fork 0
forked from extern/nushell
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Option to replace command same name (#374)

Browse Source 
* option to replace command same name

* moved order of custom value declarations

* arranged dataframe folders and objects

* sort help commands by name

* added dtypes function for debugging

* corrected name for dataframe commands

* command names using function
This commit is contained in:
Fernando Herrera 2021-11-28 19:35:02 +00:00 committed by GitHub
parent e1e7e94261
commit c8b16c14d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 490 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Cargo.lock generated
View File

@ -1444,7 +1444,6 @@ dependencies = [
"lscolors",
"meval",
"nu-ansi-term 0.39.0",
"nu-dataframe",
"nu-engine",
"nu-json",
"nu-parser",
@ -1452,6 +1451,7 @@ dependencies = [
"nu-protocol",
"nu-table",
"nu-term-grid",
"num",
"polars",
"rand",
"rayon",
@ -1469,19 +1469,6 @@ dependencies = [
"unicode-segmentation",
]
[[package]]
name = "nu-dataframe"
version = "0.1.0"
dependencies = [
"chrono",
"indexmap",
"nu-json",
"nu-protocol",
"num",
"polars",
"serde",
]
[[package]]
name = "nu-engine"
version = "0.1.0"

6
Cargo.toml
View File

@ -13,7 +13,6 @@ members = [
"crates/nu-command",
"crates/nu-protocol",
"crates/nu-plugin",
"crates/nu-dataframe",
"crates/nu_plugin_inc",
]
@ -38,9 +37,8 @@ ctrlc = "3.2.1"
[features]
plugin = ["nu-plugin", "nu-parser/plugin", "nu-command/plugin", "nu-protocol/plugin"]
custom = ["nu-command/custom", "nu-protocol/custom"]
dataframe = ["custom", "nu-command/dataframe"]
default = ["plugin", "custom"]
dataframe = ["nu-command/dataframe"]
default = ["plugin"]
[dev-dependencies]
tempfile = "3.2.0"

8
crates/nu-command/Cargo.toml
View File

@ -13,7 +13,6 @@ nu-protocol = { path = "../nu-protocol" }
nu-table = { path = "../nu-table" }
nu-term-grid = { path = "../nu-term-grid" }
nu-parser = { path = "../nu-parser" }
nu-dataframe = { path = "../nu-dataframe", optional = true }
nu-ansi-term = { path = "../nu-ansi-term" }
trash = { version = "1.3.0", optional = true }
unicode-segmentation = "1.8.0"
@ -47,13 +46,14 @@ ical = "0.7.0"
calamine = "0.18.0"
rand = "0.8"
num = {version="0.4.0", optional=true}
[dependencies.polars]
version = "0.17.0"
optional = true
features = ["default", "parquet", "json"]
features = ["default", "parquet", "json", "serde", "object", "checked_arithmetic", "strings"]
[features]
trash-support = ["trash"]
plugin = ["nu-parser/plugin"]
custom = ["nu-protocol/custom"]
dataframe = ["custom", "nu-dataframe", "polars"]
dataframe = ["polars", "num"]

2
crates/nu-command/src/core_commands/debug.rs
View File

@ -15,7 +15,7 @@ impl Command for Debug {
}
fn signature(&self) -> Signature {
Signature::build("describe").category(Category::Core)
Signature::build("debug").category(Category::Core)
}
fn run(

14
crates/nu-command/src/core_commands/help.rs
View File

@ -187,7 +187,6 @@ fn help(
.into_pipeline_data(engine_state.ctrlc.clone()))
} else {
let mut name = String::new();
let mut output = String::new();
for r in &rest {
if !name.is_empty() {
@ -196,16 +195,15 @@ fn help(
name.push_str(&r.item);
}
for cmd in full_commands {
if cmd.0.name == name {
let help = get_full_help(&cmd.0, &cmd.1, engine_state);
output.push_str(&help);
}
}
let output = full_commands
.iter()
.filter(|(signature, _, _)| signature.name == name)
.map(|(signature, examples, _)| get_full_help(signature, examples, engine_state))
.collect::<Vec<String>>();
if !output.is_empty() {
Ok(Value::String {
val: output,
val: output.join("======================\n\n"),
span: call.head,
}
.into_pipeline_data())

0
crates/nu-dataframe/README.md → crates/nu-command/src/dataframe/README.md
View File

183
crates/nu-command/src/dataframe/describe.rs Normal file
View File

@ -0,0 +1,183 @@
use super::objects::nu_dataframe::NuDataFrame;
use nu_protocol::{
ast::Call,
engine::{Command, EngineState, Stack},
Category, Example, PipelineData, ShellError, Signature,
};
use polars::{
chunked_array::ChunkedArray,
prelude::{
AnyValue, DataFrame, DataType, Float64Type, IntoSeries, NewChunkedArray, Series, Utf8Type,
},
};
#[derive(Clone)]
pub struct DescribeDF;
impl Command for DescribeDF {
fn name(&self) -> &str {
"describe"
}
fn usage(&self) -> &str {
"Describes dataframes numeric columns"
}
fn signature(&self) -> Signature {
Signature::build(self.name().to_string()).category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "dataframe description",
example: "[[a b]; [1 1] [1 1]] | to-df | describe",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head.clone())?;
let names = ChunkedArray::<Utf8Type>::new_from_opt_slice(
"descriptor",
&[
Some("count"),
Some("sum"),
Some("mean"),
Some("median"),
Some("std"),
Some("min"),
Some("25%"),
Some("50%"),
Some("75%"),
Some("max"),
],
)
.into_series();
let head = std::iter::once(names);
let tail = df
.as_ref()
.get_columns()
.iter()
.filter(|col| col.dtype() != &DataType::Object("object"))
.map(|col| {
let count = col.len() as f64;
let sum = match col.sum_as_series().cast(&DataType::Float64) {
Ok(ca) => match ca.get(0) {
AnyValue::Float64(v) => Some(v),
_ => None,
},
Err(_) => None,
};
let mean = match col.mean_as_series().get(0) {
AnyValue::Float64(v) => Some(v),
_ => None,
};
let median = match col.median_as_series().get(0) {
AnyValue::Float64(v) => Some(v),
_ => None,
};
let std = match col.std_as_series().get(0) {
AnyValue::Float64(v) => Some(v),
_ => None,
};
let min = match col.min_as_series().cast(&DataType::Float64) {
Ok(ca) => match ca.get(0) {
AnyValue::Float64(v) => Some(v),
_ => None,
},
Err(_) => None,
};
let q_25 = match col.quantile_as_series(0.25) {
Ok(ca) => match ca.cast(&DataType::Float64) {
Ok(ca) => match ca.get(0) {
AnyValue::Float64(v) => Some(v),
_ => None,
},
Err(_) => None,
},
Err(_) => None,
};
let q_50 = match col.quantile_as_series(0.50) {
Ok(ca) => match ca.cast(&DataType::Float64) {
Ok(ca) => match ca.get(0) {
AnyValue::Float64(v) => Some(v),
_ => None,
},
Err(_) => None,
},
Err(_) => None,
};
let q_75 = match col.quantile_as_series(0.75) {
Ok(ca) => match ca.cast(&DataType::Float64) {
Ok(ca) => match ca.get(0) {
AnyValue::Float64(v) => Some(v),
_ => None,
},
Err(_) => None,
},
Err(_) => None,
};
let max = match col.max_as_series().cast(&DataType::Float64) {
Ok(ca) => match ca.get(0) {
AnyValue::Float64(v) => Some(v),
_ => None,
},
Err(_) => None,
};
let name = format!("{} ({})", col.name(), col.dtype());
ChunkedArray::<Float64Type>::new_from_opt_slice(
&name,
&[
Some(count),
sum,
mean,
median,
std,
min,
q_25,
q_50,
q_75,
max,
],
)
.into_series()
});
let res = head.chain(tail).collect::<Vec<Series>>();
let df = DataFrame::new(res).map_err(|e| {
ShellError::LabeledError("Dataframe Error".into(), e.to_string(), call.head)
})?;
Ok(PipelineData::Value(NuDataFrame::dataframe_into_value(
df, call.head,
)))
}

82
crates/nu-command/src/dataframe/dtypes.rs Normal file
View File

@ -0,0 +1,82 @@
use super::objects::nu_dataframe::{Column, NuDataFrame};
use nu_protocol::{
ast::Call,
engine::{Command, EngineState, Stack},
Category, Example, PipelineData, ShellError, Signature, Value,
};
#[derive(Clone)]
pub struct DataTypes;
impl Command for DataTypes {
fn name(&self) -> &str {
"dtypes"
}
fn usage(&self) -> &str {
"Show dataframe data types"
}
fn signature(&self) -> Signature {
Signature::build(self.name().to_string()).category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "drop column a",
example: "[[a b]; [1 2] [3 4]] | to-df | dtypes",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
#[allow(clippy::needless_collect)]
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head.clone())?;
let mut dtypes: Vec<Value> = Vec::new();
let names: Vec<Value> = df
.as_ref()
.get_column_names()
.iter()
.map(|v| {
let dtype = df
.as_ref()
.column(v)
.expect("using name from list of names from dataframe")
.dtype();
let dtype_str = dtype.to_string();
dtypes.push(Value::String {
val: dtype_str.into(),
span: call.head,
});
Value::String {
val: v.to_string().into(),
span: call.head,
}
})
.collect();
let names_col = Column::new("column".to_string(), names);
let dtypes_col = Column::new("dtype".to_string(), dtypes);
let df = NuDataFrame::try_from_columns(vec![names_col, dtypes_col])?;
Ok(PipelineData::Value(df.to_value(call.head)))
}

5
crates/nu-command/src/dataframe/mod.rs
View File

@ -1,5 +1,10 @@
mod describe;
mod dtypes;
mod objects;
mod open;
mod to_df;
pub use describe::DescribeDF;
pub use dtypes::DataTypes;
pub use open::OpenDataFrame;
pub use to_df::ToDataFrame;

1
crates/nu-command/src/dataframe/objects/mod.rs Normal file
View File

@ -0,0 +1 @@
pub(super) mod nu_dataframe;

62
crates/nu-dataframe/src/between_values.rs → crates/nu-command/src/dataframe/objects/nu_dataframe/between_values.rs
View File

@ -1,6 +1,6 @@
use super::{operations::Axis, NuDataFrame};
use nu_protocol::{ast::Operator, ShellError, Span, Spanned, Value};
use nu_protocol::{ast::Operator, span, ShellError, Span, Spanned, Value};
use num::Zero;
use polars::prelude::{
BooleanType, ChunkCompare, ChunkedArray, DataType, Float64Type, Int64Type, IntoSeries,
@ -10,12 +10,12 @@ use std::ops::{Add, BitAnd, BitOr, Div, Mul, Sub};
pub fn between_dataframes(
operator: Spanned<Operator>,
left: Value,
left: &Value,
lhs: &NuDataFrame,
right: &Value,
rhs: &NuDataFrame,
operation_span: Span,
) -> Result<Value, ShellError> {
let operation_span = span(&[left.span()?, right.span()?]);
match operator.item {
Operator::Plus => match lhs.append_df(rhs, Axis::Row, operation_span) {
Ok(df) => Ok(df.to_value(operation_span)),
@ -33,12 +33,12 @@ pub fn between_dataframes(
pub fn compute_between_series(
operator: Spanned<Operator>,
left: Value,
left: &Value,
lhs: &Series,
right: &Value,
rhs: &Series,
operation_span: Span,
) -> Result<Value, ShellError> {
let operation_span = span(&[left.span()?, right.span()?]);
match operator.item {
Operator::Plus => {
let mut res = lhs + rhs;
@ -167,9 +167,8 @@ pub fn compute_between_series(
pub fn compute_series_single_value(
operator: Spanned<Operator>,
left: &Value,
lhs: &NuDataFrame,
lhs_span: &Span,
left: Value,
right: &Value,
) -> Result<Value, ShellError> {
if !lhs.is_series() {
@ -182,15 +181,16 @@ pub fn compute_series_single_value(
});
}
let lhs = lhs.as_series(*lhs_span)?;
let lhs_span = left.span()?;
let lhs = lhs.as_series(lhs_span)?;
match operator.item {
Operator::Plus => match &right {
Value::Int { val, .. } => {
compute_series_i64(&lhs, *val, <ChunkedArray<Int64Type>>::add, *lhs_span)
compute_series_i64(&lhs, *val, <ChunkedArray<Int64Type>>::add, lhs_span)
}
Value::Float { val, .. } => {
compute_series_decimal(&lhs, *val, <ChunkedArray<Float64Type>>::add, *lhs_span)
compute_series_decimal(&lhs, *val, <ChunkedArray<Float64Type>>::add, lhs_span)
}
_ => Err(ShellError::OperatorMismatch {
op_span: operator.span,
@ -202,10 +202,10 @@ pub fn compute_series_single_value(
},
Operator::Minus => match &right {
Value::Int { val, .. } => {
compute_series_i64(&lhs, *val, <ChunkedArray<Int64Type>>::sub, *lhs_span)
compute_series_i64(&lhs, *val, <ChunkedArray<Int64Type>>::sub, lhs_span)
}
Value::Float { val, .. } => {
compute_series_decimal(&lhs, *val, <ChunkedArray<Float64Type>>::sub, *lhs_span)
compute_series_decimal(&lhs, *val, <ChunkedArray<Float64Type>>::sub, lhs_span)
}
_ => Err(ShellError::OperatorMismatch {
op_span: operator.span,
@ -217,10 +217,10 @@ pub fn compute_series_single_value(
},
Operator::Multiply => match &right {
Value::Int { val, .. } => {
compute_series_i64(&lhs, *val, <ChunkedArray<Int64Type>>::mul, *lhs_span)
compute_series_i64(&lhs, *val, <ChunkedArray<Int64Type>>::mul, lhs_span)
}
Value::Float { val, .. } => {
compute_series_decimal(&lhs, *val, <ChunkedArray<Float64Type>>::mul, *lhs_span)
compute_series_decimal(&lhs, *val, <ChunkedArray<Float64Type>>::mul, lhs_span)
}
_ => Err(ShellError::OperatorMismatch {
op_span: operator.span,
@ -235,14 +235,14 @@ pub fn compute_series_single_value(
if *val == 0 {
Err(ShellError::DivisionByZero(*span))
} else {
compute_series_i64(&lhs, *val, <ChunkedArray<Int64Type>>::div, *lhs_span)
compute_series_i64(&lhs, *val, <ChunkedArray<Int64Type>>::div, lhs_span)
}
}
Value::Float { val, span } => {
if val.is_zero() {
Err(ShellError::DivisionByZero(*span))
} else {
compute_series_decimal(&lhs, *val, <ChunkedArray<Float64Type>>::div, *lhs_span)
compute_series_decimal(&lhs, *val, <ChunkedArray<Float64Type>>::div, lhs_span)
}
}
_ => Err(ShellError::OperatorMismatch {
@ -254,9 +254,9 @@ pub fn compute_series_single_value(
}),
},
Operator::Equal => match &right {
Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::eq, *lhs_span),
Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::eq, lhs_span),
Value::Float { val, .. } => {
compare_series_decimal(&lhs, *val, ChunkedArray::eq, *lhs_span)
compare_series_decimal(&lhs, *val, ChunkedArray::eq, lhs_span)
}
_ => Err(ShellError::OperatorMismatch {
op_span: operator.span,
@ -267,9 +267,9 @@ pub fn compute_series_single_value(
}),
},
Operator::NotEqual => match &right {
Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::neq, *lhs_span),
Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::neq, lhs_span),
Value::Float { val, .. } => {
compare_series_decimal(&lhs, *val, ChunkedArray::neq, *lhs_span)
compare_series_decimal(&lhs, *val, ChunkedArray::neq, lhs_span)
}
_ => Err(ShellError::OperatorMismatch {
op_span: operator.span,
@ -280,9 +280,9 @@ pub fn compute_series_single_value(
}),
},
Operator::LessThan => match &right {
Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::lt, *lhs_span),
Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::lt, lhs_span),
Value::Float { val, .. } => {
compare_series_decimal(&lhs, *val, ChunkedArray::lt, *lhs_span)
compare_series_decimal(&lhs, *val, ChunkedArray::lt, lhs_span)
}
_ => Err(ShellError::OperatorMismatch {
op_span: operator.span,
@ -293,11 +293,9 @@ pub fn compute_series_single_value(
}),
},
Operator::LessThanOrEqual => match &right {
Value::Int { val, .. } => {
compare_series_i64(&lhs, *val, ChunkedArray::lt_eq, *lhs_span)
}
Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::lt_eq, lhs_span),
Value::Float { val, .. } => {
compare_series_decimal(&lhs, *val, ChunkedArray::lt_eq, *lhs_span)
compare_series_decimal(&lhs, *val, ChunkedArray::lt_eq, lhs_span)
}
_ => Err(ShellError::OperatorMismatch {
op_span: operator.span,
@ -308,9 +306,9 @@ pub fn compute_series_single_value(
}),
},
Operator::GreaterThan => match &right {
Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::gt, *lhs_span),
Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::gt, lhs_span),
Value::Float { val, .. } => {
compare_series_decimal(&lhs, *val, ChunkedArray::gt, *lhs_span)
compare_series_decimal(&lhs, *val, ChunkedArray::gt, lhs_span)
}
_ => Err(ShellError::OperatorMismatch {
op_span: operator.span,
@ -321,11 +319,9 @@ pub fn compute_series_single_value(
}),
},
Operator::GreaterThanOrEqual => match &right {
Value::Int { val, .. } => {
compare_series_i64(&lhs, *val, ChunkedArray::gt_eq, *lhs_span)
}
Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::gt_eq, lhs_span),
Value::Float { val, .. } => {
compare_series_decimal(&lhs, *val, ChunkedArray::gt_eq, *lhs_span)
compare_series_decimal(&lhs, *val, ChunkedArray::gt_eq, lhs_span)
}
_ => Err(ShellError::OperatorMismatch {
op_span: operator.span,
@ -336,7 +332,7 @@ pub fn compute_series_single_value(
}),
},
Operator::Contains => match &right {
Value::String { val, .. } => contains_series_pat(&lhs, val, *lhs_span),
Value::String { val, .. } => contains_series_pat(&lhs, val, lhs_span),
_ => Err(ShellError::OperatorMismatch {
op_span: operator.span,
lhs_ty: left.get_type(),

9
crates/nu-dataframe/src/conversion.rs → crates/nu-command/src/dataframe/objects/nu_dataframe/conversion.rs
View File

@ -1,5 +1,4 @@
use super::NuDataFrame;
use crate::DataFrameValue;
use super::{DataFrameValue, NuDataFrame};
use chrono::{DateTime, FixedOffset, NaiveDateTime};
use indexmap::map::{Entry, IndexMap};
use nu_protocol::{ShellError, Span, Value};
@ -35,9 +34,9 @@ impl Column {
self.name.as_str()
}
pub fn iter(&self) -> impl Iterator<Item = &Value> {
self.values.iter()
}
//pub fn iter(&self) -> impl Iterator<Item = &Value> {
// self.values.iter()
//}
}
impl IntoIterator for Column {

8
crates/nu-dataframe/src/custom_value.rs → crates/nu-command/src/dataframe/objects/nu_dataframe/custom_value.rs
View File

@ -1,5 +1,5 @@
use crate::NuDataFrame;
use nu_protocol::{ast::Operator, CustomValue, ShellError, Span, Value};
use super::NuDataFrame;
use nu_protocol::{ast::Operator, Category, CustomValue, ShellError, Span, Value};
// CustomValue implementation for NuDataFrame
impl CustomValue for NuDataFrame {
@ -20,6 +20,10 @@ impl CustomValue for NuDataFrame {
}
}
fn category(&self) -> Category {
Category::Custom(self.typetag_name().into())
}
fn value_string(&self) -> String {
self.typetag_name().to_string()
}

46
crates/nu-dataframe/src/lib.rs → crates/nu-command/src/dataframe/objects/nu_dataframe/mod.rs
View File

@ -5,9 +5,9 @@ mod operations;
use std::{cmp::Ordering, fmt::Display, hash::Hasher};
use conversion::{Column, ColumnMap};
pub use conversion::{Column, ColumnMap};
use indexmap::map::IndexMap;
use nu_protocol::{did_you_mean, ShellError, Span, Value};
use nu_protocol::{did_you_mean, PipelineData, ShellError, Span, Value};
use polars::prelude::{DataFrame, PolarsObject, Series};
use serde::{Deserialize, Serialize};
@ -62,13 +62,25 @@ impl std::hash::Hash for DataFrameValue {
impl PolarsObject for DataFrameValue {
fn type_name() -> &'static str {
"value"
"object"
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct NuDataFrame(DataFrame);
impl AsRef<DataFrame> for NuDataFrame {
fn as_ref(&self) -> &polars::prelude::DataFrame {
&self.0
}
}
impl AsMut<DataFrame> for NuDataFrame {
fn as_mut(&mut self) -> &mut polars::prelude::DataFrame {
&mut self.0
}
}
impl NuDataFrame {
pub fn new(dataframe: DataFrame) -> Self {
Self(dataframe)
@ -131,12 +143,12 @@ impl NuDataFrame {
conversion::from_parsed_columns(column_values)
}
pub fn try_from_series(columns: Vec<Series>) -> Result<Self, ShellError> {
let dataframe = DataFrame::new(columns)
.map_err(|e| ShellError::InternalError(format!("Unable to create DataFrame: {}", e)))?;
//pub fn try_from_series(columns: Vec<Series>) -> Result<Self, ShellError> {
// let dataframe = DataFrame::new(columns)
// .map_err(|e| ShellError::InternalError(format!("Unable to create DataFrame: {}", e)))?;
Ok(Self::new(dataframe))
}
// Ok(Self::new(dataframe))
//}
pub fn try_from_columns(columns: Vec<Column>) -> Result<Self, ShellError> {
let mut column_values: ColumnMap = IndexMap::new();
@ -151,6 +163,24 @@ impl NuDataFrame {
conversion::from_parsed_columns(column_values)
}
pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result<Self, ShellError> {
match input.into_value(span) {
Value::CustomValue { val, span } => match val.as_any().downcast_ref::<NuDataFrame>() {
Some(df) => Ok(NuDataFrame(df.0.clone())),
None => Err(ShellError::CantConvert(
"Dataframe not found".into(),
"value is not a dataframe".into(),
span,
)),
},
_ => Err(ShellError::CantConvert(
"Dataframe not found".into(),
"value is not a dataframe".into(),
span,
)),
}
}
pub fn column(&self, column: &str, span: Span) -> Result<Self, ShellError> {
let s = self.0.column(column).map_err(|_| {
let possibilities = self

138
crates/nu-dataframe/src/operations.rs → crates/nu-command/src/dataframe/objects/nu_dataframe/operations.rs
View File

@ -1,7 +1,7 @@
use nu_protocol::{ast::Operator, span, ShellError, Span, Spanned, Value};
use nu_protocol::{ast::Operator, ShellError, Span, Spanned, Value};
use polars::prelude::{DataFrame, Series};
use crate::between_values::{
use super::between_values::{
between_dataframes, compute_between_series, compute_series_single_value,
};
@ -9,18 +9,18 @@ use super::NuDataFrame;
pub enum Axis {
Row,
Column,
//Column,
}
impl Axis {
pub fn try_from_str(axis: &str, span: Span) -> Result<Axis, ShellError> {
match axis {
"row" => Ok(Axis::Row),
"col" => Ok(Axis::Column),
_ => Err(ShellError::DidYouMean("'row' or 'col'".into(), span)),
}
}
}
//impl Axis {
// pub fn try_from_str(axis: &str, span: Span) -> Result<Axis, ShellError> {
// match axis {
// "row" => Ok(Axis::Row),
// "col" => Ok(Axis::Column),
// _ => Err(ShellError::DidYouMean("'row' or 'col'".into(), span)),
// }
// }
//}
impl NuDataFrame {
pub fn compute_with_value(
@ -42,7 +42,6 @@ impl NuDataFrame {
)
})?;
let operation_span = span(&[lhs_span, *rhs_span]);
match (self.is_series(), rhs.is_series()) {
(true, true) => {
let lhs = &self
@ -77,11 +76,10 @@ impl NuDataFrame {
compute_between_series(
op,
NuDataFrame::default_value(lhs_span),
&NuDataFrame::default_value(lhs_span),
lhs,
right,
rhs,
operation_span,
)
}
_ => {
@ -101,11 +99,10 @@ impl NuDataFrame {
between_dataframes(
op,
NuDataFrame::default_value(lhs_span),
&NuDataFrame::default_value(lhs_span),
self,
right,
rhs,
operation_span,
)
}
}
@ -116,13 +113,7 @@ impl NuDataFrame {
span: op_span,
};
compute_series_single_value(
op,
self,
&lhs_span,
NuDataFrame::default_value(lhs_span),
right,
)
compute_series_single_value(op, &NuDataFrame::default_value(lhs_span), self, right)
}
}
}
@ -131,7 +122,7 @@ impl NuDataFrame {
&self,
other: &NuDataFrame,
axis: Axis,
span: Span,
_span: Span,
) -> Result<Self, ShellError> {
match axis {
Axis::Row => {
@ -160,61 +151,60 @@ impl NuDataFrame {
.map_err(|e| ShellError::InternalError(e.to_string()))?;
Ok(NuDataFrame::new(df_new))
}
Axis::Column => {
if self.0.width() != other.0.width() {
return Err(ShellError::IncompatibleParametersSingle(
"Dataframes with different number of columns".into(),
span,
));
}
} //Axis::Column => {
// if self.0.width() != other.0.width() {
// return Err(ShellError::IncompatibleParametersSingle(
// "Dataframes with different number of columns".into(),
// span,
// ));
// }
if !self
.0
.get_column_names()
.iter()
.all(|col| other.0.get_column_names().contains(col))
{
return Err(ShellError::IncompatibleParametersSingle(
"Dataframes with different columns names".into(),
span,
));
}
// if !self
// .0
// .get_column_names()
// .iter()
// .all(|col| other.0.get_column_names().contains(col))
// {
// return Err(ShellError::IncompatibleParametersSingle(
// "Dataframes with different columns names".into(),
// span,
// ));
// }
let new_cols = self
.0
.get_columns()
.iter()
.map(|s| {
let other_col = other
.0
.column(s.name())
.expect("Already checked that dataframes have same columns");
// let new_cols = self
// .0
// .get_columns()
// .iter()
// .map(|s| {
// let other_col = other
// .0
// .column(s.name())
// .expect("Already checked that dataframes have same columns");
let mut tmp = s.clone();
let res = tmp.append(other_col);
// let mut tmp = s.clone();
// let res = tmp.append(other_col);
match res {
Ok(s) => Ok(s.clone()),
Err(e) => Err({
ShellError::InternalError(format!(
"Unable to append dataframes: {}",
e
))
}),
}
})
.collect::<Result<Vec<Series>, ShellError>>()?;
// match res {
// Ok(s) => Ok(s.clone()),
// Err(e) => Err({
// ShellError::InternalError(format!(
// "Unable to append dataframes: {}",
// e
// ))
// }),
// }
// })
// .collect::<Result<Vec<Series>, ShellError>>()?;
let df_new = DataFrame::new(new_cols).map_err(|e| {
ShellError::InternalError(format!(
"Unable to append dataframes: {}",
e.to_string()
))
})?;
// let df_new = DataFrame::new(new_cols).map_err(|e| {
// ShellError::InternalError(format!(
// "Unable to append dataframes: {}",
// e.to_string()
// ))
// })?;
Ok(NuDataFrame::new(df_new))
}
// Ok(NuDataFrame::new(df_new))
//}
}
}
}

6
crates/nu-command/src/dataframe/open.rs
View File

@ -1,6 +1,6 @@
use std::{fs::File, path::PathBuf};
use nu_dataframe::NuDataFrame;
use super::objects::nu_dataframe::NuDataFrame;
use nu_engine::CallExt;
use nu_protocol::{
ast::Call,
@ -23,7 +23,7 @@ impl Command for OpenDataFrame {
}
fn signature(&self) -> Signature {
Signature::build("open-df")
Signature::build(self.name().to_string())
.required(
"file",
SyntaxShape::Filepath,
@ -64,7 +64,7 @@ impl Command for OpenDataFrame {
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Takes a file name and creates a dataframe",
example: "dataframe open test.csv",
example: "open-df test.csv",
result: None,
}]
}

14
crates/nu-command/src/dataframe/to_df.rs
View File

@ -1,4 +1,4 @@
use nu_dataframe::NuDataFrame;
use super::objects::nu_dataframe::NuDataFrame;
use nu_protocol::{
ast::Call,
engine::{Command, EngineState, Stack},
@ -10,7 +10,7 @@ pub struct ToDataFrame;
impl Command for ToDataFrame {
fn name(&self) -> &str {
"to-df"
"to df"
}
fn usage(&self) -> &str {
@ -18,29 +18,29 @@ impl Command for ToDataFrame {
}
fn signature(&self) -> Signature {
Signature::build("to-df").category(Category::Custom("dataframe".into()))
Signature::build(self.name().to_string()).category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Takes a dictionary and creates a dataframe",
example: "[[a b];[1 2] [3 4]] | to-df",
example: "[[a b];[1 2] [3 4]] | to df",
result: None,
},
Example {
description: "Takes a list of tables and creates a dataframe",
example: "[[1 2 a] [3 4 b] [5 6 c]] | to-df",
example: "[[1 2 a] [3 4 b] [5 6 c]] | to df",
result: None,
},
Example {
description: "Takes a list and creates a dataframe",
example: "[a b c] | to-df",
example: "[a b c] | to df",
result: None,
},
Example {
description: "Takes a list of booleans and creates a dataframe",
example: "[$true $true $false] | to-df",
example: "[$true $true $false] | to df",
result: None,
},
]

10
crates/nu-command/src/default_context.rs
View File

@ -17,6 +17,13 @@ pub fn create_default_context() -> EngineState {
};
}
// If there are commands that have the same name as default declarations,
// they have to be registered before the main declarations. This helps to make
// them only accessible if the correct input value category is used with the
// declaration
#[cfg(feature = "dataframe")]
bind_command!(DataTypes, DescribeDF, OpenDataFrame, ToDataFrame);
// TODO: sort default context items categorically
bind_command!(
Alias,
@ -148,9 +155,6 @@ pub fn create_default_context() -> EngineState {
#[cfg(feature = "plugin")]
bind_command!(Register);
#[cfg(feature = "dataframe")]
bind_command!(OpenDataFrame, ToDataFrame);
// This is a WIP proof of concept
// bind_command!(ListGitBranches, Git, GitCheckout, Source);

1
crates/nu-command/src/formats/to/json.rs
View File

@ -73,7 +73,6 @@ pub fn value_to_json_value(v: &Value) -> Result<nu_json::Value, ShellError> {
}
nu_json::Value::Object(m)
}
#[cfg(feature = "custom")]
Value::CustomValue { val, .. } => val.to_json(),
})
}

1
crates/nu-command/src/viewers/table.rs
View File

@ -107,7 +107,6 @@ impl Command for Table {
.into_pipeline_data())
}
PipelineData::Value(Value::Error { error }) => Err(error),
#[cfg(feature = "custom")]
PipelineData::Value(Value::CustomValue { val, span }) => {
let base_pipeline = val.to_base_value(span)?.into_pipeline_data();
self.run(engine_state, stack, call, base_pipeline)

14
crates/nu-dataframe/Cargo.toml
View File

@ -1,14 +0,0 @@
[package]
name = "nu-dataframe"
version = "0.1.0"
edition = "2018"
[dependencies]
chrono = { version="0.4.19", features=["serde"] }
serde = {version = "1.0.130", features = ["derive"]}
num = "0.4.0"
nu-protocol = { path = "../nu-protocol", features = ["custom"] }
nu-json = { path = "../nu-json"}
indexmap = { version="1.7.0", features=["serde-1"] }
polars = { version = "0.17.0", features = ["default", "serde", "object", "checked_arithmetic", "strings"] }

3
crates/nu-engine/src/eval.rs
View File

@ -24,7 +24,8 @@ fn eval_call(
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let decl = engine_state.get_decl(call.decl_id);
let decl = engine_state.get_decl_with_input(call.decl_id, &input);
if call.named.iter().any(|(flag, _)| flag.item == "help") {
let full_help = get_full_help(&decl.signature(), &decl.examples(), engine_state);
Ok(Value::String {

1
crates/nu-protocol/Cargo.toml
View File

@ -20,7 +20,6 @@ typetag = "0.1.8"
[features]
plugin = ["serde_json"]
custom = []
[dev-dependencies]
serde_json = "1.0"

38
crates/nu-protocol/src/engine/engine_state.rs
View File

@ -1,7 +1,7 @@
use super::Command;
use crate::{
ast::Block, BlockId, DeclId, Example, Overlay, OverlayId, ShellError, Signature, Span, Type,
VarId,
ast::Block, BlockId, DeclId, Example, Overlay, OverlayId, PipelineData, ShellError, Signature,
Span, Type, Value, VarId,
};
use core::panic;
use std::{
@ -357,6 +357,39 @@ impl EngineState {
.expect("internal error: missing declaration")
}
#[allow(clippy::borrowed_box)]
pub fn get_decl_with_input(&self, decl_id: DeclId, input: &PipelineData) -> &Box<dyn Command> {
let decl = self.get_decl(decl_id);
match input {
PipelineData::Stream(_) => decl,
PipelineData::Value(value) => match value {
Value::CustomValue { val, .. } => {
// This filter works because the custom definitions were declared
// before the default nushell declarations. This means that the custom
// declarations that get overridden by the default declarations can only
// be accessed if the input value has the required category
let decls = self
.decls
.iter()
.enumerate()
.filter(|(_, decl_inner)| {
decl.name() == decl_inner.name()
&& decl_inner.signature().category == val.category()
})
.map(|(index, _)| index)
.collect::<Vec<usize>>();
match decls.first() {
Some(index) => self.get_decl(*index),
None => decl,
}
}
_ => decl,
},
}
}
pub fn get_signatures(&self) -> Vec<Signature> {
let mut output = vec![];
for decl in self.decls.iter() {
@ -384,6 +417,7 @@ impl EngineState {
}
}
output.sort_by(|a, b| a.0.name.cmp(&b.0.name));
output
}

4
crates/nu-protocol/src/lib.rs
View File

@ -26,7 +26,5 @@ pub use signature::*;
pub use span::*;
pub use syntax_shape::*;
pub use ty::*;
pub use value::*;
#[cfg(feature = "custom")]
pub use value::CustomValue;
pub use value::*;

1
crates/nu-protocol/src/pipeline_data.rs
View File

@ -31,6 +31,7 @@ use crate::{ast::PathMember, Config, ShellError, Span, Value, ValueStream};
/// * A balance of the two approaches is what we've landed on: Values are thread-safe to pass, and we can stream
/// them into any sources. Streams are still available to model the infinite streams approach of original
/// Nushell.
#[derive(Debug)]
pub enum PipelineData {
Value(Value),
Stream(ValueStream),

4
crates/nu-protocol/src/shell_error.rs
View File

@ -210,6 +210,10 @@ pub enum ShellError {
#[error("Casting error")]
#[diagnostic(code(nu::parser::downcast_not_possible), url(docsrs))]
DowncastNotPossible(String, #[label("{0}")] Span),
#[error("{0}")]
#[diagnostic()]
LabeledError(String, String, #[label("{1}")] Span),
}
impl From<std::io::Error> for ShellError {

4
crates/nu-protocol/src/value/custom_value.rs
View File

@ -1,12 +1,14 @@
use std::fmt;
use crate::{ast::Operator, ShellError, Span, Value};
use crate::{ast::Operator, Category, ShellError, Span, Value};
// Trait definition for a custom value
#[typetag::serde(tag = "type")]
pub trait CustomValue: fmt::Debug + Send + Sync {
fn clone_value(&self, span: Span) -> Value;
fn category(&self) -> Category;
// Define string representation of the custom value
fn value_string(&self) -> String;

30
crates/nu-protocol/src/value/mod.rs
View File

@ -18,10 +18,7 @@ use std::{cmp::Ordering, fmt::Debug};
use crate::ast::{CellPath, PathMember};
use crate::{did_you_mean, span, BlockId, Config, Span, Spanned, Type};
#[cfg(feature = "custom")]
use crate::ast::Operator;
#[cfg(feature = "custom")]
pub use custom_value::CustomValue;
use crate::ShellError;
@ -88,7 +85,6 @@ pub enum Value {
val: CellPath,
span: Span,
},
#[cfg(feature = "custom")]
CustomValue {
val: Box<dyn CustomValue>,
span: Span,
@ -155,7 +151,6 @@ impl Clone for Value {
val: val.clone(),
span: *span,
},
#[cfg(feature = "custom")]
Value::CustomValue { val, span } => val.clone_value(*span),
}
}
@ -224,7 +219,6 @@ impl Value {
Value::Nothing { span, .. } => Ok(*span),
Value::Binary { span, .. } => Ok(*span),
Value::CellPath { span, .. } => Ok(*span),
#[cfg(feature = "custom")]
Value::CustomValue { span, .. } => Ok(*span),
}
}
@ -247,7 +241,6 @@ impl Value {
Value::Error { .. } => {}
Value::Binary { span, .. } => *span = new_span,
Value::CellPath { span, .. } => *span = new_span,
#[cfg(feature = "custom")]
Value::CustomValue { span, .. } => *span = new_span,
}
@ -277,7 +270,6 @@ impl Value {
Value::Error { .. } => Type::Error,
Value::Binary { .. } => Type::Binary,
Value::CellPath { .. } => Type::CellPath,
#[cfg(feature = "custom")]
Value::CustomValue { .. } => Type::Custom,
}
}
@ -319,7 +311,6 @@ impl Value {
Value::Error { error } => format!("{:?}", error),
Value::Binary { val, .. } => format!("{:?}", val),
Value::CellPath { val, .. } => val.into_string(),
#[cfg(feature = "custom")]
Value::CustomValue { val, .. } => val.value_string(),
}
}
@ -361,7 +352,6 @@ impl Value {
Value::Error { error } => format!("{:?}", error),
Value::Binary { val, .. } => format!("{:?}", val),
Value::CellPath { val, .. } => val.into_string(),
#[cfg(feature = "custom")]
Value::CustomValue { val, .. } => val.value_string(),
}
}
@ -408,7 +398,6 @@ impl Value {
return Err(ShellError::AccessBeyondEndOfStream(*origin_span));
}
}
#[cfg(feature = "custom")]
Value::CustomValue { val, .. } => {
current = val.follow_path_int(*count, *origin_span)?;
}
@ -459,7 +448,6 @@ impl Value {
span: *span,
};
}
#[cfg(feature = "custom")]
Value::CustomValue { val, .. } => {
current = val.follow_path_string(column_name.clone(), *origin_span)?;
}
@ -725,7 +713,6 @@ impl Value {
}
}
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::Plus, op, rhs)
}
@ -795,7 +782,6 @@ impl Value {
}
}
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::Minus, op, rhs)
}
@ -835,7 +821,6 @@ impl Value {
val: lhs * rhs,
span,
}),
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::Multiply, op, rhs)
}
@ -900,7 +885,6 @@ impl Value {
Err(ShellError::DivisionByZero(op))
}
}
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::Divide, op, rhs)
}
@ -917,7 +901,6 @@ impl Value {
pub fn lt(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
let span = span(&[self.span()?, rhs.span()?]);
#[cfg(feature = "custom")]
if let (Value::CustomValue { val: lhs, span }, rhs) = (self, rhs) {
return lhs.operation(*span, Operator::LessThan, op, rhs);
}
@ -939,7 +922,6 @@ impl Value {
pub fn lte(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
let span = span(&[self.span()?, rhs.span()?]);
#[cfg(feature = "custom")]
if let (Value::CustomValue { val: lhs, span }, rhs) = (self, rhs) {
return lhs.operation(*span, Operator::LessThanOrEqual, op, rhs);
}
@ -961,7 +943,6 @@ impl Value {
pub fn gt(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
let span = span(&[self.span()?, rhs.span()?]);
#[cfg(feature = "custom")]
if let (Value::CustomValue { val: lhs, span }, rhs) = (self, rhs) {
return lhs.operation(*span, Operator::GreaterThan, op, rhs);
}
@ -983,7 +964,6 @@ impl Value {
pub fn gte(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
let span = span(&[self.span()?, rhs.span()?]);
#[cfg(feature = "custom")]
if let (Value::CustomValue { val: lhs, span }, rhs) = (self, rhs) {
return lhs.operation(*span, Operator::GreaterThanOrEqual, op, rhs);
}
@ -1005,7 +985,6 @@ impl Value {
pub fn eq(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
let span = span(&[self.span()?, rhs.span()?]);
#[cfg(feature = "custom")]
if let (Value::CustomValue { val: lhs, span }, rhs) = (self, rhs) {
return lhs.operation(*span, Operator::Equal, op, rhs);
}
@ -1027,7 +1006,6 @@ impl Value {
pub fn ne(&self, op: Span, rhs: &Value) -> Result<Value, ShellError> {
let span = span(&[self.span()?, rhs.span()?]);
#[cfg(feature = "custom")]
if let (Value::CustomValue { val: lhs, span }, rhs) = (self, rhs) {
return lhs.operation(*span, Operator::NotEqual, op, rhs);
}
@ -1067,7 +1045,6 @@ impl Value {
val: rhs.contains(lhs),
span,
}),
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::In, op, rhs)
}
@ -1101,7 +1078,6 @@ impl Value {
val: !rhs.contains(lhs),
span,
}),
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::NotIn, op, rhs)
}
@ -1123,7 +1099,6 @@ impl Value {
val: lhs.contains(rhs),
span,
}),
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::Contains, op, rhs)
}
@ -1145,7 +1120,6 @@ impl Value {
val: !lhs.contains(rhs),
span,
}),
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::NotContains, op, rhs)
}
@ -1203,7 +1177,6 @@ impl Value {
Err(ShellError::DivisionByZero(op))
}
}
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::Modulo, op, rhs)
}
@ -1226,7 +1199,6 @@ impl Value {
val: *lhs && *rhs,
span,
}),
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::And, op, rhs)
}
@ -1248,7 +1220,6 @@ impl Value {
val: *lhs || *rhs,
span,
}),
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::Or, op, rhs)
}
@ -1288,7 +1259,6 @@ impl Value {
val: lhs.powf(*rhs),
span,
}),
#[cfg(feature = "custom")]
(Value::CustomValue { val: lhs, span }, rhs) => {
lhs.operation(*span, Operator::Pow, op, rhs)
}