Datetime commands (#3894)

* date and duration from nu

* date commands

* Import to feature flag

* corrected to-csv example

* corrected sample example
This commit is contained in:
Fernando Herrera
2021-08-05 23:18:53 +01:00
committed by GitHub
parent 28db8022fe
commit 63abe1cb3e
24 changed files with 975 additions and 49 deletions

View File

@ -99,9 +99,9 @@ zip = { version="0.5.9", optional=true }
digest = "0.9.0"
[dependencies.polars]
version = "0.14.8"
version = "0.15"
optional = true
features = ["parquet", "json", "random", "pivot", "strings", "is_in"]
features = ["parquet", "json", "random", "pivot", "strings", "is_in", "temporal"]
[target.'cfg(unix)'.dependencies]
umask = "1.0.0"

View File

@ -73,6 +73,16 @@ pub use series::DataFrameArgTrue;
pub use series::DataFrameArgUnique;
pub use series::DataFrameConcatenate;
pub use series::DataFrameContains;
pub use series::DataFrameGetDay;
pub use series::DataFrameGetHour;
pub use series::DataFrameGetMinute;
pub use series::DataFrameGetMonth;
pub use series::DataFrameGetNanoSecond;
pub use series::DataFrameGetOrdinal;
pub use series::DataFrameGetSecond;
pub use series::DataFrameGetWeek;
pub use series::DataFrameGetWeekDay;
pub use series::DataFrameGetYear;
pub use series::DataFrameIsDuplicated;
pub use series::DataFrameIsIn;
pub use series::DataFrameIsNotNull;
@ -87,6 +97,7 @@ pub use series::DataFrameSeriesRename;
pub use series::DataFrameSet;
pub use series::DataFrameSetWithIdx;
pub use series::DataFrameShift;
pub use series::DataFrameStrFTime;
pub use series::DataFrameStringLengths;
pub use series::DataFrameStringSlice;
pub use series::DataFrameToLowercase;

View File

@ -41,7 +41,7 @@ impl WholeStreamCommand for DataFrame {
vec![
Example {
description: "Sample rows from dataframe",
example: "[[a b]; [1 2] [3 4]] | dataframe to-df | dataframe sample -r 1",
example: "[[a b]; [1 2] [3 4]] | dataframe to-df | dataframe sample -n 1",
result: None, // No expected value because sampling is random
},
Example {

View File

@ -0,0 +1,75 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, UntaggedValue,
};
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe get-day"
}
fn usage(&self) -> &str {
"[Series] Gets day from date"
}
fn signature(&self) -> Signature {
Signature::build("dataframe get-day")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns day from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe get-day"#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![UntaggedValue::int(4).into(), UntaggedValue::int(4).into()],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.day().into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -0,0 +1,75 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, UntaggedValue,
};
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe get-hour"
}
fn usage(&self) -> &str {
"[Series] Gets hour from date"
}
fn signature(&self) -> Signature {
Signature::build("dataframe get-hour")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns hour from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe get-hour"#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![UntaggedValue::int(16).into(), UntaggedValue::int(16).into()],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.hour().into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -0,0 +1,75 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, UntaggedValue,
};
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe get-minute"
}
fn usage(&self) -> &str {
"[Series] Gets minute from date"
}
fn signature(&self) -> Signature {
Signature::build("dataframe get-minute")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns minute from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe get-minute"#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![UntaggedValue::int(39).into(), UntaggedValue::int(39).into()],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.minute().into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -0,0 +1,75 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, UntaggedValue,
};
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe get-month"
}
fn usage(&self) -> &str {
"[Series] Gets month from date"
}
fn signature(&self) -> Signature {
Signature::build("dataframe get-month")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns month from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe get-month"#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![UntaggedValue::int(8).into(), UntaggedValue::int(8).into()],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.month().into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -0,0 +1,75 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, UntaggedValue,
};
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe get-nanosecond"
}
fn usage(&self) -> &str {
"[Series] Gets nanosecond from date"
}
fn signature(&self) -> Signature {
Signature::build("dataframe get-nanosecond")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns nanosecond from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe get-nanosecond"#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![UntaggedValue::int(0).into(), UntaggedValue::int(0).into()],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.nanosecond().into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -0,0 +1,78 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, UntaggedValue,
};
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe get-ordinal"
}
fn usage(&self) -> &str {
"[Series] Gets ordinal date from date"
}
fn signature(&self) -> Signature {
Signature::build("dataframe get-ordinal")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns ordinal from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe get-ordinal"#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![
UntaggedValue::int(217).into(),
UntaggedValue::int(217).into(),
],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.ordinal().into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -0,0 +1,75 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, UntaggedValue,
};
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe get-second"
}
fn usage(&self) -> &str {
"[Series] Gets second from date"
}
fn signature(&self) -> Signature {
Signature::build("dataframe get-second")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns second from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe get-second"#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![UntaggedValue::int(18).into(), UntaggedValue::int(18).into()],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.second().into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -0,0 +1,75 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, UntaggedValue,
};
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe get-week"
}
fn usage(&self) -> &str {
"[Series] Gets week from date"
}
fn signature(&self) -> Signature {
Signature::build("dataframe get-week")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns week from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe get-week"#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![UntaggedValue::int(32).into(), UntaggedValue::int(32).into()],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.week().into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -0,0 +1,75 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, UntaggedValue,
};
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe get-weekday"
}
fn usage(&self) -> &str {
"[Series] Gets weekday from date"
}
fn signature(&self) -> Signature {
Signature::build("dataframe get-weekday")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns weekday from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe get-weekday"#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![UntaggedValue::int(1).into(), UntaggedValue::int(1).into()],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.weekday().into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -0,0 +1,78 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, UntaggedValue,
};
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe get-year"
}
fn usage(&self) -> &str {
"[Series] Gets year from date"
}
fn signature(&self) -> Signature {
Signature::build("dataframe get-year")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns year from a date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe get-year"#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![
UntaggedValue::int(2020).into(),
UntaggedValue::int(2020).into(),
],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.year().into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -7,6 +7,16 @@ pub mod arg_true;
pub mod arg_unique;
pub mod concatenate;
pub mod contains;
pub mod get_day;
pub mod get_hour;
pub mod get_minute;
pub mod get_month;
pub mod get_nanosecond;
pub mod get_ordinal;
pub mod get_second;
pub mod get_week;
pub mod get_weekday;
pub mod get_year;
pub mod is_duplicated;
pub mod is_in;
pub mod is_not_null;
@ -23,6 +33,7 @@ pub mod set_with_idx;
pub mod shift;
pub mod str_lengths;
pub mod str_slice;
pub mod strftime;
pub mod to_lowercase;
pub mod to_uppercase;
pub mod unique;
@ -37,6 +48,16 @@ pub use arg_true::DataFrame as DataFrameArgTrue;
pub use arg_unique::DataFrame as DataFrameArgUnique;
pub use concatenate::DataFrame as DataFrameConcatenate;
pub use contains::DataFrame as DataFrameContains;
pub use get_day::DataFrame as DataFrameGetDay;
pub use get_hour::DataFrame as DataFrameGetHour;
pub use get_minute::DataFrame as DataFrameGetMinute;
pub use get_month::DataFrame as DataFrameGetMonth;
pub use get_nanosecond::DataFrame as DataFrameGetNanoSecond;
pub use get_ordinal::DataFrame as DataFrameGetOrdinal;
pub use get_second::DataFrame as DataFrameGetSecond;
pub use get_week::DataFrame as DataFrameGetWeek;
pub use get_weekday::DataFrame as DataFrameGetWeekDay;
pub use get_year::DataFrame as DataFrameGetYear;
pub use is_duplicated::DataFrame as DataFrameIsDuplicated;
pub use is_in::DataFrame as DataFrameIsIn;
pub use is_not_null::DataFrame as DataFrameIsNotNull;
@ -53,6 +74,7 @@ pub use set_with_idx::DataFrame as DataFrameSetWithIdx;
pub use shift::DataFrame as DataFrameShift;
pub use str_lengths::DataFrame as DataFrameStringLengths;
pub use str_slice::DataFrame as DataFrameStringSlice;
pub use strftime::DataFrame as DataFrameStrFTime;
pub use to_lowercase::DataFrame as DataFrameToLowercase;
pub use to_uppercase::DataFrame as DataFrameToUppercase;
pub use unique::DataFrame as DataFrameUnique;

View File

@ -0,0 +1,80 @@
use crate::{commands::dataframe::utils::parse_polars_error, prelude::*};
use nu_engine::WholeStreamCommand;
use nu_errors::ShellError;
use nu_protocol::{
dataframe::{Column, NuDataFrame},
Signature, SyntaxShape, UntaggedValue,
};
use nu_source::Tagged;
use polars::prelude::IntoSeries;
pub struct DataFrame;
impl WholeStreamCommand for DataFrame {
fn name(&self) -> &str {
"dataframe strftime"
}
fn usage(&self) -> &str {
"[Series] Formats date based on string rule"
}
fn signature(&self) -> Signature {
Signature::build("dataframe strftime").required("fmt", SyntaxShape::String, "Format rule")
}
fn run(&self, args: CommandArgs) -> Result<OutputStream, ShellError> {
command(args)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Formats date",
example: r#"let dt = ('2020-08-04T16:39:18+00:00' | str to-datetime -z 'UTC');
let df = ([$dt $dt] | dataframe to-df);
$df | dataframe strftime "%Y/%m/%d""#,
result: Some(vec![NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![
UntaggedValue::string("2020/08/04").into(),
UntaggedValue::string("2020/08/04").into(),
],
)],
&Span::default(),
)
.expect("simple df for test should not fail")
.into_value(Tag::default())]),
}]
}
}
fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
let tag = args.call_info.name_tag.clone();
let fmt: Tagged<String> = args.req(0)?;
let (df, df_tag) = NuDataFrame::try_from_stream(&mut args.input, &tag.span)?;
let series = df.as_series(&df_tag.span)?;
let casted = series
.date64()
.map_err(|e| parse_polars_error::<&str>(&e, &df_tag.span, None))?;
let res = casted.strftime(fmt.item.as_str()).into_series();
let df = NuDataFrame::try_from_series(vec![res], &tag.span)?;
Ok(OutputStream::one(df.into_value(df_tag)))
}
#[cfg(test)]
mod tests {
use super::DataFrame;
use super::ShellError;
#[test]
fn examples_work_as_expected() -> Result<(), ShellError> {
use crate::examples::test_dataframe as test_examples;
test_examples(DataFrame {})
}
}

View File

@ -111,9 +111,12 @@ fn command(mut args: CommandArgs) -> Result<OutputStream, ShellError> {
ShellError::labeled_error("Empty stream", "No value found in the stream", &tag)
})?;
match value.value {
match &value.value {
UntaggedValue::DataFrame(df) => {
let res = df.as_ref().take(indices);
let res = df
.as_ref()
.take(indices)
.map_err(|e| parse_polars_error::<&str>(&e, &value.tag.span, None))?;
Ok(OutputStream::one(NuDataFrame::dataframe_to_value(res, tag)))
}

View File

@ -45,7 +45,7 @@ impl WholeStreamCommand for DataFrame {
vec![
Example {
description: "Saves dataframe to csv file",
example: "[[a b]; [1 2] [3 4]] | dataframe to-df | dataframe to_csv test.csv",
example: "[[a b]; [1 2] [3 4]] | dataframe to-df | dataframe to-csv test.csv",
result: None,
},
Example {

View File

@ -29,14 +29,17 @@ pub use dataframe::{
DataFrameArgMax, DataFrameArgMin, DataFrameArgSort, DataFrameArgTrue, DataFrameArgUnique,
DataFrameColumn, DataFrameConcatenate, DataFrameContains, DataFrameDTypes, DataFrameDrop,
DataFrameDropDuplicates, DataFrameDropNulls, DataFrameDummies, DataFrameFilter, DataFrameFirst,
DataFrameGet, DataFrameGroupBy, DataFrameIsDuplicated, DataFrameIsIn, DataFrameIsNotNull,
DataFrameIsNull, DataFrameIsUnique, DataFrameJoin, DataFrameLast, DataFrameList, DataFrameMelt,
DataFrameNNull, DataFrameNUnique, DataFrameNot, DataFrameOpen, DataFramePivot,
DataFrameReplace, DataFrameReplaceAll, DataFrameSample, DataFrameSelect, DataFrameSeriesRename,
DataFrameSet, DataFrameSetWithIdx, DataFrameShape, DataFrameShift, DataFrameShow,
DataFrameSlice, DataFrameSort, DataFrameStringLengths, DataFrameStringSlice, DataFrameTake,
DataFrameToCsv, DataFrameToDF, DataFrameToLowercase, DataFrameToParquet, DataFrameToUppercase,
DataFrameUnique, DataFrameValueCounts, DataFrameWhere, DataFrameWithColumn,
DataFrameGet, DataFrameGetDay, DataFrameGetHour, DataFrameGetMinute, DataFrameGetMonth,
DataFrameGetNanoSecond, DataFrameGetOrdinal, DataFrameGetSecond, DataFrameGetWeek,
DataFrameGetWeekDay, DataFrameGetYear, DataFrameGroupBy, DataFrameIsDuplicated, DataFrameIsIn,
DataFrameIsNotNull, DataFrameIsNull, DataFrameIsUnique, DataFrameJoin, DataFrameLast,
DataFrameList, DataFrameMelt, DataFrameNNull, DataFrameNUnique, DataFrameNot, DataFrameOpen,
DataFramePivot, DataFrameReplace, DataFrameReplaceAll, DataFrameSample, DataFrameSelect,
DataFrameSeriesRename, DataFrameSet, DataFrameSetWithIdx, DataFrameShape, DataFrameShift,
DataFrameShow, DataFrameSlice, DataFrameSort, DataFrameStrFTime, DataFrameStringLengths,
DataFrameStringSlice, DataFrameTake, DataFrameToCsv, DataFrameToDF, DataFrameToLowercase,
DataFrameToParquet, DataFrameToUppercase, DataFrameUnique, DataFrameValueCounts,
DataFrameWhere, DataFrameWithColumn,
};
pub use env::*;
pub use filesystem::*;

View File

@ -329,6 +329,17 @@ pub fn create_default_context(interactive: bool) -> Result<EvaluationContext, Bo
whole_stream_command(DataFrameStringSlice),
whole_stream_command(DataFrameConcatenate),
whole_stream_command(DataFrameAppend),
whole_stream_command(DataFrameGetHour),
whole_stream_command(DataFrameGetMinute),
whole_stream_command(DataFrameGetSecond),
whole_stream_command(DataFrameGetDay),
whole_stream_command(DataFrameGetMonth),
whole_stream_command(DataFrameGetYear),
whole_stream_command(DataFrameGetWeek),
whole_stream_command(DataFrameGetWeekDay),
whole_stream_command(DataFrameGetOrdinal),
whole_stream_command(DataFrameGetNanoSecond),
whole_stream_command(DataFrameStrFTime),
]);
#[cfg(feature = "clipboard-cli")]

View File

@ -17,6 +17,7 @@ use nu_source::AnchorLocation;
#[cfg(feature = "dataframe")]
use crate::commands::{
DataFrameGroupBy, DataFrameIsNull, DataFrameShift, DataFrameToDF, DataFrameWithColumn,
StrToDatetime,
};
use crate::commands::{
@ -182,6 +183,7 @@ pub fn test_dataframe(cmd: impl WholeStreamCommand + 'static) -> Result<(), Shel
whole_stream_command(Select),
whole_stream_command(StrCollect),
whole_stream_command(Wrap),
whole_stream_command(StrToDatetime),
]);
for sample_pipeline in examples {

View File

@ -32,9 +32,9 @@ serde_yaml = "0.8.16"
toml = "0.5.8"
[dependencies.polars]
version = "0.14.8"
version = "0.15"
optional = true
features = ["default", "serde", "rows", "strings", "checked_arithmetic", "object"]
features = ["default", "serde", "rows", "strings", "checked_arithmetic", "object", "dtype-duration-ns"]
[features]
dataframe = ["polars"]

View File

@ -2,13 +2,14 @@ use indexmap::map::{Entry, IndexMap};
use polars::chunked_array::object::builder::ObjectChunkedBuilder;
use polars::chunked_array::ChunkedArray;
use bigdecimal::FromPrimitive;
use bigdecimal::{FromPrimitive, ToPrimitive};
use chrono::{DateTime, FixedOffset, NaiveDateTime};
use nu_errors::ShellError;
use nu_source::{Span, Tag};
use num_bigint::BigInt;
use polars::prelude::{
DataFrame, DataType, IntoSeries, NamedFrom, ObjectType, PolarsNumericType, Series, TimeUnit,
DataFrame, DataType, Date64Type, Int64Type, IntoSeries, NamedFrom, NewChunkedArray, ObjectType,
PolarsNumericType, Series, TimeUnit,
};
use std::ops::{Deref, DerefMut};
@ -74,6 +75,8 @@ pub enum InputType {
String,
Boolean,
Object,
Date,
Duration,
}
#[derive(Debug)]
@ -528,6 +531,12 @@ pub fn insert_value(
UntaggedValue::Primitive(Primitive::Boolean(_)) => {
col_val.column_type = Some(InputType::Boolean);
}
UntaggedValue::Primitive(Primitive::Date(_)) => {
col_val.column_type = Some(InputType::Date);
}
UntaggedValue::Primitive(Primitive::Duration(_)) => {
col_val.column_type = Some(InputType::Duration);
}
_ => col_val.column_type = Some(InputType::Object),
}
col_val.values.push(value);
@ -550,6 +559,14 @@ pub fn insert_value(
| (
UntaggedValue::Primitive(Primitive::Boolean(_)),
UntaggedValue::Primitive(Primitive::Boolean(_)),
)
| (
UntaggedValue::Primitive(Primitive::Date(_)),
UntaggedValue::Primitive(Primitive::Date(_)),
)
| (
UntaggedValue::Primitive(Primitive::Duration(_)),
UntaggedValue::Primitive(Primitive::Duration(_)),
) => col_val.values.push(value),
_ => {
col_val.column_type = Some(InputType::Object);
@ -607,6 +624,32 @@ pub fn from_parsed_columns(
let res = builder.finish();
df_series.push(res.into_series())
}
InputType::Date => {
let it = column.values.iter().map(|v| {
if let UntaggedValue::Primitive(Primitive::Date(date)) = &v.value {
Some(date.timestamp_millis())
} else {
None
}
});
let res = ChunkedArray::<Date64Type>::new_from_opt_iter(&name, it);
df_series.push(res.into_series())
}
InputType::Duration => {
let it = column.values.iter().map(|v| {
if let UntaggedValue::Primitive(Primitive::Duration(duration)) = &v.value {
Some(duration.to_i64().expect("Not expecting NAN in duration"))
} else {
None
}
});
let res = ChunkedArray::<Int64Type>::new_from_opt_iter(&name, it);
df_series.push(res.into_series())
}
}
}
}

View File

@ -174,6 +174,7 @@ impl NuDataFrame {
| UntaggedValue::Primitive(Primitive::Decimal(_))
| UntaggedValue::Primitive(Primitive::String(_))
| UntaggedValue::Primitive(Primitive::Boolean(_))
| UntaggedValue::Primitive(Primitive::Date(_))
| UntaggedValue::DataFrame(_) => {
let key = format!("{}", 0);
insert_value(value, key, &mut column_values)?