extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-02-16 18:41:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

polars open will now open a lazy frame by default (#13556)

Browse Source 
# Description
When opening a dataframe the default operation will be to create a lazy
frame if possible. This works much better with large datasets and
supports hive format.

# User-Facing Changes
- `--lazy` is nolonger a valid option. `--eager` must be used to
explicitly open an eager dataframe.
This commit is contained in:
Jack Wright 2024-08-07 04:36:08 -07:00 committed by GitHub
parent ce13ecfd10
commit ff09c7964e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
crates/nu_plugin_polars/src/dataframe/eager/open.rs
View File

@ -46,7 +46,7 @@ impl PluginCommand for OpenDataFrame {
} }
fn usage(&self) -> &str { fn usage(&self) -> &str {
"Opens CSV, JSON, JSON lines, arrow, avro, or parquet file to create dataframe." "Opens CSV, JSON, JSON lines, arrow, avro, or parquet file to create dataframe. A lazy dataframe will be created by default, if supported."
} }
fn signature(&self) -> Signature { fn signature(&self) -> Signature {
@ -56,7 +56,7 @@ impl PluginCommand for OpenDataFrame {
SyntaxShape::Filepath, SyntaxShape::Filepath,
"file path to load values from", "file path to load values from",
) )
.switch("lazy", "creates a lazy dataframe", Some('l')) .switch("eager", "Open dataframe as an eager dataframe", None)
.named( .named(
"type", "type",
SyntaxShape::String, SyntaxShape::String,
@ -173,7 +173,7 @@ fn from_parquet(
file_path: &Path, file_path: &Path,
file_span: Span, file_span: Span,
) -> Result<Value, ShellError> { ) -> Result<Value, ShellError> {
if call.has_flag("lazy")? { if !call.has_flag("eager")? {
let file: String = call.req(0)?; let file: String = call.req(0)?;
let args = ScanArgsParquet { let args = ScanArgsParquet {
n_rows: None, n_rows: None,
@ -275,7 +275,7 @@ fn from_ipc(
file_path: &Path, file_path: &Path,
file_span: Span, file_span: Span,
) -> Result<Value, ShellError> { ) -> Result<Value, ShellError> {
if call.has_flag("lazy")? { if !call.has_flag("eager")? {
let file: String = call.req(0)?; let file: String = call.req(0)?;
let args = ScanArgsIpc { let args = ScanArgsIpc {
n_rows: None, n_rows: None,
@ -389,7 +389,7 @@ fn from_jsonl(
.map(|schema| NuSchema::try_from(&schema)) .map(|schema| NuSchema::try_from(&schema))
.transpose()?; .transpose()?;
if call.has_flag("lazy")? { if !call.has_flag("eager")? {
let start_time = std::time::Instant::now(); let start_time = std::time::Instant::now();
let df = LazyJsonLineReader::new(file_path) let df = LazyJsonLineReader::new(file_path)
@ -473,7 +473,7 @@ fn from_csv(
.map(|schema| NuSchema::try_from(&schema)) .map(|schema| NuSchema::try_from(&schema))
.transpose()?; .transpose()?;
if call.has_flag("lazy")? { if !call.has_flag("eager")? {
let csv_reader = LazyCsvReader::new(file_path); let csv_reader = LazyCsvReader::new(file_path);
let csv_reader = match delimiter { let csv_reader = match delimiter {