Merge branch 'main' into ecow-record

Cargo.lock

@@ -3429,6 +3429,7 @@ dependencies = [
  "polars",
  "polars-arrow",
  "polars-io",
+ "polars-lazy",
  "polars-ops",
  "polars-plan",
  "polars-utils",

crates/nu-cmd-lang/src/core_commands/describe.rs

@@ -158,10 +158,10 @@ fn run(
     input: PipelineData,
     options: Options,
 ) -> Result<PipelineData, ShellError> {
-    let metadata = input.metadata().clone().map(Box::new);
     let head = call.head;
+    let metadata = input.metadata();
 
-    let description: Value = match input {
+    let description = match input {
         PipelineData::ExternalStream {
             ref stdout,
             ref stderr,
@@ -169,45 +169,54 @@ fn run(
             ..
         } => {
             if options.detailed {
+                let stdout = if stdout.is_some() {
                     Value::record(
-                    record!(
-                        "type" => Value::string("stream", head),
-                        "origin" => Value::string("external", head),
-                        "stdout" => match stdout {
-                            Some(_) => Value::record(
-                                    record!(
+                        record! {
                             "type" => Value::string("stream", head),
                             "origin" => Value::string("external", head),
                             "subtype" => Value::string("any", head),
-                                    ),
-                                    head,
-                                ),
-                            None => Value::nothing(head),
                         },
-                        "stderr" => match stderr {
-                            Some(_) => Value::record(
-                                    record!(
+                        head,
+                    )
+                } else {
+                    Value::nothing(head)
+                };
+
+                let stderr = if stderr.is_some() {
+                    Value::record(
+                        record! {
                             "type" => Value::string("stream", head),
                             "origin" => Value::string("external", head),
                             "subtype" => Value::string("any", head),
-                                    ),
-                                    head,
-                                ),
-                            None => Value::nothing(head),
                         },
-                        "exit_code" => match exit_code {
-                            Some(_) => Value::record(
-                                    record!(
+                        head,
+                    )
+                } else {
+                    Value::nothing(head)
+                };
+
+                let exit_code = if exit_code.is_some() {
+                    Value::record(
+                        record! {
                             "type" => Value::string("stream", head),
                             "origin" => Value::string("external", head),
                             "subtype" => Value::string("int", head),
-                                    ),
-                                    head,
-                                ),
-                            None => Value::nothing(head),
                         },
+                        head,
+                    )
+                } else {
+                    Value::nothing(head)
+                };
+
+                Value::record(
+                    record! {
+                        "type" => Value::string("stream", head),
+                        "origin" => Value::string("external", head),
+                        "stdout" => stdout,
+                        "stderr" => stderr,
+                        "exit_code" => exit_code,
                         "metadata" => metadata_to_value(metadata, head),
-                    ),
+                    },
                     head,
                 )
             } else {
@@ -216,19 +225,18 @@ fn run(
         }
         PipelineData::ListStream(_, _) => {
             if options.detailed {
-                Value::record(
-                    record!(
-                        "type" => Value::string("stream", head),
-                        "origin" => Value::string("nushell", head),
-                        "subtype" => {
-                           if options.no_collect {
+                let subtype = if options.no_collect {
                     Value::string("any", head)
                 } else {
-                            describe_value(input.into_value(head), head, engine_state, )
-                           }
-                        },
+                    describe_value(input.into_value(head), head, engine_state)
+                };
+                Value::record(
+                    record! {
+                        "type" => Value::string("stream", head),
+                        "origin" => Value::string("nushell", head),
+                        "subtype" => subtype,
                         "metadata" => metadata_to_value(metadata, head),
-                    ),
+                    },
                     head,
                 )
             } else if options.no_collect {
@@ -236,7 +244,6 @@ fn run(
             } else {
                 let value = input.into_value(head);
                 let base_description = value.get_type().to_string();
-
                 Value::string(format!("{} (stream)", base_description), head)
             }
         }
@@ -253,31 +260,34 @@ fn run(
     Ok(description.into_pipeline_data())
 }
 
-fn compact_primitive_description(mut value: Value) -> Value {
-    if let Value::Record { ref mut val, .. } = value {
-        if val.len() != 1 {
-            return value;
-        }
-        if let Some(type_name) = val.get_mut("type") {
-            return std::mem::take(type_name);
-        }
-    }
-    value
+enum Description {
+    String(String),
+    Record(Record),
 }
 
-fn describe_value(
+impl Description {
+    fn into_value(self, span: Span) -> Value {
+        match self {
+            Description::String(ty) => Value::string(ty, span),
+            Description::Record(record) => Value::record(record, span),
+        }
+    }
+}
+
+fn describe_value(value: Value, head: Span, engine_state: Option<&EngineState>) -> Value {
+    let record = match describe_value_inner(value, head, engine_state) {
+        Description::String(ty) => record! { "type" => Value::string(ty, head) },
+        Description::Record(record) => record,
+    };
+    Value::record(record, head)
+}
+
+fn describe_value_inner(
     value: Value,
-    head: nu_protocol::Span,
+    head: Span,
     engine_state: Option<&EngineState>,
-) -> Value {
+) -> Description {
     match value {
-        Value::Custom { val, .. } => Value::record(
-            record!(
-                "type" => Value::string("custom", head),
-                "subtype" => Value::string(val.type_name(), head),
-            ),
-            head,
-        ),
         Value::Bool { .. }
         | Value::Int { .. }
         | Value::Float { .. }
@@ -287,100 +297,73 @@ fn describe_value(
         | Value::Range { .. }
         | Value::String { .. }
         | Value::Glob { .. }
-        | Value::Nothing { .. } => Value::record(
-            record!(
-                "type" => Value::string(value.get_type().to_string(), head),
-            ),
-            head,
-        ),
+        | Value::Nothing { .. } => Description::String(value.get_type().to_string()),
         Value::Record { mut val, .. } => {
-            for (_k, v) in val.iter_mut() {
-                *v = compact_primitive_description(describe_value(
-                    std::mem::take(v),
-                    head,
-                    engine_state,
-                ));
+            for (_, val) in &mut val {
+                *val =
+                    describe_value_inner(std::mem::take(val), head, engine_state).into_value(head);
             }
 
-            Value::record(
-                record!(
+            Description::Record(record! {
                 "type" => Value::string("record", head),
                 "columns" => Value::record(val, head),
-                ),
-                head,
-            )
+            })
         }
-        Value::List { vals, .. } => Value::record(
-            record!(
+        Value::List { mut vals, .. } => {
+            for val in &mut vals {
+                *val =
+                    describe_value_inner(std::mem::take(val), head, engine_state).into_value(head);
+            }
+
+            Description::Record(record! {
                 "type" => Value::string("list", head),
                 "length" => Value::int(vals.len() as i64, head),
-                "values" => Value::list(vals.into_iter().map(|v|
-                    compact_primitive_description(describe_value(v, head, engine_state))
-                )
-                .collect(), head),
-            ),
-            head,
-        ),
+                "values" => Value::list(vals, head),
+            })
+        }
         Value::Closure { val, .. } => {
             let block = engine_state.map(|engine_state| engine_state.get_block(val.block_id));
 
+            let mut record = record! { "type" => Value::string("closure", head) };
             if let Some(block) = block {
-                let mut record = Record::new();
-                record.push("type", Value::string("closure", head));
                 record.push(
                     "signature",
                     Value::record(
-                        record!(
+                        record! {
                             "name" => Value::string(block.signature.name.clone(), head),
                             "category" => Value::string(block.signature.category.to_string(), head),
-                        ),
+                        },
                         head,
                     ),
                 );
-                Value::record(record, head)
-            } else {
-                Value::record(
-                    record!(
-                        "type" => Value::string("closure", head),
-                    ),
-                    head,
-                )
             }
+            Description::Record(record)
         }
-
-        Value::Error { error, .. } => Value::record(
-            record!(
+        Value::Error { error, .. } => Description::Record(record! {
             "type" => Value::string("error", head),
             "subtype" => Value::string(error.to_string(), head),
-            ),
-            head,
-        ),
-        Value::Binary { val, .. } => Value::record(
-            record!(
+        }),
+        Value::Binary { val, .. } => Description::Record(record! {
             "type" => Value::string("binary", head),
             "length" => Value::int(val.len() as i64, head),
-            ),
-            head,
-        ),
-        Value::CellPath { val, .. } => Value::record(
-            record!(
-                "type" => Value::string("cellpath", head),
+        }),
+        Value::CellPath { val, .. } => Description::Record(record! {
+            "type" => Value::string("cell-path", head),
             "length" => Value::int(val.members.len() as i64, head),
-            ),
-            head,
-        ),
+        }),
+        Value::Custom { val, .. } => Description::Record(record! {
+            "type" => Value::string("custom", head),
+            "subtype" => Value::string(val.type_name(), head),
+        }),
     }
 }
 
-fn metadata_to_value(metadata: Option<Box<PipelineMetadata>>, head: nu_protocol::Span) -> Value {
-    match metadata {
-        Some(metadata) => Value::record(
-            record!(
-                "data_source" => Value::string(format!("{:?}", metadata.data_source), head),
-            ),
-            head,
-        ),
-        _ => Value::nothing(head),
+fn metadata_to_value(metadata: Option<PipelineMetadata>, head: Span) -> Value {
+    if let Some(metadata) = metadata {
+        let data_source = Value::string(format!("{:?}", metadata.data_source), head);
+        Value::record(record! { "data_source" => data_source }, head)
+    } else {
+        Value::nothing(head)
     }
 }
 

crates/nu-command/src/default_context.rs

@@ -119,6 +119,12 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState {
             Exec,
             NuCheck,
             Sys,
+            SysCpu,
+            SysDisks,
+            SysHost,
+            SysMem,
+            SysNet,
+            SysTemp,
             UName,
 
         };

crates/nu-command/src/filesystem/ls.rs

@@ -1,9 +1,10 @@
 use super::util::get_rest_for_glob_pattern;
 use crate::{DirBuilder, DirInfo};
 use chrono::{DateTime, Local, LocalResult, TimeZone, Utc};
+use nu_engine::glob_from;
 #[allow(deprecated)]
 use nu_engine::{command_prelude::*, env::current_dir};
-use nu_glob::{MatchOptions, Pattern};
+use nu_glob::MatchOptions;
 use nu_path::expand_to_real_path;
 use nu_protocol::{DataSource, NuGlob, PipelineMetadata};
 use pathdiff::diff_paths;
@@ -31,6 +32,8 @@ struct Args {
     call_span: Span,
 }
 
+const GLOB_CHARS: &[char] = &['*', '?', '['];
+
 impl Command for Ls {
     fn name(&self) -> &str {
         "ls"
@@ -235,24 +238,20 @@ fn ls_for_one_pattern(
         }
     };
 
-    // it indicates we need to append an extra '*' after pattern for listing given directory
-    // Example: 'ls directory' -> 'ls directory/*'
-    let mut extra_star_under_given_directory = false;
-    let (path, p_tag, absolute_path, quoted) = match pattern_arg {
+    let mut just_read_dir = false;
+    let p_tag: Span = pattern_arg.as_ref().map(|p| p.span).unwrap_or(call_span);
+    let (pattern_arg, absolute_path) = match pattern_arg {
         Some(pat) => {
-            let p_tag = pat.span;
-            let expanded = nu_path::expand_path_with(
-                pat.item.as_ref(),
-                &cwd,
-                matches!(pat.item, NuGlob::Expand(..)),
-            );
+            // expand with cwd here is only used for checking
+            let tmp_expanded =
+                nu_path::expand_path_with(pat.item.as_ref(), &cwd, pat.item.is_expand());
             // Avoid checking and pushing "*" to the path when directory (do not show contents) flag is true
-            if !directory && expanded.is_dir() {
-                if permission_denied(&expanded) {
+            if !directory && tmp_expanded.is_dir() {
+                if permission_denied(&tmp_expanded) {
                     #[cfg(unix)]
                     let error_msg = format!(
                         "The permissions of {:o} do not allow access for this user",
-                        expanded
+                        tmp_expanded
                             .metadata()
                             .expect("this shouldn't be called since we already know there is a dir")
                             .permissions()
@@ -269,10 +268,10 @@ fn ls_for_one_pattern(
                         inner: vec![],
                     });
                 }
-                if is_empty_dir(&expanded) {
+                if is_empty_dir(&tmp_expanded) {
                     return Ok(Box::new(vec![].into_iter()));
                 }
-                extra_star_under_given_directory = true;
+                just_read_dir = !(pat.item.is_expand() && pat.item.as_ref().contains(GLOB_CHARS));
             }
 
             // it's absolute path if:
@@ -282,52 +281,28 @@ fn ls_for_one_pattern(
             //    path.
             let absolute_path = Path::new(pat.item.as_ref()).is_absolute()
                 || (pat.item.is_expand() && expand_to_real_path(pat.item.as_ref()).is_absolute());
-            (
-                expanded,
-                p_tag,
-                absolute_path,
-                matches!(pat.item, NuGlob::DoNotExpand(_)),
-            )
+            (pat.item, absolute_path)
         }
         None => {
             // Avoid pushing "*" to the default path when directory (do not show contents) flag is true
             if directory {
-                (PathBuf::from("."), call_span, false, false)
+                (NuGlob::Expand(".".to_string()), false)
             } else if is_empty_dir(&cwd) {
                 return Ok(Box::new(vec![].into_iter()));
             } else {
-                (PathBuf::from("*"), call_span, false, false)
+                (NuGlob::Expand("*".to_string()), false)
             }
         }
     };
 
-    let hidden_dir_specified = is_hidden_dir(&path);
-    // when it's quoted, we need to escape our glob pattern(but without the last extra
-    // start which may be added under given directory)
-    // so we can do ls for a file or directory like `a[123]b`
-    let path = if quoted {
-        let p = path.display().to_string();
-        let mut glob_escaped = Pattern::escape(&p);
-        if extra_star_under_given_directory {
-            glob_escaped.push(std::path::MAIN_SEPARATOR);
-            glob_escaped.push('*');
-        }
-        glob_escaped
+    let hidden_dir_specified = is_hidden_dir(pattern_arg.as_ref());
+    let path = pattern_arg.into_spanned(p_tag);
+    let (prefix, paths) = if just_read_dir {
+        let expanded = nu_path::expand_path_with(path.item.as_ref(), &cwd, path.item.is_expand());
+        let paths = read_dir(&expanded)?;
+        // just need to read the directory, so prefix is path itself.
+        (Some(expanded), paths)
     } else {
-        let mut p = path.display().to_string();
-        if extra_star_under_given_directory {
-            p.push(std::path::MAIN_SEPARATOR);
-            p.push('*');
-        }
-        p
-    };
-
-    let glob_path = Spanned {
-        // use NeedExpand, the relative escaping logic is handled previously
-        item: NuGlob::Expand(path.clone()),
-        span: p_tag,
-    };
-
         let glob_options = if all {
             None
         } else {
@@ -337,12 +312,13 @@ fn ls_for_one_pattern(
             };
             Some(glob_options)
         };
-    let (prefix, paths) = nu_engine::glob_from(&glob_path, &cwd, call_span, glob_options)?;
+        glob_from(&path, &cwd, call_span, glob_options)?
+    };
 
     let mut paths_peek = paths.peekable();
     if paths_peek.peek().is_none() {
         return Err(ShellError::GenericError {
-            error: format!("No matches found for {}", &path),
+            error: format!("No matches found for {:?}", path.item),
             msg: "Pattern, file or folder not found".into(),
             span: Some(p_tag),
             help: Some("no matches found".into()),
@@ -904,3 +880,14 @@ mod windows_helper {
         false
     }
 }
+
+#[allow(clippy::type_complexity)]
+fn read_dir(
+    f: &Path,
+) -> Result<Box<dyn Iterator<Item = Result<PathBuf, ShellError>> + Send>, ShellError> {
+    let iter = f.read_dir()?.map(|d| {
+        d.map(|r| r.path())
+            .map_err(|e| ShellError::IOError { msg: e.to_string() })
+    });
+    Ok(Box::new(iter))
+}

crates/nu-command/src/filters/get.rs

@@ -124,11 +124,6 @@ If multiple cell paths are given, this will produce a list of values."#
                 example: "ls | get 2.name",
                 result: None,
             },
-            Example {
-                description: "Extract the cpu list from the sys information record",
-                example: "sys | get cpu",
-                result: None,
-            },
             Example {
                 description: "Getting Path/PATH in a case insensitive way",
                 example: "$env | get paTH",

crates/nu-command/src/help/help_.rs

@@ -66,8 +66,8 @@ Each stage in the pipeline works together to load, parse, and display informatio
 List the files in the current directory, sorted by size:
     ls | sort-by size
 
-Get information about the current system:
-    sys | get host
+Get the current system host name:
+    sys host | get hostname
 
 Get the processes on your system actively using CPU:
     ps | where cpu > 0

crates/nu-command/src/system/mod.rs

@@ -30,6 +30,6 @@ pub use ps::Ps;
 #[cfg(windows)]
 pub use registry_query::RegistryQuery;
 pub use run_external::{External, ExternalCommand};
-pub use sys::Sys;
+pub use sys::*;
 pub use uname::UName;
 pub use which_::Which;

crates/nu-command/src/system/sys.rs

@@ -1,269 +0,0 @@
-use chrono::{DateTime, Local};
-use nu_engine::command_prelude::*;
-use std::time::{Duration, UNIX_EPOCH};
-use sysinfo::{
-    Components, CpuRefreshKind, Disks, Networks, System, Users, MINIMUM_CPU_UPDATE_INTERVAL,
-};
-
-#[derive(Clone)]
-pub struct Sys;
-
-impl Command for Sys {
-    fn name(&self) -> &str {
-        "sys"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("sys")
-            .filter()
-            .category(Category::System)
-            .input_output_types(vec![(Type::Nothing, Type::record())])
-    }
-
-    fn usage(&self) -> &str {
-        "View information about the system."
-    }
-
-    fn run(
-        &self,
-        _engine_state: &EngineState,
-        _stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        Ok(all_columns(call.head).into_pipeline_data())
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Show info about the system",
-                example: "sys",
-                result: None,
-            },
-            Example {
-                description: "Show the os system name with get",
-                example: "(sys).host | get name",
-                result: None,
-            },
-            Example {
-                description: "Show the os system name",
-                example: "(sys).host.name",
-                result: None,
-            },
-        ]
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct SysResult {
-    pub span: Span,
-}
-
-fn all_columns(span: Span) -> Value {
-    Value::record(
-        record! {
-            "host" => host(span),
-            "cpu" => cpu(span),
-            "disks" => disks(span),
-            "mem" => mem(span),
-            "temp" => temp(span),
-            "net" => net(span),
-        },
-        span,
-    )
-}
-
-pub fn trim_cstyle_null(s: String) -> String {
-    s.trim_matches(char::from(0)).to_string()
-}
-
-pub fn disks(span: Span) -> Value {
-    let disks = Disks::new_with_refreshed_list();
-
-    let mut output = vec![];
-    for disk in disks.list() {
-        let device = trim_cstyle_null(disk.name().to_string_lossy().to_string());
-        let typ = trim_cstyle_null(disk.file_system().to_string_lossy().to_string());
-
-        let record = record! {
-            "device" => Value::string(device, span),
-            "type" => Value::string(typ, span),
-            "mount" => Value::string(disk.mount_point().to_string_lossy(), span),
-            "total" => Value::filesize(disk.total_space() as i64, span),
-            "free" => Value::filesize(disk.available_space() as i64, span),
-            "removable" => Value::bool(disk.is_removable(), span),
-            "kind" => Value::string(format!("{:?}", disk.kind()), span),
-        };
-
-        output.push(Value::record(record, span));
-    }
-    Value::list(output, span)
-}
-
-pub fn net(span: Span) -> Value {
-    let networks = Networks::new_with_refreshed_list();
-
-    let mut output = vec![];
-    for (iface, data) in networks.list() {
-        let record = record! {
-            "name" => Value::string(trim_cstyle_null(iface.to_string()), span),
-            "sent" => Value::filesize(data.total_transmitted() as i64, span),
-            "recv" => Value::filesize(data.total_received() as i64, span),
-        };
-
-        output.push(Value::record(record, span));
-    }
-    Value::list(output, span)
-}
-
-pub fn cpu(span: Span) -> Value {
-    let mut sys = System::new();
-    sys.refresh_cpu_specifics(CpuRefreshKind::everything());
-    // We must refresh the CPU twice a while apart to get valid usage data.
-    // In theory we could just sleep MINIMUM_CPU_UPDATE_INTERVAL, but I've noticed that
-    // that gives poor results (error of ~5%). Decided to wait 2x that long, somewhat arbitrarily
-    std::thread::sleep(MINIMUM_CPU_UPDATE_INTERVAL * 2);
-    sys.refresh_cpu_specifics(CpuRefreshKind::new().with_cpu_usage());
-
-    let mut output = vec![];
-    for cpu in sys.cpus() {
-        // sysinfo CPU usage numbers are not very precise unless you wait a long time between refreshes.
-        // Round to 1DP (chosen somewhat arbitrarily) so people aren't misled by high-precision floats.
-        let rounded_usage = (cpu.cpu_usage() * 10.0).round() / 10.0;
-
-        let load_avg = System::load_average();
-        let load_avg = trim_cstyle_null(format!(
-            "{:.2}, {:.2}, {:.2}",
-            load_avg.one, load_avg.five, load_avg.fifteen
-        ));
-
-        let record = record! {
-            "name" => Value::string(trim_cstyle_null(cpu.name().to_string()), span),
-            "brand" => Value::string(trim_cstyle_null(cpu.brand().to_string()), span),
-            "freq" => Value::int(cpu.frequency() as i64, span),
-            "cpu_usage" => Value::float(rounded_usage as f64, span),
-            "load_average" => Value::string(load_avg, span),
-            "vendor_id" => Value::string(trim_cstyle_null(cpu.vendor_id().to_string()), span),
-        };
-
-        output.push(Value::record(record, span));
-    }
-
-    Value::list(output, span)
-}
-
-pub fn mem(span: Span) -> Value {
-    let mut sys = System::new();
-    sys.refresh_memory();
-
-    let total_mem = sys.total_memory();
-    let free_mem = sys.free_memory();
-    let used_mem = sys.used_memory();
-    let avail_mem = sys.available_memory();
-
-    let total_swap = sys.total_swap();
-    let free_swap = sys.free_swap();
-    let used_swap = sys.used_swap();
-
-    let record = record! {
-        "total" => Value::filesize(total_mem as i64, span),
-        "free" => Value::filesize(free_mem as i64, span),
-        "used" => Value::filesize(used_mem as i64, span),
-        "available" => Value::filesize(avail_mem as i64, span),
-        "swap total" => Value::filesize(total_swap as i64, span),
-        "swap free" => Value::filesize(free_swap as i64, span),
-        "swap used" => Value::filesize(used_swap as i64, span),
-    };
-
-    Value::record(record, span)
-}
-
-pub fn host(span: Span) -> Value {
-    let mut record = Record::new();
-
-    if let Some(name) = System::name() {
-        record.push("name", Value::string(trim_cstyle_null(name), span));
-    }
-    if let Some(version) = System::os_version() {
-        record.push("os_version", Value::string(trim_cstyle_null(version), span));
-    }
-
-    if let Some(long_version) = System::long_os_version() {
-        record.push(
-            "long_os_version",
-            Value::string(trim_cstyle_null(long_version), span),
-        );
-    }
-
-    if let Some(version) = System::kernel_version() {
-        record.push(
-            "kernel_version",
-            Value::string(trim_cstyle_null(version), span),
-        );
-    }
-    if let Some(hostname) = System::host_name() {
-        record.push("hostname", Value::string(trim_cstyle_null(hostname), span));
-    }
-
-    record.push(
-        "uptime",
-        Value::duration(1000000000 * System::uptime() as i64, span),
-    );
-
-    // Creates a new SystemTime from the specified number of whole seconds
-    let d = UNIX_EPOCH + Duration::from_secs(System::boot_time());
-    // Create DateTime from SystemTime
-    let datetime = DateTime::<Local>::from(d);
-    // Convert to local time and then rfc3339
-    let timestamp_str = datetime.with_timezone(datetime.offset()).to_rfc3339();
-
-    record.push("boot_time", Value::string(timestamp_str, span));
-
-    let users = Users::new_with_refreshed_list();
-
-    let mut users_list = vec![];
-    for user in users.list() {
-        let mut groups = vec![];
-        for group in user.groups() {
-            groups.push(Value::string(
-                trim_cstyle_null(group.name().to_string()),
-                span,
-            ));
-        }
-
-        let record = record! {
-            "name" => Value::string(trim_cstyle_null(user.name().to_string()), span),
-            "groups" => Value::list(groups, span),
-        };
-
-        users_list.push(Value::record(record, span));
-    }
-
-    if !users.is_empty() {
-        record.push("sessions", Value::list(users_list, span));
-    }
-
-    Value::record(record, span)
-}
-
-pub fn temp(span: Span) -> Value {
-    let components = Components::new_with_refreshed_list();
-
-    let mut output = vec![];
-
-    for component in components.list() {
-        let mut record = record! {
-            "unit" => Value::string(component.label(), span),
-            "temp" => Value::float(component.temperature() as f64, span),
-            "high" => Value::float(component.max() as f64, span),
-        };
-
-        if let Some(critical) = component.critical() {
-            record.push("critical", Value::float(critical as f64, span));
-        }
-        output.push(Value::record(record, span));
-    }
-
-    Value::list(output, span)
-}

crates/nu-command/src/system/sys/cpu.rs

@@ -0,0 +1,39 @@
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct SysCpu;
+
+impl Command for SysCpu {
+    fn name(&self) -> &str {
+        "sys cpu"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("sys cpu")
+            .filter()
+            .category(Category::System)
+            .input_output_types(vec![(Type::Nothing, Type::table())])
+    }
+
+    fn usage(&self) -> &str {
+        "View information about the system CPUs."
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(super::cpu(call.head).into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Show info about the system CPUs",
+            example: "sys cpu",
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/system/sys/disks.rs

@@ -0,0 +1,39 @@
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct SysDisks;
+
+impl Command for SysDisks {
+    fn name(&self) -> &str {
+        "sys disks"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("sys disks")
+            .filter()
+            .category(Category::System)
+            .input_output_types(vec![(Type::Nothing, Type::table())])
+    }
+
+    fn usage(&self) -> &str {
+        "View information about the system disks."
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(super::disks(call.head).into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Show info about the system disks",
+            example: "sys disks",
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/system/sys/host.rs

@@ -0,0 +1,39 @@
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct SysHost;
+
+impl Command for SysHost {
+    fn name(&self) -> &str {
+        "sys host"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("sys host")
+            .filter()
+            .category(Category::System)
+            .input_output_types(vec![(Type::Nothing, Type::record())])
+    }
+
+    fn usage(&self) -> &str {
+        "View information about the system host."
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(super::host(call.head).into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Show info about the system host",
+            example: "sys host",
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/system/sys/mem.rs

@@ -0,0 +1,39 @@
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct SysMem;
+
+impl Command for SysMem {
+    fn name(&self) -> &str {
+        "sys mem"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("sys mem")
+            .filter()
+            .category(Category::System)
+            .input_output_types(vec![(Type::Nothing, Type::record())])
+    }
+
+    fn usage(&self) -> &str {
+        "View information about the system memory."
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(super::mem(call.head).into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Show info about the system memory",
+            example: "sys mem",
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/system/sys/mod.rs

@@ -0,0 +1,205 @@
+mod cpu;
+mod disks;
+mod host;
+mod mem;
+mod net;
+mod sys_;
+mod temp;
+
+pub use cpu::SysCpu;
+pub use disks::SysDisks;
+pub use host::SysHost;
+pub use mem::SysMem;
+pub use net::SysNet;
+pub use sys_::Sys;
+pub use temp::SysTemp;
+
+use chrono::{DateTime, Local};
+use nu_protocol::{record, Record, Span, Value};
+use std::time::{Duration, UNIX_EPOCH};
+use sysinfo::{
+    Components, CpuRefreshKind, Disks, Networks, System, Users, MINIMUM_CPU_UPDATE_INTERVAL,
+};
+
+pub fn trim_cstyle_null(s: impl AsRef<str>) -> String {
+    s.as_ref().trim_matches('\0').into()
+}
+
+pub fn disks(span: Span) -> Value {
+    let disks = Disks::new_with_refreshed_list()
+        .iter()
+        .map(|disk| {
+            let device = trim_cstyle_null(disk.name().to_string_lossy());
+            let typ = trim_cstyle_null(disk.file_system().to_string_lossy());
+
+            let record = record! {
+                "device" => Value::string(device, span),
+                "type" => Value::string(typ, span),
+                "mount" => Value::string(disk.mount_point().to_string_lossy(), span),
+                "total" => Value::filesize(disk.total_space() as i64, span),
+                "free" => Value::filesize(disk.available_space() as i64, span),
+                "removable" => Value::bool(disk.is_removable(), span),
+                "kind" => Value::string(disk.kind().to_string(), span),
+            };
+
+            Value::record(record, span)
+        })
+        .collect();
+
+    Value::list(disks, span)
+}
+
+pub fn net(span: Span) -> Value {
+    let networks = Networks::new_with_refreshed_list()
+        .iter()
+        .map(|(iface, data)| {
+            let record = record! {
+                "name" => Value::string(trim_cstyle_null(iface), span),
+                "sent" => Value::filesize(data.total_transmitted() as i64, span),
+                "recv" => Value::filesize(data.total_received() as i64, span),
+            };
+
+            Value::record(record, span)
+        })
+        .collect();
+
+    Value::list(networks, span)
+}
+
+pub fn cpu(span: Span) -> Value {
+    let mut sys = System::new();
+    sys.refresh_cpu_specifics(CpuRefreshKind::everything());
+    // We must refresh the CPU twice a while apart to get valid usage data.
+    // In theory we could just sleep MINIMUM_CPU_UPDATE_INTERVAL, but I've noticed that
+    // that gives poor results (error of ~5%). Decided to wait 2x that long, somewhat arbitrarily
+    std::thread::sleep(MINIMUM_CPU_UPDATE_INTERVAL * 2);
+    sys.refresh_cpu_specifics(CpuRefreshKind::new().with_cpu_usage());
+
+    let cpus = sys
+        .cpus()
+        .iter()
+        .map(|cpu| {
+            // sysinfo CPU usage numbers are not very precise unless you wait a long time between refreshes.
+            // Round to 1DP (chosen somewhat arbitrarily) so people aren't misled by high-precision floats.
+            let rounded_usage = (cpu.cpu_usage() * 10.0).round() / 10.0;
+
+            let load_avg = System::load_average();
+            let load_avg = format!(
+                "{:.2}, {:.2}, {:.2}",
+                load_avg.one, load_avg.five, load_avg.fifteen
+            );
+
+            let record = record! {
+                "name" => Value::string(trim_cstyle_null(cpu.name()), span),
+                "brand" => Value::string(trim_cstyle_null(cpu.brand()), span),
+                "freq" => Value::int(cpu.frequency() as i64, span),
+                "cpu_usage" => Value::float(rounded_usage.into(), span),
+                "load_average" => Value::string(load_avg, span),
+                "vendor_id" => Value::string(trim_cstyle_null(cpu.vendor_id()), span),
+            };
+
+            Value::record(record, span)
+        })
+        .collect();
+
+    Value::list(cpus, span)
+}
+
+pub fn mem(span: Span) -> Value {
+    let mut sys = System::new();
+    sys.refresh_memory();
+
+    let record = record! {
+        "total" => Value::filesize(sys.total_memory() as i64, span),
+        "free" => Value::filesize(sys.free_memory() as i64, span),
+        "used" => Value::filesize(sys.used_memory() as i64, span),
+        "available" => Value::filesize(sys.available_memory() as i64, span),
+        "swap total" => Value::filesize(sys.total_swap() as i64, span),
+        "swap free" => Value::filesize(sys.free_swap() as i64, span),
+        "swap used" => Value::filesize(sys.used_swap() as i64, span),
+    };
+
+    Value::record(record, span)
+}
+
+pub fn host(span: Span) -> Value {
+    let mut record = Record::new();
+
+    if let Some(name) = System::name() {
+        record.push("name", Value::string(trim_cstyle_null(name), span));
+    }
+    if let Some(version) = System::os_version() {
+        record.push("os_version", Value::string(trim_cstyle_null(version), span));
+    }
+    if let Some(long_version) = System::long_os_version() {
+        record.push(
+            "long_os_version",
+            Value::string(trim_cstyle_null(long_version), span),
+        );
+    }
+    if let Some(version) = System::kernel_version() {
+        record.push(
+            "kernel_version",
+            Value::string(trim_cstyle_null(version), span),
+        );
+    }
+    if let Some(hostname) = System::host_name() {
+        record.push("hostname", Value::string(trim_cstyle_null(hostname), span));
+    }
+
+    record.push(
+        "uptime",
+        Value::duration(1000000000 * System::uptime() as i64, span),
+    );
+
+    // Creates a new SystemTime from the specified number of whole seconds
+    let d = UNIX_EPOCH + Duration::from_secs(System::boot_time());
+    // Create DateTime from SystemTime
+    let datetime = DateTime::<Local>::from(d);
+    // Convert to local time and then rfc3339
+    let timestamp_str = datetime.with_timezone(datetime.offset()).to_rfc3339();
+    record.push("boot_time", Value::string(timestamp_str, span));
+
+    let users = Users::new_with_refreshed_list()
+        .iter()
+        .map(|user| {
+            let groups = user
+                .groups()
+                .iter()
+                .map(|group| Value::string(trim_cstyle_null(group.name()), span))
+                .collect();
+
+            let record = record! {
+                "name" => Value::string(trim_cstyle_null(user.name()), span),
+                "groups" => Value::list(groups, span),
+            };
+
+            Value::record(record, span)
+        })
+        .collect();
+
+    record.push("sessions", Value::list(users, span));
+
+    Value::record(record, span)
+}
+
+pub fn temp(span: Span) -> Value {
+    let components = Components::new_with_refreshed_list()
+        .iter()
+        .map(|component| {
+            let mut record = record! {
+                "unit" => Value::string(component.label(), span),
+                "temp" => Value::float(component.temperature().into(), span),
+                "high" => Value::float(component.max().into(), span),
+            };
+
+            if let Some(critical) = component.critical() {
+                record.push("critical", Value::float(critical.into(), span));
+            }
+
+            Value::record(record, span)
+        })
+        .collect();
+
+    Value::list(components, span)
+}

crates/nu-command/src/system/sys/net.rs

@@ -0,0 +1,39 @@
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct SysNet;
+
+impl Command for SysNet {
+    fn name(&self) -> &str {
+        "sys net"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("sys net")
+            .filter()
+            .category(Category::System)
+            .input_output_types(vec![(Type::Nothing, Type::table())])
+    }
+
+    fn usage(&self) -> &str {
+        "View information about the system network interfaces."
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(super::net(call.head).into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Show info about the system network",
+            example: "sys net",
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/system/sys/sys_.rs

@@ -0,0 +1,64 @@
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct Sys;
+
+impl Command for Sys {
+    fn name(&self) -> &str {
+        "sys"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("sys")
+            .filter()
+            .category(Category::System)
+            .input_output_types(vec![(Type::Nothing, Type::record())])
+    }
+
+    fn usage(&self) -> &str {
+        "View information about the system."
+    }
+
+    fn extra_usage(&self) -> &str {
+        "Note that this command may take a noticeable amount of time to run. To reduce the time taken, you can use the various `sys` sub commands to get the subset of information you are interested in."
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        nu_protocol::report_error_new(
+            engine_state,
+            &ShellError::GenericError {
+                error: "Deprecated command".into(),
+                msg: "the `sys` command is deprecated, please use the new subcommands (`sys host`, `sys mem`, etc.)."
+                    .into(),
+                span: Some(call.head),
+                help: None,
+                inner: vec![],
+            },
+        );
+
+        let head = call.head;
+        let record = record! {
+            "host" => super::host(head),
+            "cpu" => super::cpu(head),
+            "disks" => super::disks(head),
+            "mem" => super::mem(head),
+            "temp" => super::temp(head),
+            "net" => super::net(head),
+        };
+        Ok(Value::record(record, head).into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Show info about the system",
+            example: "sys",
+            result: None,
+        }]
+    }
+}

crates/nu-command/src/system/sys/temp.rs

@@ -0,0 +1,43 @@
+use nu_engine::command_prelude::*;
+
+#[derive(Clone)]
+pub struct SysTemp;
+
+impl Command for SysTemp {
+    fn name(&self) -> &str {
+        "sys temp"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("sys temp")
+            .filter()
+            .category(Category::System)
+            .input_output_types(vec![(Type::Nothing, Type::table())])
+    }
+
+    fn usage(&self) -> &str {
+        "View the temperatures of system components."
+    }
+
+    fn extra_usage(&self) -> &str {
+        "Some system components do not support temperature readings, so this command may return an empty list if no components support temperature."
+    }
+
+    fn run(
+        &self,
+        _engine_state: &EngineState,
+        _stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        Ok(super::temp(call.head).into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Show the system temperatures",
+            example: "sys temp",
+            result: None,
+        }]
+    }
+}

crates/nu-command/tests/commands/ls.rs

@@ -85,13 +85,13 @@ fn lists_regular_files_in_special_folder() {
         assert_eq!(actual.out, "2");
         let actual = nu!(
             cwd: dirs.test().join("abcd/*"), format!(r#"ls ../* | length"#));
-        assert_eq!(actual.out, "3");
+        assert_eq!(actual.out, "2");
         let actual = nu!(
             cwd: dirs.test().join("abcd/?"), format!(r#"ls -D ../* | length"#));
         assert_eq!(actual.out, "2");
         let actual = nu!(
             cwd: dirs.test().join("abcd/?"), format!(r#"ls ../* | length"#));
-        assert_eq!(actual.out, "3");
+        assert_eq!(actual.out, "2");
     })
 }
 
@@ -237,6 +237,12 @@ fn list_files_from_two_parents_up_using_multiple_dots() {
         );
 
         assert_eq!(actual.out, "5");
+
+        let actual = nu!(
+            cwd: dirs.test().join("foo/bar"),
+            r#"ls ... | sort-by name | get name.0 | str replace -a '\' '/'"#
+        );
+        assert_eq!(actual.out, "../../andres.xml");
     })
 }
 
@@ -759,3 +765,46 @@ fn list_with_multiple_path() {
         assert_eq!(actual.out, "0");
     })
 }
+
+#[test]
+fn list_inside_glob_metachars_dir() {
+    Playground::setup("list_files_inside_glob_metachars_dir", |dirs, sandbox| {
+        let sub_dir = "test[]";
+        sandbox
+            .within(sub_dir)
+            .with_files(&[EmptyFile("test_file.txt")]);
+
+        let actual = nu!(
+            cwd: dirs.test().join(sub_dir),
+            "ls test_file.txt | get name.0 | path basename",
+        );
+        assert!(actual.out.contains("test_file.txt"));
+    });
+}
+
+#[test]
+fn list_inside_tilde_glob_metachars_dir() {
+    Playground::setup(
+        "list_files_inside_tilde_glob_metachars_dir",
+        |dirs, sandbox| {
+            let sub_dir = "~test[]";
+            sandbox
+                .within(sub_dir)
+                .with_files(&[EmptyFile("test_file.txt")]);
+
+            // need getname.0 | path basename because the output path
+            // might be too long to output as a single line.
+            let actual = nu!(
+                cwd: dirs.test().join(sub_dir),
+                "ls test_file.txt | get name.0 | path basename",
+            );
+            assert!(actual.out.contains("test_file.txt"));
+
+            let actual = nu!(
+                cwd: dirs.test(),
+                "ls '~test[]' | get name.0 | path basename"
+            );
+            assert!(actual.out.contains("test_file.txt"));
+        },
+    );
+}

crates/nu-command/tests/commands/select.rs

@@ -172,7 +172,7 @@ fn select_ignores_errors_successfully2() {
 
 #[test]
 fn select_ignores_errors_successfully3() {
-    let actual = nu!("sys | select invalid_key? | to nuon");
+    let actual = nu!("{foo: bar} | select invalid_key? | to nuon");
 
     assert_eq!(actual.out, "{invalid_key: null}".to_string());
     assert!(actual.err.is_empty());

crates/nu-explore/src/explore.rs

@@ -110,8 +110,8 @@ impl Command for Explore {
     fn examples(&self) -> Vec<Example> {
         vec![
             Example {
-                description: "Explore the system information record",
-                example: r#"sys | explore"#,
+                description: "Explore the system host information record",
+                example: r#"sys host | explore"#,
                 result: None,
             },
             Example {

crates/nu-parser/src/lex.rs

@@ -225,6 +225,19 @@ pub fn lex_item(
                     )),
                 );
             }
+        } else if c == b'r' && input.get(*curr_offset + 1) == Some(b'#').as_ref() {
+            // already checked `r#` pattern, so it's a raw string.
+            let lex_result = lex_raw_string(input, curr_offset, span_offset);
+            let span = Span::new(span_offset + token_start, span_offset + *curr_offset);
+            if let Err(e) = lex_result {
+                return (
+                    Token {
+                        contents: TokenContents::Item,
+                        span,
+                    },
+                    Some(e),
+                );
+            }
         } else if is_item_terminator(&block_level, c, additional_whitespace, special_tokens) {
             break;
         }
@@ -331,6 +344,65 @@ pub fn lex_item(
     (output, err)
 }
 
+fn lex_raw_string(
+    input: &[u8],
+    curr_offset: &mut usize,
+    span_offset: usize,
+) -> Result<(), ParseError> {
+    // A raw string literal looks like `echo r#'Look, I can use 'single quotes'!'#`
+    // If the next character is `#` we're probably looking at a raw string literal
+    // so we need to read all the text until we find a closing `#`. This raw string
+    // can contain any character, including newlines and double quotes without needing
+    // to escape them.
+    //
+    // A raw string can contain many `#` as prefix,
+    // incase if there is a `'#` or `#'` in the string itself.
+    // E.g: r##'I can use '#' in a raw string'##
+    let mut prefix_sharp_cnt = 0;
+    let start = *curr_offset;
+    while let Some(b'#') = input.get(start + prefix_sharp_cnt + 1) {
+        prefix_sharp_cnt += 1;
+    }
+
+    // curr_offset is the character `r`, we need to move forward and skip all `#`
+    // characters.
+    //
+    // e.g: r###'<body>
+    //      ^
+    //      ^
+    //   curr_offset
+    *curr_offset += prefix_sharp_cnt + 1;
+    // the next one should be a single quote.
+    if input.get(*curr_offset) != Some(&b'\'') {
+        return Err(ParseError::Expected(
+            "'",
+            Span::new(span_offset + *curr_offset, span_offset + *curr_offset + 1),
+        ));
+    }
+
+    *curr_offset += 1;
+    let mut matches = false;
+    while let Some(ch) = input.get(*curr_offset) {
+        // check for postfix '###
+        if *ch == b'#' {
+            let start_ch = input[*curr_offset - prefix_sharp_cnt];
+            let postfix = &input[*curr_offset - prefix_sharp_cnt + 1..=*curr_offset];
+            if start_ch == b'\'' && postfix.iter().all(|x| *x == b'#') {
+                matches = true;
+                break;
+            }
+        }
+        *curr_offset += 1
+    }
+    if !matches {
+        return Err(ParseError::UnexpectedEof(
+            "#".to_string(),
+            Span::new(span_offset + *curr_offset, span_offset + *curr_offset),
+        ));
+    }
+    Ok(())
+}
+
 pub fn lex_signature(
     input: &[u8],
     span_offset: usize,
@@ -503,79 +575,6 @@ fn lex_internal(
         } else if c == b' ' || c == b'\t' || additional_whitespace.contains(&c) {
             // If the next character is non-newline whitespace, skip it.
             curr_offset += 1;
-        } else if c == b'r' {
-            // A raw string literal looks like `echo r#'Look, I can use 'single quotes'!'#`
-            // If the next character is `#` we're probably looking at a raw string literal
-            // so we need to read all the text until we find a closing `#`. This raw string
-            // can contain any character, including newlines and double quotes without needing
-            // to escape them.
-            //
-            // A raw string can contain many `#` as prefix,
-            // incase if there is a `'#` or `#'` in the string itself.
-            // E.g: r##'I can use '#' in a raw string'##
-            let mut prefix_sharp_cnt = 0;
-            let start = curr_offset;
-            while let Some(b'#') = input.get(start + prefix_sharp_cnt + 1) {
-                prefix_sharp_cnt += 1;
-            }
-
-            if prefix_sharp_cnt != 0 {
-                // curr_offset is the character `r`, we need to move forward and skip all `#`
-                // characters.
-                //
-                // e.g: r###'<body>
-                //      ^
-                //      ^
-                //   curr_offset
-                curr_offset += prefix_sharp_cnt + 1;
-                // the next one should be a single quote.
-                if input.get(curr_offset) != Some(&b'\'') {
-                    error = Some(ParseError::Expected(
-                        "'",
-                        Span::new(span_offset + curr_offset, span_offset + curr_offset + 1),
-                    ));
-                }
-
-                curr_offset += 1;
-                let mut matches = false;
-                while let Some(ch) = input.get(curr_offset) {
-                    // check for postfix '###
-                    if *ch == b'#' {
-                        let start_ch = input[curr_offset - prefix_sharp_cnt];
-                        let postfix = &input[curr_offset - prefix_sharp_cnt + 1..=curr_offset];
-                        if start_ch == b'\'' && postfix.iter().all(|x| *x == b'#') {
-                            matches = true;
-                            curr_offset += 1;
-                            break;
-                        }
-                    }
-                    curr_offset += 1
-                }
-                if matches {
-                    output.push(Token::new(
-                        TokenContents::Item,
-                        Span::new(span_offset + start, span_offset + curr_offset),
-                    ));
-                } else if error.is_none() {
-                    error = Some(ParseError::UnexpectedEof(
-                        "#".to_string(),
-                        Span::new(span_offset + curr_offset, span_offset + curr_offset),
-                    ))
-                }
-            } else {
-                let (token, err) = lex_item(
-                    input,
-                    &mut curr_offset,
-                    span_offset,
-                    additional_whitespace,
-                    special_tokens,
-                    in_signature,
-                );
-                if error.is_none() {
-                    error = err;
-                }
-                output.push(token);
-            }
         } else {
             let token = try_lex_special_piped_item(input, &mut curr_offset, span_offset);
             if let Some(token) = token {

crates/nu-std/std/help.nu

@@ -739,8 +739,8 @@ Each stage in the pipeline works together to load, parse, and display informatio
     List the files in the current directory, sorted by size
     > ('ls | sort-by size' | nu-highlight)
 
-    Get information about the current system
-    > ('sys | get host' | nu-highlight)
+    Get the current system host name
+    > ('sys host | get hostname' | nu-highlight)
 
     Get the processes on your system actively using CPU
     > ('ps | where cpu > 0' | nu-highlight)

crates/nu-std/testing.nu

@@ -1,7 +1,7 @@
 use std log
 
 def "nu-complete threads" [] {
-    seq 1 (sys|get cpu|length)
+    seq 1 (sys cpu | length)
 }
 
 # Here we store the map of annotations internal names and the annotation actually used during test creation

crates/nu_plugin_polars/Cargo.toml

@@ -34,6 +34,7 @@ polars-arrow = { version = "0.39"}
 polars-ops = { version = "0.39"}
 polars-plan = { version = "0.39", features = ["regex"]}
 polars-utils = { version = "0.39"}
+polars-lazy = { version = "0.39"}
 typetag = "0.2"
 uuid = { version = "1.7", features = ["v4", "serde"] }
 

crates/nu_plugin_polars/src/dataframe/eager/filter_with.rs

@@ -1,162 +0,0 @@
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
-use nu_protocol::{
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
-    Value,
-};
-use polars::prelude::LazyFrame;
-
-use crate::{
-    dataframe::values::{NuExpression, NuLazyFrame},
-    values::{cant_convert_err, CustomValueSupport, PolarsPluginObject, PolarsPluginType},
-    PolarsPlugin,
-};
-
-use super::super::values::{Column, NuDataFrame};
-
-#[derive(Clone)]
-pub struct FilterWith;
-
-impl PluginCommand for FilterWith {
-    type Plugin = PolarsPlugin;
-
-    fn name(&self) -> &str {
-        "polars filter-with"
-    }
-
-    fn usage(&self) -> &str {
-        "Filters dataframe using a mask or expression as reference."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "mask or expression",
-                SyntaxShape::Any,
-                "boolean mask used to filter data",
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe or lazyframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Filter dataframe using a bool mask",
-                example: r#"let mask = ([true false] | polars into-df);
-    [[a b]; [1 2] [3 4]] | polars into-df | polars filter-with $mask"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new("a".to_string(), vec![Value::test_int(1)]),
-                            Column::new("b".to_string(), vec![Value::test_int(2)]),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Filter dataframe using an expression",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars filter-with ((polars col a) > 1)",
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new("a".to_string(), vec![Value::test_int(3)]),
-                            Column::new("b".to_string(), vec![Value::test_int(4)]),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        plugin: &Self::Plugin,
-        engine: &EngineInterface,
-        call: &EvaluatedCall,
-        input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
-        let value = input.into_value(call.head);
-        match PolarsPluginObject::try_from_value(plugin, &value)? {
-            PolarsPluginObject::NuDataFrame(df) => command_eager(plugin, engine, call, df),
-            PolarsPluginObject::NuLazyFrame(lazy) => command_lazy(plugin, engine, call, lazy),
-            _ => Err(cant_convert_err(
-                &value,
-                &[PolarsPluginType::NuDataFrame, PolarsPluginType::NuLazyFrame],
-            )),
-        }
-        .map_err(LabeledError::from)
-    }
-}
-
-fn command_eager(
-    plugin: &PolarsPlugin,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
-    df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let mask_value: Value = call.req(0)?;
-    let mask_span = mask_value.span();
-
-    if NuExpression::can_downcast(&mask_value) {
-        let expression = NuExpression::try_from_value(plugin, &mask_value)?;
-        let lazy = df.lazy();
-        let lazy = lazy.apply_with_expr(expression, LazyFrame::filter);
-
-        lazy.to_pipeline_data(plugin, engine, call.head)
-    } else {
-        let mask = NuDataFrame::try_from_value_coerce(plugin, &mask_value, mask_span)?
-            .as_series(mask_span)?;
-        let mask = mask.bool().map_err(|e| ShellError::GenericError {
-            error: "Error casting to bool".into(),
-            msg: e.to_string(),
-            span: Some(mask_span),
-            help: Some("Perhaps you want to use a series with booleans as mask".into()),
-            inner: vec![],
-        })?;
-
-        let polars_df = df
-            .as_ref()
-            .filter(mask)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error filtering dataframe".into(),
-                msg: e.to_string(),
-                span: Some(call.head),
-                help: Some("The only allowed column types for dummies are String or Int".into()),
-                inner: vec![],
-            })?;
-        let df = NuDataFrame::new(df.from_lazy, polars_df);
-        df.to_pipeline_data(plugin, engine, call.head)
-    }
-}
-
-fn command_lazy(
-    plugin: &PolarsPlugin,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
-    lazy: NuLazyFrame,
-) -> Result<PipelineData, ShellError> {
-    let expr: Value = call.req(0)?;
-    let expr = NuExpression::try_from_value(plugin, &expr)?;
-    let lazy = lazy.apply_with_expr(expr, LazyFrame::filter);
-    lazy.to_pipeline_data(plugin, engine, call.head)
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::test::test_polars_plugin_command;
-
-    #[test]
-    fn test_examples() -> Result<(), ShellError> {
-        test_polars_plugin_command(&FilterWith)
-    }
-}

crates/nu_plugin_polars/src/dataframe/eager/mod.rs

@@ -1,22 +1,11 @@
 mod append;
-mod cast;
 mod columns;
-mod drop;
-mod drop_duplicates;
-mod drop_nulls;
 mod dummies;
-mod filter_with;
-mod first;
-mod get;
-mod last;
-mod melt;
 mod open;
 mod query_df;
-mod rename;
 mod sample;
 mod schema;
 mod shape;
-mod slice;
 mod sql_context;
 mod sql_expr;
 mod summary;
@@ -28,30 +17,18 @@ mod to_df;
 mod to_json_lines;
 mod to_nu;
 mod to_parquet;
-mod with_column;
 
 use crate::PolarsPlugin;
 
 pub use self::open::OpenDataFrame;
 pub use append::AppendDF;
-pub use cast::CastDF;
 pub use columns::ColumnsDF;
-pub use drop::DropDF;
-pub use drop_duplicates::DropDuplicates;
-pub use drop_nulls::DropNulls;
 pub use dummies::Dummies;
-pub use filter_with::FilterWith;
-pub use first::FirstDF;
-pub use get::GetDF;
-pub use last::LastDF;
-pub use melt::MeltDF;
 use nu_plugin::PluginCommand;
 pub use query_df::QueryDf;
-pub use rename::RenameDF;
 pub use sample::SampleDF;
 pub use schema::SchemaCmd;
 pub use shape::ShapeDF;
-pub use slice::SliceDF;
 pub use sql_context::SQLContext;
 pub use summary::Summary;
 pub use take::TakeDF;
@@ -62,28 +39,16 @@ pub use to_df::ToDataFrame;
 pub use to_json_lines::ToJsonLines;
 pub use to_nu::ToNu;
 pub use to_parquet::ToParquet;
-pub use with_column::WithColumn;
 
 pub(crate) fn eager_commands() -> Vec<Box<dyn PluginCommand<Plugin = PolarsPlugin>>> {
     vec![
         Box::new(AppendDF),
-        Box::new(CastDF),
         Box::new(ColumnsDF),
-        Box::new(DropDF),
-        Box::new(DropDuplicates),
-        Box::new(DropNulls),
         Box::new(Dummies),
-        Box::new(FilterWith),
-        Box::new(GetDF),
         Box::new(OpenDataFrame),
-        Box::new(MeltDF),
         Box::new(Summary),
-        Box::new(FirstDF),
-        Box::new(LastDF),
-        Box::new(RenameDF),
         Box::new(SampleDF),
         Box::new(ShapeDF),
-        Box::new(SliceDF),
         Box::new(SchemaCmd),
         Box::new(TakeDF),
         Box::new(ToNu),
@@ -94,6 +59,5 @@ pub(crate) fn eager_commands() -> Vec<Box<dyn PluginCommand<Plugin = PolarsPlugi
         Box::new(ToJsonLines),
         Box::new(ToParquet),
         Box::new(QueryDf),
-        Box::new(WithColumn),
     ]
 }

crates/nu_plugin_polars/src/dataframe/eager/open.rs

@@ -4,6 +4,7 @@ use crate::{
     PolarsPlugin,
 };
 use nu_path::expand_path_with;
+use polars_lazy::frame::LazyJsonLineReader;
 
 use super::super::values::NuDataFrame;
 use nu_plugin::PluginCommand;
@@ -19,8 +20,8 @@ use std::{
 };
 
 use polars::prelude::{
-    CsvEncoding, CsvReader, IpcReader, JsonFormat, JsonReader, LazyCsvReader, LazyFileListReader,
-    LazyFrame, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader,
+    col, Expr, JsonReader, LazyCsvReader, LazyFileListReader, LazyFrame, ScanArgsIpc,
+    ScanArgsParquet, SerReader,
 };
 
 use polars_io::{avro::AvroReader, prelude::ParallelStrategy, HiveOptions};
@@ -46,7 +47,6 @@ impl PluginCommand for OpenDataFrame {
                 SyntaxShape::Filepath,
                 "file path to load values from",
             )
-            .switch("lazy", "creates a lazy dataframe", Some('l'))
             .named(
                 "type",
                 SyntaxShape::String,
@@ -161,10 +161,8 @@ fn from_parquet(
     engine: &nu_plugin::EngineInterface,
     call: &nu_plugin::EvaluatedCall,
     file_path: &Path,
-    file_span: Span,
+    _file_span: Span,
 ) -> Result<Value, ShellError> {
-    if call.has_flag("lazy")? {
-        let file: String = call.req(0)?;
     let args = ScanArgsParquet {
         n_rows: None,
         cache: true,
@@ -177,47 +175,25 @@ fn from_parquet(
         hive_options: HiveOptions::default(),
     };
 
-        let df: NuLazyFrame = LazyFrame::scan_parquet(file, args)
-            .map_err(|e| ShellError::GenericError {
+    let maybe_columns: Option<Vec<Expr>> = call
+        .get_flag::<Vec<String>>("columns")?
+        .map(|cols| cols.iter().map(|s| col(s)).collect());
+
+    let mut polars_df =
+        LazyFrame::scan_parquet(file_path, args).map_err(|e| ShellError::GenericError {
             error: "Parquet reader error".into(),
             msg: format!("{e:?}"),
             span: Some(call.head),
             help: None,
             inner: vec![],
-            })?
-            .into();
-
-        df.cache_and_to_value(plugin, engine, call.head)
-    } else {
-        let columns: Option<Vec<String>> = call.get_flag("columns")?;
-
-        let r = File::open(file_path).map_err(|e| ShellError::GenericError {
-            error: "Error opening file".into(),
-            msg: e.to_string(),
-            span: Some(file_span),
-            help: None,
-            inner: vec![],
         })?;
-        let reader = ParquetReader::new(r);
 
-        let reader = match columns {
-            None => reader,
-            Some(columns) => reader.with_columns(Some(columns)),
-        };
-
-        let df: NuDataFrame = reader
-            .finish()
-            .map_err(|e| ShellError::GenericError {
-                error: "Parquet reader error".into(),
-                msg: format!("{e:?}"),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?
-            .into();
-
-        df.cache_and_to_value(plugin, engine, call.head)
+    if let Some(columns) = maybe_columns {
+        polars_df = polars_df.select(columns);
     }
+
+    let df: NuLazyFrame = polars_df.into();
+    df.cache_and_to_value(plugin, engine, call.head)
 }
 
 fn from_avro(
@@ -262,10 +238,8 @@ fn from_ipc(
     engine: &nu_plugin::EngineInterface,
     call: &nu_plugin::EvaluatedCall,
     file_path: &Path,
-    file_span: Span,
+    _file_span: Span,
 ) -> Result<Value, ShellError> {
-    if call.has_flag("lazy")? {
-        let file: String = call.req(0)?;
     let args = ScanArgsIpc {
         n_rows: None,
         cache: true,
@@ -275,47 +249,25 @@ fn from_ipc(
         cloud_options: None,
     };
 
-        let df: NuLazyFrame = LazyFrame::scan_ipc(file, args)
-            .map_err(|e| ShellError::GenericError {
+    let maybe_columns: Option<Vec<Expr>> = call
+        .get_flag::<Vec<String>>("columns")?
+        .map(|cols| cols.iter().map(|s| col(s)).collect());
+
+    let mut polars_df =
+        LazyFrame::scan_ipc(file_path, args).map_err(|e| ShellError::GenericError {
             error: "IPC reader error".into(),
             msg: format!("{e:?}"),
             span: Some(call.head),
             help: None,
             inner: vec![],
-            })?
-            .into();
-
-        df.cache_and_to_value(plugin, engine, call.head)
-    } else {
-        let columns: Option<Vec<String>> = call.get_flag("columns")?;
-
-        let r = File::open(file_path).map_err(|e| ShellError::GenericError {
-            error: "Error opening file".into(),
-            msg: e.to_string(),
-            span: Some(file_span),
-            help: None,
-            inner: vec![],
         })?;
-        let reader = IpcReader::new(r);
 
-        let reader = match columns {
-            None => reader,
-            Some(columns) => reader.with_columns(Some(columns)),
-        };
-
-        let df: NuDataFrame = reader
-            .finish()
-            .map_err(|e| ShellError::GenericError {
-                error: "IPC reader error".into(),
-                msg: format!("{e:?}"),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?
-            .into();
-
-        df.cache_and_to_value(plugin, engine, call.head)
+    if let Some(columns) = maybe_columns {
+        polars_df = polars_df.select(columns);
     }
+
+    let df: NuLazyFrame = polars_df.into();
+    df.cache_and_to_value(plugin, engine, call.head)
 }
 
 fn from_json(
@@ -364,32 +316,21 @@ fn from_jsonl(
     engine: &nu_plugin::EngineInterface,
     call: &nu_plugin::EvaluatedCall,
     file_path: &Path,
-    file_span: Span,
+    _file_span: Span,
 ) -> Result<Value, ShellError> {
     let infer_schema: Option<usize> = call.get_flag("infer-schema")?;
     let maybe_schema = call
         .get_flag("schema")?
         .map(|schema| NuSchema::try_from(&schema))
         .transpose()?;
-    let file = File::open(file_path).map_err(|e| ShellError::GenericError {
-        error: "Error opening file".into(),
-        msg: e.to_string(),
-        span: Some(file_span),
-        help: None,
-        inner: vec![],
-    })?;
 
-    let buf_reader = BufReader::new(file);
-    let reader = JsonReader::new(buf_reader)
-        .with_json_format(JsonFormat::JsonLines)
-        .infer_schema_len(infer_schema);
+    let maybe_columns: Option<Vec<Expr>> = call
+        .get_flag::<Vec<String>>("columns")?
+        .map(|cols| cols.iter().map(|s| col(s)).collect());
 
-    let reader = match maybe_schema {
-        Some(schema) => reader.with_schema(schema.into()),
-        None => reader,
-    };
-
-    let df: NuDataFrame = reader
+    let mut polars_df = LazyJsonLineReader::new(file_path)
+        .with_infer_schema_length(infer_schema)
+        .with_schema(maybe_schema.map(|s| s.into()))
         .finish()
         .map_err(|e| ShellError::GenericError {
             error: "Json lines reader error".into(),
@@ -397,9 +338,13 @@ fn from_jsonl(
             span: Some(call.head),
             help: None,
             inner: vec![],
-        })?
-        .into();
+        })?;
 
+    if let Some(columns) = maybe_columns {
+        polars_df = polars_df.select(columns);
+    }
+
+    let df: NuLazyFrame = polars_df.into();
     df.cache_and_to_value(plugin, engine, call.head)
 }
 
@@ -408,20 +353,21 @@ fn from_csv(
     engine: &nu_plugin::EngineInterface,
     call: &nu_plugin::EvaluatedCall,
     file_path: &Path,
-    file_span: Span,
+    _file_span: Span,
 ) -> Result<Value, ShellError> {
     let delimiter: Option<Spanned<String>> = call.get_flag("delimiter")?;
     let no_header: bool = call.has_flag("no-header")?;
     let infer_schema: Option<usize> = call.get_flag("infer-schema")?;
     let skip_rows: Option<usize> = call.get_flag("skip-rows")?;
-    let columns: Option<Vec<String>> = call.get_flag("columns")?;
+    let maybe_columns: Option<Vec<Expr>> = call
+        .get_flag::<Vec<String>>("columns")?
+        .map(|cols| cols.iter().map(|s| col(s)).collect());
 
     let maybe_schema = call
         .get_flag("schema")?
         .map(|schema| NuSchema::try_from(&schema))
         .transpose()?;
 
-    if call.has_flag("lazy")? {
     let csv_reader = LazyCsvReader::new(file_path);
 
     let csv_reader = match delimiter {
@@ -462,83 +408,18 @@ fn from_csv(
         Some(r) => csv_reader.with_skip_rows(r),
     };
 
-        let df: NuLazyFrame = csv_reader
-            .finish()
-            .map_err(|e| ShellError::GenericError {
-                error: "Parquet reader error".into(),
+    let mut polars_df = csv_reader.finish().map_err(|e| ShellError::GenericError {
+        error: "CSV reader error".into(),
         msg: format!("{e:?}"),
         span: Some(call.head),
         help: None,
         inner: vec![],
-            })?
-            .into();
+    })?;
 
-        df.cache_and_to_value(plugin, engine, call.head)
-    } else {
-        let csv_reader = CsvReader::from_path(file_path)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error creating CSV reader".into(),
-                msg: e.to_string(),
-                span: Some(file_span),
-                help: None,
-                inner: vec![],
-            })?
-            .with_encoding(CsvEncoding::LossyUtf8);
-
-        let csv_reader = match delimiter {
-            None => csv_reader,
-            Some(d) => {
-                if d.item.len() != 1 {
-                    return Err(ShellError::GenericError {
-                        error: "Incorrect delimiter".into(),
-                        msg: "Delimiter has to be one character".into(),
-                        span: Some(d.span),
-                        help: None,
-                        inner: vec![],
-                    });
-                } else {
-                    let delimiter = match d.item.chars().next() {
-                        Some(d) => d as u8,
-                        None => unreachable!(),
-                    };
-                    csv_reader.with_separator(delimiter)
+    if let Some(columns) = maybe_columns {
+        polars_df = polars_df.select(columns);
     }
-            }
-        };
-
-        let csv_reader = csv_reader.has_header(!no_header);
-
-        let csv_reader = match maybe_schema {
-            Some(schema) => csv_reader.with_schema(Some(schema.into())),
-            None => csv_reader,
-        };
-
-        let csv_reader = match infer_schema {
-            None => csv_reader,
-            Some(r) => csv_reader.infer_schema(Some(r)),
-        };
-
-        let csv_reader = match skip_rows {
-            None => csv_reader,
-            Some(r) => csv_reader.with_skip_rows(r),
-        };
-
-        let csv_reader = match columns {
-            None => csv_reader,
-            Some(columns) => csv_reader.with_columns(Some(columns)),
-        };
-
-        let df: NuDataFrame = csv_reader
-            .finish()
-            .map_err(|e| ShellError::GenericError {
-                error: "Parquet reader error".into(),
-                msg: format!("{e:?}"),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?
-            .into();
 
+    let df: NuLazyFrame = polars_df.into();
     df.cache_and_to_value(plugin, engine, call.head)
 }
-}

crates/nu_plugin_polars/src/dataframe/eager/query_df.rs

@@ -91,7 +91,7 @@ fn command(
             help: None,
             inner: vec![],
         })?;
-    let lazy = NuLazyFrame::new(!df.from_lazy, df_sql);
+    let lazy = NuLazyFrame::new(df_sql);
     lazy.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/eager/sample.rs

@@ -147,7 +147,7 @@ fn command(
             inner: vec![],
         }),
     };
-    let df = NuDataFrame::new(false, df?);
+    let df = NuDataFrame::new(df?);
     df.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/eager/summary.rs

@@ -272,7 +272,7 @@ fn command(
         inner: vec![],
     })?;
 
-    let df = NuDataFrame::new(df.from_lazy, polars_df);
+    let df = NuDataFrame::new(polars_df);
 
     df.to_pipeline_data(plugin, engine, call.head)
 }

crates/nu_plugin_polars/src/dataframe/eager/take.rs

@@ -143,7 +143,7 @@ fn command(
             inner: vec![],
         })?;
 
-    let df = NuDataFrame::new(df.from_lazy, polars_df);
+    let df = NuDataFrame::new(polars_df);
     df.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/eager/with_column.rs

@@ -1,196 +0,0 @@
-use super::super::values::{Column, NuDataFrame};
-use crate::{
-    dataframe::values::{NuExpression, NuLazyFrame},
-    values::{CustomValueSupport, PolarsPluginObject},
-    PolarsPlugin,
-};
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
-use nu_protocol::{
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
-    Value,
-};
-
-#[derive(Clone)]
-pub struct WithColumn;
-
-impl PluginCommand for WithColumn {
-    type Plugin = PolarsPlugin;
-
-    fn name(&self) -> &str {
-        "polars with-column"
-    }
-
-    fn usage(&self) -> &str {
-        "Adds a series to the dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .named("name", SyntaxShape::String, "new column name", Some('n'))
-            .rest(
-                "series or expressions",
-                SyntaxShape::Any,
-                "series to be added or expressions used to define the new columns",
-            )
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe or lazyframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Adds a series to the dataframe",
-                example: r#"[[a b]; [1 2] [3 4]]
-    | polars into-df
-    | polars with-column ([5 6] | polars into-df) --name c"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "c".to_string(),
-                                vec![Value::test_int(5), Value::test_int(6)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Adds a series to the dataframe",
-                example: r#"[[a b]; [1 2] [3 4]]
-    | polars into-lazy
-    | polars with-column [
-        ((polars col a) * 2 | polars as "c")
-        ((polars col a) * 3 | polars as "d")
-      ]
-    | polars collect"#,
-                result: Some(
-                    NuDataFrame::try_from_columns(
-                        vec![
-                            Column::new(
-                                "a".to_string(),
-                                vec![Value::test_int(1), Value::test_int(3)],
-                            ),
-                            Column::new(
-                                "b".to_string(),
-                                vec![Value::test_int(2), Value::test_int(4)],
-                            ),
-                            Column::new(
-                                "c".to_string(),
-                                vec![Value::test_int(2), Value::test_int(6)],
-                            ),
-                            Column::new(
-                                "d".to_string(),
-                                vec![Value::test_int(3), Value::test_int(9)],
-                            ),
-                        ],
-                        None,
-                    )
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        plugin: &Self::Plugin,
-        engine: &EngineInterface,
-        call: &EvaluatedCall,
-        input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
-        let value = input.into_value(call.head);
-        match PolarsPluginObject::try_from_value(plugin, &value)? {
-            PolarsPluginObject::NuDataFrame(df) => command_eager(plugin, engine, call, df),
-            PolarsPluginObject::NuLazyFrame(lazy) => command_lazy(plugin, engine, call, lazy),
-            _ => Err(ShellError::CantConvert {
-                to_type: "lazy or eager dataframe".into(),
-                from_type: value.get_type().to_string(),
-                span: value.span(),
-                help: None,
-            }),
-        }
-        .map_err(LabeledError::from)
-    }
-}
-
-fn command_eager(
-    plugin: &PolarsPlugin,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
-    df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let new_column: Value = call.req(0)?;
-    let column_span = new_column.span();
-
-    if NuExpression::can_downcast(&new_column) {
-        let vals: Vec<Value> = call.rest(0)?;
-        let value = Value::list(vals, call.head);
-        let expressions = NuExpression::extract_exprs(plugin, value)?;
-        let lazy = NuLazyFrame::new(true, df.lazy().to_polars().with_columns(&expressions));
-        let df = lazy.collect(call.head)?;
-        df.to_pipeline_data(plugin, engine, call.head)
-    } else {
-        let mut other = NuDataFrame::try_from_value_coerce(plugin, &new_column, call.head)?
-            .as_series(column_span)?;
-
-        let name = match call.get_flag::<String>("name")? {
-            Some(name) => name,
-            None => other.name().to_string(),
-        };
-
-        let series = other.rename(&name).clone();
-
-        let mut polars_df = df.to_polars();
-        polars_df
-            .with_column(series)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error adding column to dataframe".into(),
-                msg: e.to_string(),
-                span: Some(column_span),
-                help: None,
-                inner: vec![],
-            })?;
-
-        let df = NuDataFrame::new(df.from_lazy, polars_df);
-        df.to_pipeline_data(plugin, engine, call.head)
-    }
-}
-
-fn command_lazy(
-    plugin: &PolarsPlugin,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
-    lazy: NuLazyFrame,
-) -> Result<PipelineData, ShellError> {
-    let vals: Vec<Value> = call.rest(0)?;
-    let value = Value::list(vals, call.head);
-    let expressions = NuExpression::extract_exprs(plugin, value)?;
-    let lazy: NuLazyFrame = lazy.to_polars().with_columns(&expressions).into();
-    lazy.to_pipeline_data(plugin, engine, call.head)
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::test::test_polars_plugin_command;
-
-    #[test]
-    fn test_examples() -> Result<(), ShellError> {
-        test_polars_plugin_command(&WithColumn)
-    }
-}

crates/nu_plugin_polars/src/dataframe/expressions/expressions_macro.rs

@@ -164,7 +164,6 @@ macro_rules! lazy_expr_command {
                     let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)
                         .map_err(LabeledError::from)?;
                     let lazy = NuLazyFrame::new(
-                        lazy.from_eager,
                         lazy.to_polars()
                             .$func()
                             .map_err(|e| ShellError::GenericError {
@@ -245,7 +244,6 @@ macro_rules! lazy_expr_command {
                     let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)
                         .map_err(LabeledError::from)?;
                     let lazy = NuLazyFrame::new(
-                        lazy.from_eager,
                         lazy.to_polars()
                             .$func($ddof)
                             .map_err(|e| ShellError::GenericError {

crates/nu_plugin_polars/src/dataframe/expressions/is_in.rs

@@ -181,8 +181,7 @@ fn command_df(
 
     res.rename("is_in");
 
-    let mut new_df = NuDataFrame::try_from_series_vec(vec![res.into_series()], call.head)?;
-    new_df.from_lazy = df.from_lazy;
+    let new_df = NuDataFrame::try_from_series_vec(vec![res.into_series()], call.head)?;
     new_df.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/expressions/otherwise.rs

@@ -48,7 +48,7 @@ impl PluginCommand for ExprOtherwise {
             Example {
                 description: "Create a new column for the dataframe",
                 example: r#"[[a b]; [6 2] [1 4] [4 1]]
-   | polars into-lazy
+   | polars into-df
    | polars with-column (
     polars when ((polars col a) > 2) 4 | polars otherwise 5 | polars as c
      )

crates/nu_plugin_polars/src/dataframe/expressions/when.rs

@@ -57,7 +57,7 @@ impl PluginCommand for ExprWhen {
             Example {
                 description: "Create a new column for the dataframe",
                 example: r#"[[a b]; [6 2] [1 4] [4 1]]
-   | polars into-lazy
+   | polars into-df
    | polars with-column (
     polars when ((polars col a) > 2) 4 | polars otherwise 5 | polars as c
      )

crates/nu_plugin_polars/src/dataframe/lazy/aggregate.rs

@@ -80,7 +80,7 @@ impl PluginCommand for LazyAggregate {
             Example {
                 description: "Group by and perform an aggregation",
                 example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]]
-    | polars into-lazy
+    | polars into-df
     | polars group-by a
     | polars agg [
         (polars col b | polars min | polars as "b_min")
@@ -147,7 +147,7 @@ impl PluginCommand for LazyAggregate {
         }
 
         let polars = group_by.to_polars();
-        let lazy = NuLazyFrame::new(false, polars.agg(&expressions));
+        let lazy = NuLazyFrame::new(polars.agg(&expressions));
         lazy.to_pipeline_data(plugin, engine, call.head)
             .map_err(LabeledError::from)
     }

crates/nu_plugin_polars/src/dataframe/lazy/cast.rs

@@ -4,7 +4,6 @@ use crate::{
     PolarsPlugin,
 };
 
-use super::super::values::NuDataFrame;
 use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
 use nu_protocol::{
     record, Category, Example, LabeledError, PipelineData, ShellError, Signature, Span,
@@ -67,7 +66,7 @@ impl PluginCommand for CastDF {
             Example {
                 description: "Cast a column in a lazy dataframe to a different dtype",
                 example:
-                    "[[a b]; [1 2] [3 4]] | polars into-df | polars into-lazy | polars cast u8 a | polars schema",
+                    "[[a b]; [1 2] [3 4]] |  polars into-df | polars cast u8 a | polars schema",
                 result: Some(Value::record(
                     record! {
                         "a" => Value::string("u8", Span::test_data()),
@@ -99,7 +98,7 @@ impl PluginCommand for CastDF {
             }
             PolarsPluginObject::NuDataFrame(df) => {
                 let (dtype, column_nm) = df_args(call)?;
-                command_eager(plugin, engine, call, column_nm, dtype, df)
+                command_lazy(plugin, engine, call, column_nm, dtype, df.lazy())
             }
             PolarsPluginObject::NuExpression(expr) => {
                 let dtype: String = call.req(0)?;
@@ -144,51 +143,10 @@ fn command_lazy(
 ) -> Result<PipelineData, ShellError> {
     let column = col(&column_nm).cast(dtype);
     let lazy = lazy.to_polars().with_columns(&[column]);
-    let lazy = NuLazyFrame::new(false, lazy);
+    let lazy = NuLazyFrame::new(lazy);
     lazy.to_pipeline_data(plugin, engine, call.head)
 }
 
-fn command_eager(
-    plugin: &PolarsPlugin,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
-    column_nm: String,
-    dtype: DataType,
-    nu_df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let mut df = (*nu_df.df).clone();
-    let column = df
-        .column(&column_nm)
-        .map_err(|e| ShellError::GenericError {
-            error: format!("{e}"),
-            msg: "".into(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let casted = column.cast(&dtype).map_err(|e| ShellError::GenericError {
-        error: format!("{e}"),
-        msg: "".into(),
-        span: Some(call.head),
-        help: None,
-        inner: vec![],
-    })?;
-
-    let _ = df
-        .with_column(casted)
-        .map_err(|e| ShellError::GenericError {
-            error: format!("{e}"),
-            msg: "".into(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let df = NuDataFrame::new(false, df);
-    df.to_pipeline_data(plugin, engine, call.head)
-}
-
 #[cfg(test)]
 mod test {
 

crates/nu_plugin_polars/src/dataframe/lazy/collect.rs

@@ -32,8 +32,8 @@ impl PluginCommand for LazyCollect {
 
     fn examples(&self) -> Vec<Example> {
         vec![Example {
-            description: "drop duplicates",
-            example: "[[a b]; [1 2] [3 4]] | polars into-lazy | polars collect",
+            description: "collect a lazy dataframe",
+            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars select [(polars col a) (polars col b)] | polars collect",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![
@@ -64,9 +64,7 @@ impl PluginCommand for LazyCollect {
         let value = input.into_value(call.head);
         match PolarsPluginObject::try_from_value(plugin, &value)? {
             PolarsPluginObject::NuLazyFrame(lazy) => {
-                let mut eager = lazy.collect(call.head)?;
-                // We don't want this converted back to a lazy frame
-                eager.from_lazy = true;
+                let eager = lazy.collect(call.head)?;
                 Ok(PipelineData::Value(
                     eager
                         .cache(plugin, engine, call.head)?

crates/nu_plugin_polars/src/dataframe/lazy/drop.rs

@@ -4,7 +4,7 @@ use nu_protocol::{
     Value,
 };
 
-use crate::values::CustomValueSupport;
+use crate::values::{CustomValueSupport, NuLazyFrame};
 use crate::PolarsPlugin;
 
 use super::super::values::utils::convert_columns;
@@ -37,7 +37,7 @@ impl PluginCommand for DropDF {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "drop column a",
-            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars drop a",
+            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars drop a | polars collect",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![Column::new(
@@ -70,46 +70,11 @@ fn command(
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
     let columns: Vec<Value> = call.rest(0)?;
-    let (col_string, col_span) = convert_columns(columns, call.head)?;
+    let (col_string, _col_span) = convert_columns(columns, call.head)?;
 
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
-
-    let new_df = col_string
-        .first()
-        .ok_or_else(|| ShellError::GenericError {
-            error: "Empty names list".into(),
-            msg: "No column names were found".into(),
-            span: Some(col_span),
-            help: None,
-            inner: vec![],
-        })
-        .and_then(|col| {
-            df.as_ref()
-                .drop(&col.item)
-                .map_err(|e| ShellError::GenericError {
-                    error: "Error dropping column".into(),
-                    msg: e.to_string(),
-                    span: Some(col.span),
-                    help: None,
-                    inner: vec![],
-                })
-        })?;
-
-    // If there are more columns in the drop selection list, these
-    // are added from the resulting dataframe
-    let polars_df = col_string.iter().skip(1).try_fold(new_df, |new_df, col| {
-        new_df
-            .drop(&col.item)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error dropping column".into(),
-                msg: e.to_string(),
-                span: Some(col.span),
-                help: None,
-                inner: vec![],
-            })
-    })?;
-
-    let final_df = NuDataFrame::new(df.from_lazy, polars_df);
+    let df = NuLazyFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let polars_df = df.to_polars().drop(col_string.iter().map(|s| &s.item));
+    let final_df = NuLazyFrame::new(polars_df);
 
     final_df.to_pipeline_data(plugin, engine, call.head)
 }

crates/nu_plugin_polars/src/dataframe/lazy/drop_duplicates.rs

@@ -5,11 +5,11 @@ use nu_protocol::{
 };
 use polars::prelude::UniqueKeepStrategy;
 
-use crate::values::CustomValueSupport;
+use crate::values::{CustomValueSupport, NuDataFrame};
 use crate::PolarsPlugin;
 
 use super::super::values::utils::convert_columns_string;
-use super::super::values::{Column, NuDataFrame};
+use super::super::values::{Column, NuLazyFrame};
 
 #[derive(Clone)]
 pub struct DropDuplicates;
@@ -48,7 +48,7 @@ impl PluginCommand for DropDuplicates {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "drop duplicates",
-            example: "[[a b]; [1 2] [3 4] [1 2]] | polars into-df | polars drop-duplicates",
+            example: "[[a b]; [1 2] [3 4] [1 2]] | polars into-df | polars drop-duplicates | polars collect",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![
@@ -87,7 +87,7 @@ fn command(
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
     let columns: Option<Vec<Value>> = call.opt(0)?;
-    let (subset, col_span) = match columns {
+    let (subset, _col_span) = match columns {
         Some(cols) => {
             let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
             (Some(agg_string), col_span)
@@ -95,9 +95,7 @@ fn command(
         None => (None, call.head),
     };
 
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
-
-    let subset_slice = subset.as_ref().map(|cols| &cols[..]);
+    let df = NuLazyFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
 
     let keep_strategy = if call.has_flag("last")? {
         UniqueKeepStrategy::Last
@@ -105,18 +103,9 @@ fn command(
         UniqueKeepStrategy::First
     };
 
-    let polars_df = df
-        .as_ref()
-        .unique(subset_slice, keep_strategy, None)
-        .map_err(|e| ShellError::GenericError {
-            error: "Error dropping duplicates".into(),
-            msg: e.to_string(),
-            span: Some(col_span),
-            help: None,
-            inner: vec![],
-        })?;
+    let polars_df = df.to_polars().unique(subset, keep_strategy);
 
-    let df = NuDataFrame::new(df.from_lazy, polars_df);
+    let df = NuLazyFrame::new(polars_df);
     df.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/lazy/drop_nulls.rs

@@ -4,11 +4,13 @@ use nu_protocol::{
     Value,
 };
 
-use crate::values::CustomValueSupport;
+use polars_lazy::dsl::col;
+
+use crate::values::{CustomValueSupport, NuDataFrame};
 use crate::PolarsPlugin;
 
 use super::super::values::utils::convert_columns_string;
-use super::super::values::{Column, NuDataFrame};
+use super::super::values::{Column, NuLazyFrame};
 
 #[derive(Clone)]
 pub struct DropNulls;
@@ -43,8 +45,7 @@ impl PluginCommand for DropNulls {
             Example {
                 description: "drop null values in dataframe",
                 example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | polars into-df);
-    let res = ($df.b / $df.b);
-    let a = ($df | polars with-column $res --name res);
+    let a = ($df | polars with-column [((polars col b) / (polars col b) | polars as res)]);
     $a | polars drop-nulls"#,
                 result: Some(
                     NuDataFrame::try_from_columns(
@@ -109,31 +110,20 @@ fn command(
     call: &EvaluatedCall,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
-
+    let df = NuLazyFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
     let columns: Option<Vec<Value>> = call.opt(0)?;
 
-    let (subset, col_span) = match columns {
+    let (subset, _col_span) = match columns {
         Some(cols) => {
             let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
-            (Some(agg_string), col_span)
+            let agg_expr = agg_string.iter().map(|s| col(s)).collect();
+            (Some(agg_expr), col_span)
         }
         None => (None, call.head),
     };
 
-    let subset_slice = subset.as_ref().map(|cols| &cols[..]);
-
-    let polars_df = df
-        .as_ref()
-        .drop_nulls(subset_slice)
-        .map_err(|e| ShellError::GenericError {
-            error: "Error dropping nulls".into(),
-            msg: e.to_string(),
-            span: Some(col_span),
-            help: None,
-            inner: vec![],
-        })?;
-    let df = NuDataFrame::new(df.from_lazy, polars_df);
+    let polars_df = df.to_polars().drop_nulls(subset);
+    let df = NuLazyFrame::new(polars_df);
     df.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/lazy/fetch.rs

@@ -70,7 +70,7 @@ impl PluginCommand for LazyFetch {
         let value = input.into_value(call.head);
         let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?;
 
-        let mut eager: NuDataFrame = lazy
+        let eager: NuDataFrame = lazy
             .to_polars()
             .fetch(rows as usize)
             .map_err(|e| ShellError::GenericError {
@@ -82,8 +82,6 @@ impl PluginCommand for LazyFetch {
             })?
             .into();
 
-        // mark this as not from lazy so it doesn't get converted back to a lazy frame
-        eager.from_lazy = false;
         eager
             .to_pipeline_data(plugin, engine, call.head)
             .map_err(LabeledError::from)

crates/nu_plugin_polars/src/dataframe/lazy/fill_null.rs

@@ -40,7 +40,7 @@ impl PluginCommand for LazyFillNull {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Fills the null values by 0",
-            example: "[1 2 2 3 3] | polars into-df | polars shift 2 | polars fill-null 0",
+            example: "[1 2 2 3 3] | polars into-df | polars shift 2 | polars fill-null 0 | polars collect",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![Column::new(
@@ -96,7 +96,7 @@ fn cmd_lazy(
     fill: Value,
 ) -> Result<PipelineData, ShellError> {
     let expr = NuExpression::try_from_value(plugin, &fill)?.into_polars();
-    let lazy = NuLazyFrame::new(lazy.from_eager, lazy.to_polars().fill_null(expr));
+    let lazy = NuLazyFrame::new(lazy.to_polars().fill_null(expr));
     lazy.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/lazy/filter.rs

@@ -42,7 +42,7 @@ impl PluginCommand for LazyFilter {
         vec![Example {
             description: "Filter dataframe using an expression",
             example:
-                "[[a b]; [6 2] [4 2] [2 2]] | polars into-df | polars filter ((polars col a) >= 4)",
+                "[[a b]; [6 2] [4 2] [2 2]] | polars into-df | polars filter ((polars col a) >= 4) | polars collect",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![
@@ -85,10 +85,7 @@ fn command(
     lazy: NuLazyFrame,
     filter_expr: NuExpression,
 ) -> Result<PipelineData, ShellError> {
-    let lazy = NuLazyFrame::new(
-        lazy.from_eager,
-        lazy.to_polars().filter(filter_expr.into_polars()),
-    );
+    let lazy = NuLazyFrame::new(lazy.to_polars().filter(filter_expr.into_polars()));
     lazy.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/lazy/filter_with.rs

@@ -0,0 +1,97 @@
+use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use nu_protocol::{
+    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
+    Value,
+};
+use polars::prelude::LazyFrame;
+
+use crate::{
+    dataframe::values::{NuExpression, NuLazyFrame},
+    values::CustomValueSupport,
+    PolarsPlugin,
+};
+
+use super::super::values::{Column, NuDataFrame};
+
+#[derive(Clone)]
+pub struct FilterWith;
+
+impl PluginCommand for FilterWith {
+    type Plugin = PolarsPlugin;
+
+    fn name(&self) -> &str {
+        "polars filter-with"
+    }
+
+    fn usage(&self) -> &str {
+        "Filters dataframe using an expression."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .required(
+                "filter expression",
+                SyntaxShape::Any,
+                "filter expression used to filter dataframe",
+            )
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
+            .category(Category::Custom("dataframe or lazyframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Filter dataframe using an expression",
+            example:
+                "[[a b]; [1 2] [3 4]] | polars into-df | polars filter-with ((polars col a) > 1)",
+            result: Some(
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new("a".to_string(), vec![Value::test_int(3)]),
+                        Column::new("b".to_string(), vec![Value::test_int(4)]),
+                    ],
+                    None,
+                )
+                .expect("simple df for test should not fail")
+                .into_value(Span::test_data()),
+            ),
+        }]
+    }
+
+    fn run(
+        &self,
+        plugin: &Self::Plugin,
+        engine: &EngineInterface,
+        call: &EvaluatedCall,
+        input: PipelineData,
+    ) -> Result<PipelineData, LabeledError> {
+        let value = input.into_value(call.head);
+        let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?;
+        command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from)
+    }
+}
+
+fn command_lazy(
+    plugin: &PolarsPlugin,
+    engine: &EngineInterface,
+    call: &EvaluatedCall,
+    lazy: NuLazyFrame,
+) -> Result<PipelineData, ShellError> {
+    let expr: Value = call.req(0)?;
+    let expr = NuExpression::try_from_value(plugin, &expr)?;
+    let lazy = lazy.apply_with_expr(expr, LazyFrame::filter);
+    lazy.to_pipeline_data(plugin, engine, call.head)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::test::test_polars_plugin_command;
+
+    #[test]
+    fn test_examples() -> Result<(), ShellError> {
+        test_polars_plugin_command(&FilterWith)
+    }
+}

crates/nu_plugin_polars/src/dataframe/lazy/first.rs

@@ -48,7 +48,7 @@ impl PluginCommand for FirstDF {
         vec![
             Example {
                 description: "Return the first row of a dataframe",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars first",
+                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars first | polars collect",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -63,7 +63,7 @@ impl PluginCommand for FirstDF {
             },
             Example {
                 description: "Return the first two rows of a dataframe",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars first 2",
+                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars first 2 | polars collect",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -98,13 +98,12 @@ impl PluginCommand for FirstDF {
         input: PipelineData,
     ) -> Result<PipelineData, LabeledError> {
         let value = input.into_value(call.head);
-        if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) {
-            let df = NuDataFrame::try_from_value_coerce(plugin, &value, call.head)?;
-            command(plugin, engine, call, df).map_err(|e| e.into())
+        if NuLazyFrame::can_downcast(&value) || NuDataFrame::can_downcast(&value) {
+            let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?;
+            command(plugin, engine, call, lazy).map_err(LabeledError::from)
         } else {
             let expr = NuExpression::try_from_value(plugin, &value)?;
             let expr: NuExpression = expr.into_polars().first().into();
-
             expr.to_pipeline_data(plugin, engine, call.head)
                 .map_err(LabeledError::from)
         }
@@ -115,13 +114,13 @@ fn command(
     plugin: &PolarsPlugin,
     engine: &EngineInterface,
     call: &EvaluatedCall,
-    df: NuDataFrame,
+    df: NuLazyFrame,
 ) -> Result<PipelineData, ShellError> {
     let rows: Option<usize> = call.opt(0)?;
     let rows = rows.unwrap_or(1);
 
-    let res = df.as_ref().head(Some(rows));
-    let res = NuDataFrame::new(false, res);
+    let res = df.to_polars().slice(0, rows as u32);
+    let res: NuLazyFrame = res.into();
 
     res.to_pipeline_data(plugin, engine, call.head)
 }

crates/nu_plugin_polars/src/dataframe/lazy/get.rs

@@ -5,10 +5,13 @@ use nu_protocol::{
 };
 
 use crate::{
-    dataframe::values::utils::convert_columns_string, values::CustomValueSupport, PolarsPlugin,
+    dataframe::values::utils::convert_columns_string,
+    values::{CustomValueSupport, NuDataFrame},
+    PolarsPlugin,
 };
 
-use super::super::values::{Column, NuDataFrame};
+use super::super::values::{Column, NuLazyFrame};
+use polars::prelude::{col, Expr};
 
 #[derive(Clone)]
 pub struct GetDF;
@@ -37,7 +40,7 @@ impl PluginCommand for GetDF {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Returns the selected column",
-            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars get a",
+            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars get a | polars collect",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![Column::new(
@@ -70,21 +73,13 @@ fn command(
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
     let columns: Vec<Value> = call.rest(0)?;
-    let (col_string, col_span) = convert_columns_string(columns, call.head)?;
+    let (col_string, _col_span) = convert_columns_string(columns, call.head)?;
+    let col_expr: Vec<Expr> = col_string.iter().map(|s| col(s)).collect();
 
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuLazyFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
 
-    let df = df
-        .as_ref()
-        .select(col_string)
-        .map_err(|e| ShellError::GenericError {
-            error: "Error selecting columns".into(),
-            msg: e.to_string(),
-            span: Some(col_span),
-            help: None,
-            inner: vec![],
-        })?;
-    let df = NuDataFrame::new(false, df);
+    let df = df.to_polars().select(col_expr);
+    let df = NuLazyFrame::new(df);
     df.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/lazy/groupby.rs

@@ -79,7 +79,7 @@ impl PluginCommand for ToLazyGroupBy {
             Example {
                 description: "Group by and perform an aggregation",
                 example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]]
-    | polars into-lazy
+    | polars into-df
     | polars group-by a
     | polars agg [
         (polars col b | polars min | polars as "b_min")
@@ -152,7 +152,7 @@ fn command(
     expressions: Vec<Expr>,
 ) -> Result<PipelineData, ShellError> {
     let group_by = lazy.to_polars().group_by(expressions);
-    let group_by = NuLazyGroupBy::new(group_by, lazy.from_eager, lazy.schema()?);
+    let group_by = NuLazyGroupBy::new(group_by, lazy.schema()?);
     group_by.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/lazy/join.rs

@@ -54,8 +54,8 @@ impl PluginCommand for LazyJoin {
         vec![
             Example {
                 description: "Join two lazy dataframes",
-                example: r#"let df_a = ([[a b c];[1 "a" 0] [2 "b" 1] [1 "c" 2] [1 "c" 3]] | polars into-lazy);
-    let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | polars into-lazy);
+                example: r#"let df_a = ([[a b c];[1 "a" 0] [2 "b" 1] [1 "c" 2] [1 "c" 3]] | polars into-df);
+    let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | polars into-df);
     $df_a | polars join $df_b a foo | polars collect"#,
                 result: Some(
                     NuDataFrame::try_from_columns(
@@ -115,7 +115,7 @@ impl PluginCommand for LazyJoin {
             Example {
                 description: "Join one eager dataframe with a lazy dataframe",
                 example: r#"let df_a = ([[a b c];[1 "a" 0] [2 "b" 1] [1 "c" 2] [1 "c" 3]] | polars into-df);
-    let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | polars into-lazy);
+    let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | polars into-df);
     $df_a | polars join $df_b a foo"#,
                 result: Some(
                     NuDataFrame::try_from_columns(
@@ -230,7 +230,6 @@ impl PluginCommand for LazyJoin {
 
         let value = input.into_value(call.head);
         let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?;
-        let from_eager = lazy.from_eager;
         let lazy = lazy.to_polars();
 
         let lazy = lazy
@@ -243,7 +242,7 @@ impl PluginCommand for LazyJoin {
             .suffix(suffix)
             .finish();
 
-        let lazy = NuLazyFrame::new(from_eager, lazy);
+        let lazy = NuLazyFrame::new(lazy);
         lazy.to_pipeline_data(plugin, engine, call.head)
             .map_err(LabeledError::from)
     }

crates/nu_plugin_polars/src/dataframe/lazy/last.rs

@@ -3,7 +3,7 @@ use crate::{
     PolarsPlugin,
 };
 
-use super::super::values::{utils::DEFAULT_ROWS, NuDataFrame, NuExpression};
+use super::super::values::{NuDataFrame, NuExpression};
 use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
 use nu_protocol::{
     Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
@@ -44,7 +44,7 @@ impl PluginCommand for LastDF {
         vec![
             Example {
                 description: "Create new dataframe with last rows",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars last 1",
+                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars last | polars collect",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -74,7 +74,7 @@ impl PluginCommand for LastDF {
     ) -> Result<PipelineData, LabeledError> {
         let value = input.into_value(call.head);
         if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) {
-            let df = NuDataFrame::try_from_value_coerce(plugin, &value, call.head)?;
+            let df = NuLazyFrame::try_from_value_coerce(plugin, &value)?;
             command(plugin, engine, call, df).map_err(|e| e.into())
         } else {
             let expr = NuExpression::try_from_value(plugin, &value)?;
@@ -90,13 +90,13 @@ fn command(
     plugin: &PolarsPlugin,
     engine: &EngineInterface,
     call: &EvaluatedCall,
-    df: NuDataFrame,
+    df: NuLazyFrame,
 ) -> Result<PipelineData, ShellError> {
     let rows: Option<usize> = call.opt(0)?;
-    let rows = rows.unwrap_or(DEFAULT_ROWS);
+    let rows = rows.unwrap_or(1);
 
-    let res = df.as_ref().tail(Some(rows));
-    let res = NuDataFrame::new(false, res);
+    let res = df.to_polars().tail(rows as u32);
+    let res = NuLazyFrame::new(res);
     res.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/lazy/macro_commands.rs

@@ -46,7 +46,7 @@ macro_rules! lazy_command {
             ) -> Result<PipelineData, LabeledError> {
                 let lazy = NuLazyFrame::try_from_pipeline_coerce(plugin, input, call.head)
                     .map_err(LabeledError::from)?;
-                let lazy = NuLazyFrame::new(lazy.from_eager, lazy.to_polars().$func());
+                let lazy = NuLazyFrame::new(lazy.to_polars().$func());
                 lazy.to_pipeline_data(plugin, engine, call.head)
                     .map_err(LabeledError::from)
             }

crates/nu_plugin_polars/src/dataframe/lazy/median.rs

@@ -126,7 +126,7 @@ fn command(
             span: None,
             inner: vec![],
         })?;
-    let lazy = NuLazyFrame::new(lazy.from_eager, polars_lazy);
+    let lazy = NuLazyFrame::new(polars_lazy);
     lazy.to_pipeline_data(plugin, engine, call.head)
 }
 

crates/nu_plugin_polars/src/dataframe/lazy/melt.rs

@@ -1,11 +1,14 @@
 use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
 use nu_protocol::{
-    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, Spanned,
-    SyntaxShape, Type, Value,
+    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
+    Value,
 };
+use polars::frame::explode::MeltArgs;
 
 use crate::{
-    dataframe::values::utils::convert_columns_string, values::CustomValueSupport, PolarsPlugin,
+    dataframe::values::utils::convert_columns_string,
+    values::{CustomValueSupport, NuLazyFrame},
+    PolarsPlugin,
 };
 
 use super::super::values::{Column, NuDataFrame};
@@ -50,6 +53,11 @@ impl PluginCommand for MeltDF {
                 "optional name for value column",
                 Some('l'),
             )
+            .switch(
+                "streamable",
+                "Use polar's streaming engine. Results will not have a stable ordering.",
+                Some('s'),
+            )
             .input_output_type(
                 Type::Custom("dataframe".into()),
                 Type::Custom("dataframe".into()),
@@ -61,7 +69,7 @@ impl PluginCommand for MeltDF {
         vec![Example {
             description: "melt dataframe",
             example:
-                "[[a b c d]; [x 1 4 a] [y 2 5 b] [z 3 6 c]] | polars into-df | polars melt -c [b c] -v [a d]",
+                "[[a b c d]; [x 1 4 a] [y 2 5 b] [z 3 6 c]] | polars into-df | polars melt -c [b c] -v [a d] | polars collect",
             result: Some(
                 NuDataFrame::try_from_columns(vec![
                     Column::new(
@@ -135,111 +143,31 @@ fn command(
     let id_col: Vec<Value> = call.get_flag("columns")?.expect("required value");
     let val_col: Vec<Value> = call.get_flag("values")?.expect("required value");
 
-    let value_name: Option<Spanned<String>> = call.get_flag("value-name")?;
-    let variable_name: Option<Spanned<String>> = call.get_flag("variable-name")?;
+    let value_name = call.get_flag("value-name")?.map(|v: String| v.into());
+    let variable_name = call.get_flag("variable-name")?.map(|v: String| v.into());
+    let streamable = call.has_flag("streamable")?;
 
-    let (id_col_string, id_col_span) = convert_columns_string(id_col, call.head)?;
-    let (val_col_string, val_col_span) = convert_columns_string(val_col, call.head)?;
+    let (id_vars, _id_col_span) = convert_columns_string(id_col, call.head)?;
+    let id_vars = id_vars.into_iter().map(Into::into).collect();
+    let (value_vars, _val_col_span) = convert_columns_string(val_col, call.head)?;
+    let value_vars = value_vars.into_iter().map(Into::into).collect();
 
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuLazyFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let polars_df = df.to_polars();
 
-    check_column_datatypes(df.as_ref(), &id_col_string, id_col_span)?;
-    check_column_datatypes(df.as_ref(), &val_col_string, val_col_span)?;
+    let args = MeltArgs {
+        id_vars,
+        value_vars,
+        variable_name,
+        value_name,
+        streamable,
+    };
 
-    let mut res = df
-        .as_ref()
-        .melt(&id_col_string, &val_col_string)
-        .map_err(|e| ShellError::GenericError {
-            error: "Error calculating melt".into(),
-            msg: e.to_string(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?;
-
-    if let Some(name) = &variable_name {
-        res.rename("variable", &name.item)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error renaming column".into(),
-                msg: e.to_string(),
-                span: Some(name.span),
-                help: None,
-                inner: vec![],
-            })?;
-    }
-
-    if let Some(name) = &value_name {
-        res.rename("value", &name.item)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error renaming column".into(),
-                msg: e.to_string(),
-                span: Some(name.span),
-                help: None,
-                inner: vec![],
-            })?;
-    }
-
-    let res = NuDataFrame::new(false, res);
+    let res = polars_df.melt(args);
+    let res = NuLazyFrame::new(res);
     res.to_pipeline_data(plugin, engine, call.head)
 }
 
-fn check_column_datatypes<T: AsRef<str>>(
-    df: &polars::prelude::DataFrame,
-    cols: &[T],
-    col_span: Span,
-) -> Result<(), ShellError> {
-    if cols.is_empty() {
-        return Err(ShellError::GenericError {
-            error: "Merge error".into(),
-            msg: "empty column list".into(),
-            span: Some(col_span),
-            help: None,
-            inner: vec![],
-        });
-    }
-
-    // Checking if they are same type
-    if cols.len() > 1 {
-        for w in cols.windows(2) {
-            let l_series = df
-                .column(w[0].as_ref())
-                .map_err(|e| ShellError::GenericError {
-                    error: "Error selecting columns".into(),
-                    msg: e.to_string(),
-                    span: Some(col_span),
-                    help: None,
-                    inner: vec![],
-                })?;
-
-            let r_series = df
-                .column(w[1].as_ref())
-                .map_err(|e| ShellError::GenericError {
-                    error: "Error selecting columns".into(),
-                    msg: e.to_string(),
-                    span: Some(col_span),
-                    help: None,
-                    inner: vec![],
-                })?;
-
-            if l_series.dtype() != r_series.dtype() {
-                return Err(ShellError::GenericError {
-                    error: "Merge error".into(),
-                    msg: "found different column types in list".into(),
-                    span: Some(col_span),
-                    help: Some(format!(
-                        "datatypes {} and {} are incompatible",
-                        l_series.dtype(),
-                        r_series.dtype()
-                    )),
-                    inner: vec![],
-                });
-            }
-        }
-    }
-
-    Ok(())
-}
-
 #[cfg(test)]
 mod test {
     use crate::test::test_polars_plugin_command;

crates/nu_plugin_polars/src/dataframe/lazy/mod.rs

@@ -1,19 +1,30 @@
 mod aggregate;
+mod cast;
 mod collect;
+mod drop;
+mod drop_duplicates;
+mod drop_nulls;
 mod explode;
 mod fetch;
 mod fill_nan;
 mod fill_null;
 mod filter;
+mod filter_with;
+mod first;
 mod flatten;
+mod get;
 pub mod groupby;
 mod join;
+mod last;
 mod macro_commands;
 mod median;
+mod melt;
 mod quantile;
+mod rename;
 mod select;
+mod slice;
 mod sort_by_expr;
-mod to_lazy;
+mod with_column;
 
 use nu_plugin::PluginCommand;
 
@@ -29,13 +40,20 @@ pub(crate) use crate::dataframe::lazy::macro_commands::*;
 use crate::dataframe::lazy::quantile::LazyQuantile;
 pub(crate) use crate::dataframe::lazy::select::LazySelect;
 use crate::dataframe::lazy::sort_by_expr::LazySortBy;
-pub use crate::dataframe::lazy::to_lazy::ToLazyFrame;
 use crate::PolarsPlugin;
 pub use explode::LazyExplode;
 pub use flatten::LazyFlatten;
 
 pub(crate) fn lazy_commands() -> Vec<Box<dyn PluginCommand<Plugin = PolarsPlugin>>> {
     vec![
+        Box::new(cast::CastDF),
+        Box::new(drop::DropDF),
+        Box::new(drop_duplicates::DropDuplicates),
+        Box::new(drop_nulls::DropNulls),
+        Box::new(filter_with::FilterWith),
+        Box::new(first::FirstDF),
+        Box::new(get::GetDF),
+        Box::new(last::LastDF),
         Box::new(LazyAggregate),
         Box::new(LazyCache),
         Box::new(LazyCollect),
@@ -47,11 +65,14 @@ pub(crate) fn lazy_commands() -> Vec<Box<dyn PluginCommand<Plugin = PolarsPlugin
         Box::new(LazyFlatten),
         Box::new(LazyJoin),
         Box::new(median::LazyMedian),
+        Box::new(melt::MeltDF),
         Box::new(LazyReverse),
         Box::new(LazySelect),
         Box::new(LazySortBy),
         Box::new(LazyQuantile),
-        Box::new(ToLazyFrame),
+        Box::new(rename::RenameDF),
+        Box::new(slice::SliceDF),
         Box::new(ToLazyGroupBy),
+        Box::new(with_column::WithColumn),
     ]
 }

crates/nu_plugin_polars/src/dataframe/lazy/quantile.rs

@@ -132,7 +132,6 @@ fn command(
     quantile: f64,
 ) -> Result<PipelineData, ShellError> {
     let lazy = NuLazyFrame::new(
-        lazy.from_eager,
         lazy.to_polars()
             .quantile(lit(quantile), QuantileInterpolOptions::default())
             .map_err(|e| ShellError::GenericError {

crates/nu_plugin_polars/src/dataframe/lazy/rename.rs

@@ -6,7 +6,7 @@ use nu_protocol::{
 
 use crate::{
     dataframe::{utils::extract_strings, values::NuLazyFrame},
-    values::{CustomValueSupport, PolarsPluginObject},
+    values::CustomValueSupport,
     PolarsPlugin,
 };
 
@@ -49,7 +49,7 @@ impl PluginCommand for RenameDF {
         vec![
             Example {
                 description: "Renames a series",
-                example: "[5 6 7 8] | polars into-df | polars rename '0' new_name",
+                example: "[5 6 7 8] | polars into-df | polars rename '0' new_name | polars collect",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![Column::new(
@@ -69,7 +69,7 @@ impl PluginCommand for RenameDF {
             },
             Example {
                 description: "Renames a dataframe column",
-                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars rename a a_new",
+                example: "[[a b]; [1 2] [3 4]] | polars into-df | polars rename a a_new | polars collect",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -91,7 +91,7 @@ impl PluginCommand for RenameDF {
             Example {
                 description: "Renames two dataframe columns",
                 example:
-                    "[[a b]; [1 2] [3 4]] | polars into-df | polars rename [a b] [a_new b_new]",
+                    "[[a b]; [1 2] [3 4]] | polars into-df | polars rename [a b] [a_new b_new] | polars collect",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -121,47 +121,9 @@ impl PluginCommand for RenameDF {
         input: PipelineData,
     ) -> Result<PipelineData, LabeledError> {
         let value = input.into_value(call.head);
-        match PolarsPluginObject::try_from_value(plugin, &value).map_err(LabeledError::from)? {
-            PolarsPluginObject::NuDataFrame(df) => {
-                command_eager(plugin, engine, call, df).map_err(LabeledError::from)
-            }
-            PolarsPluginObject::NuLazyFrame(lazy) => {
+        let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?;
         command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from)
     }
-            _ => Err(LabeledError::new(format!("Unsupported type: {value:?}"))
-                .with_label("Unsupported Type", call.head)),
-        }
-    }
-}
-
-fn command_eager(
-    plugin: &PolarsPlugin,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
-    df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let columns: Value = call.req(0)?;
-    let columns = extract_strings(columns)?;
-
-    let new_names: Value = call.req(1)?;
-    let new_names = extract_strings(new_names)?;
-
-    let mut polars_df = df.to_polars();
-
-    for (from, to) in columns.iter().zip(new_names.iter()) {
-        polars_df
-            .rename(from, to)
-            .map_err(|e| ShellError::GenericError {
-                error: "Error renaming".into(),
-                msg: e.to_string(),
-                span: Some(call.head),
-                help: None,
-                inner: vec![],
-            })?;
-    }
-
-    let df = NuDataFrame::new(false, polars_df);
-    df.to_pipeline_data(plugin, engine, call.head)
 }
 
 fn command_lazy(

crates/nu_plugin_polars/src/dataframe/lazy/select.rs

@@ -67,7 +67,7 @@ impl PluginCommand for LazySelect {
 
         let pipeline_value = input.into_value(call.head);
         let lazy = NuLazyFrame::try_from_value_coerce(plugin, &pipeline_value)?;
-        let lazy = NuLazyFrame::new(lazy.from_eager, lazy.to_polars().select(&expressions));
+        let lazy = NuLazyFrame::new(lazy.to_polars().select(&expressions));
         lazy.to_pipeline_data(plugin, engine, call.head)
             .map_err(LabeledError::from)
     }

crates/nu_plugin_polars/src/dataframe/lazy/slice.rs

@@ -4,7 +4,11 @@ use nu_protocol::{
     Value,
 };
 
-use crate::{dataframe::values::Column, values::CustomValueSupport, PolarsPlugin};
+use crate::{
+    dataframe::values::Column,
+    values::{CustomValueSupport, NuLazyFrame},
+    PolarsPlugin,
+};
 
 use super::super::values::NuDataFrame;
 
@@ -36,7 +40,7 @@ impl PluginCommand for SliceDF {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Create new dataframe from a slice of the rows",
-            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars slice 0 1",
+            example: "[[a b]; [1 2] [3 4]] | polars into-df | polars slice 0 1 | polars collect",
             result: Some(
                 NuDataFrame::try_from_columns(
                     vec![
@@ -69,12 +73,12 @@ fn command(
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
     let offset: i64 = call.req(0)?;
-    let size: usize = call.req(1)?;
+    let size: i64 = call.req(1)?;
 
-    let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
+    let df = NuLazyFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
 
-    let res = df.as_ref().slice(offset, size);
-    let res = NuDataFrame::new(false, res);
+    let res = df.to_polars().slice(offset, size as u32);
+    let res = NuLazyFrame::new(res);
 
     res.to_pipeline_data(plugin, engine, call.head)
 }

crates/nu_plugin_polars/src/dataframe/lazy/sort_by_expr.rs

@@ -147,10 +147,7 @@ impl PluginCommand for LazySortBy {
 
         let pipeline_value = input.into_value(call.head);
         let lazy = NuLazyFrame::try_from_value_coerce(plugin, &pipeline_value)?;
-        let lazy = NuLazyFrame::new(
-            lazy.from_eager,
-            lazy.to_polars().sort_by_exprs(&expressions, sort_options),
-        );
+        let lazy = NuLazyFrame::new(lazy.to_polars().sort_by_exprs(&expressions, sort_options));
         lazy.to_pipeline_data(plugin, engine, call.head)
             .map_err(LabeledError::from)
     }

crates/nu_plugin_polars/src/dataframe/lazy/to_lazy.rs

@@ -1,90 +0,0 @@
-use crate::{dataframe::values::NuSchema, values::CustomValueSupport, Cacheable, PolarsPlugin};
-
-use super::super::values::{NuDataFrame, NuLazyFrame};
-
-use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
-use nu_protocol::{Category, Example, LabeledError, PipelineData, Signature, SyntaxShape, Type};
-
-#[derive(Clone)]
-pub struct ToLazyFrame;
-
-impl PluginCommand for ToLazyFrame {
-    type Plugin = PolarsPlugin;
-
-    fn name(&self) -> &str {
-        "polars into-lazy"
-    }
-
-    fn usage(&self) -> &str {
-        "Converts a dataframe into a lazy dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .named(
-                "schema",
-                SyntaxShape::Record(vec![]),
-                r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
-                Some('s'),
-            )
-            .input_output_type(Type::Any, Type::Custom("dataframe".into()))
-            .category(Category::Custom("lazyframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Takes a table and creates a lazyframe",
-            example: "[[a b];[1 2] [3 4]] | polars into-lazy",
-            result: None,
-        },
-        Example {
-            description: "Takes a table, creates a lazyframe, assigns column 'b' type str, displays the schema",
-            example: "[[a b];[1 2] [3 4]] | polars into-lazy --schema {b: str} | polars schema",
-            result: None
-        },
-        ]
-    }
-
-    fn run(
-        &self,
-        plugin: &Self::Plugin,
-        engine: &EngineInterface,
-        call: &EvaluatedCall,
-        input: PipelineData,
-    ) -> Result<PipelineData, LabeledError> {
-        let maybe_schema = call
-            .get_flag("schema")?
-            .map(|schema| NuSchema::try_from(&schema))
-            .transpose()?;
-
-        let df = NuDataFrame::try_from_iter(plugin, input.into_iter(), maybe_schema)?;
-        let mut lazy = NuLazyFrame::from_dataframe(df);
-        // We don't want this converted back to an eager dataframe at some point
-        lazy.from_eager = false;
-        Ok(PipelineData::Value(
-            lazy.cache(plugin, engine, call.head)?.into_value(call.head),
-            None,
-        ))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use nu_plugin_test_support::PluginTest;
-    use nu_protocol::{ShellError, Span};
-
-    use super::*;
-
-    #[test]
-    fn test_to_lazy() -> Result<(), ShellError> {
-        let plugin: Arc<PolarsPlugin> = PolarsPlugin::new_test_mode().into();
-        let mut plugin_test = PluginTest::new("polars", Arc::clone(&plugin))?;
-        let pipeline_data = plugin_test.eval("[[a b]; [6 2] [1 4] [4 1]] | polars into-lazy")?;
-        let value = pipeline_data.into_value(Span::test_data());
-        let df = NuLazyFrame::try_from_value(&plugin, &value)?;
-        assert!(!df.from_eager);
-        Ok(())
-    }
-}

crates/nu_plugin_polars/src/dataframe/lazy/with_column.rs

@@ -0,0 +1,114 @@
+use super::super::values::{Column, NuDataFrame};
+use crate::{
+    dataframe::values::{NuExpression, NuLazyFrame},
+    values::CustomValueSupport,
+    PolarsPlugin,
+};
+use nu_plugin::{EngineInterface, EvaluatedCall, PluginCommand};
+use nu_protocol::{
+    Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
+    Value,
+};
+
+#[derive(Clone)]
+pub struct WithColumn;
+
+impl PluginCommand for WithColumn {
+    type Plugin = PolarsPlugin;
+
+    fn name(&self) -> &str {
+        "polars with-column"
+    }
+
+    fn usage(&self) -> &str {
+        "Adds a series to the dataframe."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .rest(
+                "series or expressions",
+                SyntaxShape::Any,
+                "series to be added or expressions used to define the new columns",
+            )
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
+            .category(Category::Custom("dataframe or lazyframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Adds a series to the dataframe",
+            example: r#"[[a b]; [1 2] [3 4]]
+    | polars into-df
+    | polars with-column [
+        ((polars col a) * 2 | polars as "c")
+        ((polars col a) * 3 | polars as "d")
+      ]
+    | polars collect"#,
+            result: Some(
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
+                        ),
+                        Column::new(
+                            "c".to_string(),
+                            vec![Value::test_int(2), Value::test_int(6)],
+                        ),
+                        Column::new(
+                            "d".to_string(),
+                            vec![Value::test_int(3), Value::test_int(9)],
+                        ),
+                    ],
+                    None,
+                )
+                .expect("simple df for test should not fail")
+                .into_value(Span::test_data()),
+            ),
+        }]
+    }
+
+    fn run(
+        &self,
+        plugin: &Self::Plugin,
+        engine: &EngineInterface,
+        call: &EvaluatedCall,
+        input: PipelineData,
+    ) -> Result<PipelineData, LabeledError> {
+        let value = input.into_value(call.head);
+        let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?;
+        command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from)
+    }
+}
+
+fn command_lazy(
+    plugin: &PolarsPlugin,
+    engine: &EngineInterface,
+    call: &EvaluatedCall,
+    lazy: NuLazyFrame,
+) -> Result<PipelineData, ShellError> {
+    let vals: Vec<Value> = call.rest(0)?;
+    let value = Value::list(vals, call.head);
+    let expressions = NuExpression::extract_exprs(plugin, value)?;
+    let lazy: NuLazyFrame = lazy.to_polars().with_columns(&expressions).into();
+    lazy.to_pipeline_data(plugin, engine, call.head)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::test::test_polars_plugin_command;
+
+    #[test]
+    fn test_examples() -> Result<(), ShellError> {
+        test_polars_plugin_command(&WithColumn)
+    }
+}

crates/nu_plugin_polars/src/dataframe/series/shift.rs

@@ -1,6 +1,6 @@
 use crate::{
     dataframe::values::{NuExpression, NuLazyFrame},
-    values::{cant_convert_err, CustomValueSupport, PolarsPluginObject, PolarsPluginType},
+    values::CustomValueSupport,
     PolarsPlugin,
 };
 
@@ -63,8 +63,7 @@ impl PluginCommand for Shift {
             },
             Example {
                 description: "Shifts the values by a given period, fill absent values with 0",
-                example:
-                    "[1 2 2 3 3] | polars into-lazy | polars shift 2 --fill 0 | polars collect",
+                example: "[1 2 2 3 3] | polars into-df | polars shift 2 --fill 0 | polars collect",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![Column::new(
@@ -94,33 +93,9 @@ impl PluginCommand for Shift {
         input: PipelineData,
     ) -> Result<PipelineData, LabeledError> {
         let value = input.into_value(call.head);
-
-        match PolarsPluginObject::try_from_value(plugin, &value)? {
-            PolarsPluginObject::NuDataFrame(df) => command_eager(plugin, engine, call, df),
-            PolarsPluginObject::NuLazyFrame(lazy) => command_lazy(plugin, engine, call, lazy),
-            _ => Err(cant_convert_err(
-                &value,
-                &[
-                    PolarsPluginType::NuDataFrame,
-                    PolarsPluginType::NuLazyGroupBy,
-                ],
-            )),
+        let lazy = NuLazyFrame::try_from_value_coerce(plugin, &value)?;
+        command_lazy(plugin, engine, call, lazy).map_err(LabeledError::from)
     }
-        .map_err(LabeledError::from)
-    }
-}
-
-fn command_eager(
-    plugin: &PolarsPlugin,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
-    df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let period: i64 = call.req(0)?;
-    let series = df.as_series(call.head)?.shift(period);
-
-    let df = NuDataFrame::try_from_series_vec(vec![series], call.head)?;
-    df.to_pipeline_data(plugin, engine, call.head)
 }
 
 fn command_lazy(

crates/nu_plugin_polars/src/dataframe/series/unique.rs

@@ -1,6 +1,6 @@
 use crate::{
     dataframe::{utils::extract_strings, values::NuLazyFrame},
-    values::{cant_convert_err, CustomValueSupport, PolarsPluginObject, PolarsPluginType},
+    values::CustomValueSupport,
     PolarsPlugin,
 };
 
@@ -11,7 +11,7 @@ use nu_protocol::{
     Category, Example, LabeledError, PipelineData, ShellError, Signature, Span, SyntaxShape, Type,
     Value,
 };
-use polars::prelude::{IntoSeries, UniqueKeepStrategy};
+use polars::prelude::UniqueKeepStrategy;
 
 #[derive(Clone)]
 pub struct Unique;
@@ -68,7 +68,7 @@ impl PluginCommand for Unique {
             },
             Example {
                 description: "Returns unique values in a subset of lazyframe columns",
-                example: "[[a b c]; [1 2 1] [2 2 2] [3 2 1]] | polars into-lazy | polars unique --subset [b c] | polars collect",
+                example: "[[a b c]; [1 2 1] [2 2 2] [3 2 1]] | polars into-df | polars unique --subset [b c] | polars collect",
                 result: Some(
                     NuDataFrame::try_from_columns(
                         vec![
@@ -94,7 +94,7 @@ impl PluginCommand for Unique {
             Example {
                 description: "Returns unique values in a subset of lazyframe columns",
                 example: r#"[[a b c]; [1 2 1] [2 2 2] [3 2 1]]
-    | polars into-lazy
+    | polars into-df
     | polars unique --subset [b c] --last
     | polars collect"#,
                 result: Some(
@@ -135,40 +135,9 @@ impl PluginCommand for Unique {
         input: PipelineData,
     ) -> Result<PipelineData, LabeledError> {
         let value = input.into_value(call.head);
-
-        match PolarsPluginObject::try_from_value(plugin, &value)? {
-            PolarsPluginObject::NuDataFrame(df) => command_eager(plugin, engine, call, df),
-            PolarsPluginObject::NuLazyFrame(lazy) => command_lazy(plugin, engine, call, lazy),
-            _ => Err(cant_convert_err(
-                &value,
-                &[
-                    PolarsPluginType::NuDataFrame,
-                    PolarsPluginType::NuLazyGroupBy,
-                ],
-            )),
+        let df = NuLazyFrame::try_from_value_coerce(plugin, &value)?;
+        command_lazy(plugin, engine, call, df).map_err(LabeledError::from)
     }
-        .map_err(LabeledError::from)
-    }
-}
-
-fn command_eager(
-    plugin: &PolarsPlugin,
-    engine: &EngineInterface,
-    call: &EvaluatedCall,
-    df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let series = df.as_series(call.head)?;
-
-    let res = series.unique().map_err(|e| ShellError::GenericError {
-        error: "Error calculating unique values".into(),
-        msg: e.to_string(),
-        span: Some(call.head),
-        help: Some("The str-slice command can only be used with string columns".into()),
-        inner: vec![],
-    })?;
-
-    let df = NuDataFrame::try_from_series_vec(vec![res.into_series()], call.head)?;
-    df.to_pipeline_data(plugin, engine, call.head)
 }
 
 fn command_lazy(

crates/nu_plugin_polars/src/dataframe/values/mod.rs

@@ -323,19 +323,7 @@ pub trait CustomValueSupport: Cacheable {
         engine: &EngineInterface,
         span: Span,
     ) -> Result<Value, ShellError> {
-        match self.to_cache_value()? {
-            // if it was from a lazy value, make it lazy again
-            PolarsPluginObject::NuDataFrame(df) if df.from_lazy => {
-                let df = df.lazy();
-                Ok(df.cache(plugin, engine, span)?.into_value(span))
-            }
-            // if it was from an eager value, make it eager again
-            PolarsPluginObject::NuLazyFrame(lf) if lf.from_eager => {
-                let lf = lf.collect(span)?;
-                Ok(lf.cache(plugin, engine, span)?.into_value(span))
-            }
-            _ => Ok(self.cache(plugin, engine, span)?.into_value(span)),
-        }
+        Ok(self.cache(plugin, engine, span)?.into_value(span))
     }
 
     /// Caches the object, converts it to a it's CustomValue counterpart

crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/conversion.rs

@@ -472,7 +472,7 @@ pub fn from_parsed_columns(column_values: ColumnMap) -> Result<NuDataFrame, Shel
     }
 
     DataFrame::new(df_series)
-        .map(|df| NuDataFrame::new(false, df))
+        .map(NuDataFrame::new)
         .map_err(|e| ShellError::GenericError {
             error: "Error creating dataframe".into(),
             msg: e.to_string(),

crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/mod.rs

@@ -104,7 +104,6 @@ impl PolarsObject for DataFrameValue {
 pub struct NuDataFrame {
     pub id: Uuid,
     pub df: Arc<DataFrame>,
-    pub from_lazy: bool,
 }
 
 impl AsRef<DataFrame> for NuDataFrame {
@@ -115,17 +114,16 @@ impl AsRef<DataFrame> for NuDataFrame {
 
 impl From<DataFrame> for NuDataFrame {
     fn from(df: DataFrame) -> Self {
-        Self::new(false, df)
+        Self::new(df)
     }
 }
 
 impl NuDataFrame {
-    pub fn new(from_lazy: bool, df: DataFrame) -> Self {
+    pub fn new(df: DataFrame) -> Self {
         let id = Uuid::new_v4();
         Self {
             id,
             df: Arc::new(df),
-            from_lazy,
         }
     }
 
@@ -134,12 +132,12 @@ impl NuDataFrame {
     }
 
     pub fn lazy(&self) -> NuLazyFrame {
-        NuLazyFrame::new(true, self.to_polars().lazy())
+        NuLazyFrame::new(self.to_polars().lazy())
     }
 
     pub fn try_from_series(series: Series, span: Span) -> Result<Self, ShellError> {
         match DataFrame::new(vec![series]) {
-            Ok(dataframe) => Ok(NuDataFrame::new(false, dataframe)),
+            Ok(dataframe) => Ok(NuDataFrame::new(dataframe)),
             Err(e) => Err(ShellError::GenericError {
                 error: "Error creating dataframe".into(),
                 msg: e.to_string(),
@@ -200,7 +198,7 @@ impl NuDataFrame {
             inner: vec![],
         })?;
 
-        Ok(Self::new(false, dataframe))
+        Ok(Self::new(dataframe))
     }
 
     pub fn try_from_columns(
@@ -274,7 +272,7 @@ impl NuDataFrame {
             inner: vec![],
         })?;
 
-        Ok(Self::new(false, df))
+        Ok(Self::new(df))
     }
 
     pub fn is_series(&self) -> bool {

crates/nu_plugin_polars/src/dataframe/values/nu_dataframe/operations.rs

@@ -147,7 +147,7 @@ impl NuDataFrame {
                     inner: vec![],
                 })?;
 
-                Ok(NuDataFrame::new(false, df_new))
+                Ok(NuDataFrame::new(df_new))
             }
             Axis::Column => {
                 if self.df.width() != other.df.width() {
@@ -205,7 +205,7 @@ impl NuDataFrame {
                     inner: vec![],
                 })?;
 
-                Ok(NuDataFrame::new(false, df_new))
+                Ok(NuDataFrame::new(df_new))
             }
         }
     }

crates/nu_plugin_polars/src/dataframe/values/nu_expression/custom_value.rs

@@ -63,33 +63,6 @@ fn compute_with_value(
     op: Span,
     right: &Value,
 ) -> Result<Value, ShellError> {
-    let rhs_span = right.span();
-    match right {
-        Value::Custom { val: rhs, .. } => {
-            let rhs = rhs.as_any().downcast_ref::<NuExpression>().ok_or_else(|| {
-                ShellError::DowncastNotPossible {
-                    msg: "Unable to create expression".into(),
-                    span: rhs_span,
-                }
-            })?;
-
-            match rhs.as_ref() {
-                polars::prelude::Expr::Literal(..) => with_operator(
-                    (plugin, engine),
-                    operator,
-                    left,
-                    rhs,
-                    lhs_span,
-                    right.span(),
-                    op,
-                ),
-                _ => Err(ShellError::TypeMismatch {
-                    err_message: "Only literal expressions or number".into(),
-                    span: right.span(),
-                }),
-            }
-        }
-        _ => {
     let rhs = NuExpression::try_from_value(plugin, right)?;
     with_operator(
         (plugin, engine),
@@ -101,8 +74,6 @@ fn compute_with_value(
         op,
     )
 }
-    }
-}
 
 fn with_operator(
     (plugin, engine): (&PolarsPlugin, &EngineInterface),

crates/nu_plugin_polars/src/dataframe/values/nu_lazyframe/mod.rs

@@ -21,7 +21,6 @@ pub use custom_value::NuLazyFrameCustomValue;
 pub struct NuLazyFrame {
     pub id: Uuid,
     pub lazy: Arc<LazyFrame>,
-    pub from_eager: bool,
 }
 
 impl fmt::Debug for NuLazyFrame {
@@ -32,22 +31,21 @@ impl fmt::Debug for NuLazyFrame {
 
 impl From<LazyFrame> for NuLazyFrame {
     fn from(lazy_frame: LazyFrame) -> Self {
-        NuLazyFrame::new(false, lazy_frame)
+        NuLazyFrame::new(lazy_frame)
     }
 }
 
 impl NuLazyFrame {
-    pub fn new(from_eager: bool, lazy: LazyFrame) -> Self {
+    pub fn new(lazy: LazyFrame) -> Self {
         Self {
             id: Uuid::new_v4(),
             lazy: Arc::new(lazy),
-            from_eager,
         }
     }
 
     pub fn from_dataframe(df: NuDataFrame) -> Self {
         let lazy = df.as_ref().clone().lazy();
-        NuLazyFrame::new(true, lazy)
+        NuLazyFrame::new(lazy)
     }
 
     pub fn to_polars(&self) -> LazyFrame {
@@ -64,7 +62,7 @@ impl NuLazyFrame {
                 help: None,
                 inner: vec![],
             })
-            .map(|df| NuDataFrame::new(true, df))
+            .map(NuDataFrame::new)
     }
 
     pub fn apply_with_expr<F>(self, expr: NuExpression, f: F) -> Self
@@ -74,7 +72,7 @@ impl NuLazyFrame {
         let df = self.to_polars();
         let expr = expr.into_polars();
         let new_frame = f(df, expr);
-        Self::new(self.from_eager, new_frame)
+        Self::new(new_frame)
     }
 
     pub fn schema(&self) -> Result<NuSchema, ShellError> {

crates/nu_plugin_polars/src/dataframe/values/nu_lazygroupby/mod.rs

@@ -20,7 +20,6 @@ pub struct NuLazyGroupBy {
     pub id: Uuid,
     pub group_by: Arc<LazyGroupBy>,
     pub schema: NuSchema,
-    pub from_eager: bool,
 }
 
 impl fmt::Debug for NuLazyGroupBy {
@@ -30,11 +29,10 @@ impl fmt::Debug for NuLazyGroupBy {
 }
 
 impl NuLazyGroupBy {
-    pub fn new(group_by: LazyGroupBy, from_eager: bool, schema: NuSchema) -> Self {
+    pub fn new(group_by: LazyGroupBy, schema: NuSchema) -> Self {
         NuLazyGroupBy {
             id: Uuid::new_v4(),
             group_by: Arc::new(group_by),
-            from_eager,
             schema,
         }
     }

crates/nu_plugin_polars/src/dataframe/values/utils.rs

@@ -48,8 +48,6 @@ pub(crate) fn convert_columns(
 
 // Converts a Vec<Value> to a Vec<String> with a Span marking the whole
 // location of the columns for error referencing
-// todo - fix
-#[allow(dead_code)]
 pub(crate) fn convert_columns_string(
     columns: Vec<Value>,
     span: Span,

devdocs/HOWTOS.md

@@ -0,0 +1,42 @@
+# Developer how to guides and SOPs
+
+## Adding a dependency
+
+- Check the existing dependencies of Nushell if they already provide the needed functionality
+- Choosing the crate: Align the choice of crate with Nushell's goals and be ready to explain why.
+    - Trust/Reliability/Existing support: Crates should have an existing userbase to establish trust, repository can't be archived
+    - License compatibility: Nushell is MIT-licensed. Any crate you pick needs to be compatible with that. This excludes strong copyleft licenses (e.g. GPL) or use-restricting licenses (e.g. Business Source License)
+    - Match to the requirements: Relevant technical considerations if it provides the necessary function in the best possible way and in doing so has no significant negative externalities.
+    - Stability: can the crate provide the necessary stability promises that the behavior of Nushell is not affected by version updates.
+- (**If existing dependency on this crate exists**:) check if the version you add causes a duplication as two versions have to be compiled to satisfy another version requirement
+    - if possible upgrade older dependency specification (if an outside upstream change is easily possible, may be worth contributing to the crate ecosystem)
+    - if not check if it would be responsible to use the older version for your requirements
+    - else point out the future work necessary to reconcile this and add a duplication
+    - (you can check with `cargo tree --duplicates`)
+- Git dependencies can not be published as a release, thus are only to be used to to pull in critical new development/fixes on an existing dependencies. (PRs including a new dependency as a git dependency **will be postponed** until a version is released on crates.io!)
+- Specify the full semver string of the version you added and tested ("major.minor.patch") this ensures if cargo would downgrade the version, no features or critical bug fixes are missing.
+- try to specify the minimal required set of [features](https://doc.rust-lang.org/cargo/reference/features.html)
+- **If** the dependencies is used in more than one crate in the workspace add the version specification to the [`workspace.dependencies`](https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#inheriting-a-dependency-from-a-workspace) in the root `Cargo.toml` and simply inherit it with `cratename = { workspace = true }`
+- (**If adding a dependency referring to a crate inside the nushell workspace**) make sure the addition does not cause a circular dependency structure. Otherwise it can not be published. (We also consider the `dev-dependencies` with our [script to check the crate release order](https://github.com/nushell/nu_scripts/blame/main/make_release/nu_deps.nu) )
+
+
+## Creating a new crate
+
+- Generally new crates should be part of `crates/` in the nushell monorepo, unless:
+    - both Nushell and a crate that gets published outside the monorepo (like `reedline`) depend on a common interface, so it needs to be released before the nushell workspace AND an existing outside crate -> new independent crate in the `nushell` organization
+    - it is a fork of a full existing crate that does not require deep integration into the nushell monorepo, maintaining the fork in the `nushell` organization has the benefit of maintaining the commit history and being more accessible to other interested users (e.g. `nu-ansi-term`)
+- If you redistribute code that is part of another project, make sure proper attribution is given and license compatibility is given (we are MIT, if your crate use something compatible but different we may need to dual license the new crate)
+- Ensure that no circles in the crate graph are created.
+- **Currently** use the same version as the other workspace crates
+- Use workspace dependencies in the `Cargo.toml` if necessary/possible
+- Ensure all necessary `package` fields are specified: https://doc.rust-lang.org/cargo/reference/publishing.html#before-publishing-a-new-crate
+- For author attribution: `The Nushell Project authors`
+- Include a proper MIT license file with necessary attribution (releasing via several distribution paths requires this)
+- (If crate in the monorepo) add the crate to the `workspace.members` table in the root `Cargo.toml` (currently dependabot only finds the crates specified there)
+- Let an experienced `nushell/publishing` member review the PR
+
+## Publishing a release
+
+This is handled by members of the `nushell/publishing` team
+
+The current procedure is specified [adjacent to the helper scripts](https://github.com/nushell/nu_scripts/tree/main/make_release)

devdocs/README.md

@@ -7,5 +7,6 @@ A complementary (currently stale) resource has been the [Nushell contributor boo
 ## Contents
 
 - [Developer FAQ](FAQ.md)
+- [How to/SOPs](HOWTOS.md)
 - [Platform support policy](PLATFORM_SUPPORT.md)
 - [Our Rust style](devdocs/rust_style.md)

src/tests/test_strings.rs

@@ -87,6 +87,72 @@ fn raw_string() -> TestResult {
     )
 }
 
+#[test]
+fn raw_string_inside_parentheses() -> TestResult {
+    let (left, right) = ('(', ')');
+    run_test(
+        &format!(r#"{left}r#'abcde""fghi"''''jkl'#{right}"#),
+        r#"abcde""fghi"''''jkl"#,
+    )?;
+    run_test(
+        &format!(r#"{left}r##'abcde""fghi"''''#jkl'##{right}"#),
+        r#"abcde""fghi"''''#jkl"#,
+    )?;
+    run_test(
+        &format!(r#"{left}r###'abcde""fghi"'''##'#jkl'###{right}"#),
+        r#"abcde""fghi"'''##'#jkl"#,
+    )?;
+    run_test(&format!("{left}r#''#{right}"), "")?;
+    run_test(
+        &format!(r#"{left}r#'a string with sharp inside # and ends with #'#{right}"#),
+        "a string with sharp inside # and ends with #",
+    )
+}
+
+#[test]
+fn raw_string_inside_list() -> TestResult {
+    let (left, right) = ('[', ']');
+    run_test(
+        &format!(r#"{left}r#'abcde""fghi"''''jkl'#{right} | get 0"#),
+        r#"abcde""fghi"''''jkl"#,
+    )?;
+    run_test(
+        &format!(r#"{left}r##'abcde""fghi"''''#jkl'##{right} | get 0"#),
+        r#"abcde""fghi"''''#jkl"#,
+    )?;
+    run_test(
+        &format!(r#"{left}r###'abcde""fghi"'''##'#jkl'###{right} | get 0"#),
+        r#"abcde""fghi"'''##'#jkl"#,
+    )?;
+    run_test(&format!("{left}r#''#{right} | get 0"), "")?;
+    run_test(
+        &format!(r#"{left}r#'a string with sharp inside # and ends with #'#{right} | get 0"#),
+        "a string with sharp inside # and ends with #",
+    )
+}
+
+#[test]
+fn raw_string_inside_closure() -> TestResult {
+    let (left, right) = ('{', '}');
+    run_test(
+        &format!(r#"do {left}r#'abcde""fghi"''''jkl'#{right}"#),
+        r#"abcde""fghi"''''jkl"#,
+    )?;
+    run_test(
+        &format!(r#"do {left}r##'abcde""fghi"''''#jkl'##{right}"#),
+        r#"abcde""fghi"''''#jkl"#,
+    )?;
+    run_test(
+        &format!(r#"do {left}r###'abcde""fghi"'''##'#jkl'###{right}"#),
+        r#"abcde""fghi"'''##'#jkl"#,
+    )?;
+    run_test(&format!("do {left}r#''#{right}"), "")?;
+    run_test(
+        &format!(r#"do {left}r#'a string with sharp inside # and ends with #'#{right}"#),
+        "a string with sharp inside # and ends with #",
+    )
+}
+
 #[test]
 fn incomplete_raw_string() -> TestResult {
     fail_test("r#abc", "expected '")