Bump to 0.90 (#11730)

<!--
if this PR closes one or more issues, you can automatically link the PR
with
them by using one of the [*linking
keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword),
e.g.
- this PR should close #xxxx
- fixes #xxxx

you can also mention related issues, PRs or discussions!
-->

# Description
<!--
Thank you for improving Nushell. Please, check our [contributing
guide](../CONTRIBUTING.md) and talk to the core team before making major
changes.

Description of your pull request goes here. **Provide examples and/or
screenshots** if your changes affect the user experience.
-->

# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to
check that you're using the standard code style
- `cargo test --workspace` to check that all tests pass (on Windows make
sure to [enable developer
mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
- `cargo run -- -c "use std testing; testing run-tests --path
crates/nu-std"` to run the tests for the standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->

.github/workflows/ci.yml

@@ -41,7 +41,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
         with:
           rustflags: ""
 
@@ -63,6 +63,10 @@ jobs:
         platform: [windows-latest, macos-latest, ubuntu-20.04]
         feature: [default, dataframe, extra]
         include:
+          # linux CI cannot handle clipboard feature
+          - default-flags: ""
+          - platform:  ubuntu-20.04
+            default-flags: "--no-default-features --features=default-no-clipboard"
           - feature: default
             flags: ""
           - feature: dataframe
@@ -85,12 +89,12 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
         with:
           rustflags: ""
 
       - name: Tests
-        run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.flags }}
+        run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} ${{ matrix.flags }}
 
       - name: Check for clean repo
         shell: bash
@@ -117,7 +121,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
         with:
           rustflags: ""
 
@@ -125,7 +129,7 @@ jobs:
         run: cargo install --path . --locked --no-default-features
 
       - name: Standard library tests
-        run: nu -c 'use std testing; testing run-tests --path crates/nu-std'
+        run: nu -c 'use crates/nu-std/testing.nu; testing run-tests --path crates/nu-std'
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -163,7 +167,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
         with:
           rustflags: ""
 

.github/workflows/nightly-build.yml

@@ -81,7 +81,7 @@ jobs:
         - x86_64-unknown-linux-musl
         - aarch64-unknown-linux-gnu
         - armv7-unknown-linux-gnueabihf
-        - riscv64gc-unknown-linux-gnu
+        # - riscv64gc-unknown-linux-gnu
         extra: ['bin']
         include:
         - target: aarch64-apple-darwin
@@ -118,9 +118,9 @@ jobs:
         - target: armv7-unknown-linux-gnueabihf
           os: ubuntu-20.04
           target_rustflags: ''
-        - target: riscv64gc-unknown-linux-gnu
-          os: ubuntu-latest
-          target_rustflags: ''
+        # - target: riscv64gc-unknown-linux-gnu
+        #   os: ubuntu-latest
+        #   target_rustflags: ''
 
     runs-on: ${{matrix.os}}
 
@@ -135,7 +135,7 @@ jobs:
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
       # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
         rustflags: ''
@@ -249,7 +249,7 @@ jobs:
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
       # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
         rustflags: ''

.github/workflows/release.yml

@@ -28,7 +28,7 @@ jobs:
         - x86_64-unknown-linux-musl
         - aarch64-unknown-linux-gnu
         - armv7-unknown-linux-gnueabihf
-        - riscv64gc-unknown-linux-gnu
+        # - riscv64gc-unknown-linux-gnu
         extra: ['bin']
         include:
         - target: aarch64-apple-darwin
@@ -65,9 +65,9 @@ jobs:
         - target: armv7-unknown-linux-gnueabihf
           os: ubuntu-20.04
           target_rustflags: ''
-        - target: riscv64gc-unknown-linux-gnu
-          os: ubuntu-latest
-          target_rustflags: ''
+        # - target: riscv64gc-unknown-linux-gnu
+        #   os: ubuntu-latest
+        #   target_rustflags: ''
 
     runs-on: ${{matrix.os}}
 
@@ -79,7 +79,7 @@ jobs:
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
       # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
         rustflags: ''
@@ -170,7 +170,7 @@ jobs:
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
+      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
       # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
         rustflags: ''

.github/workflows/typos.yml

@@ -10,6 +10,6 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Check spelling
-        uses: crate-ci/typos@v1.17.0
+        uses: crate-ci/typos@v1.18.0
         with:
           config: ./.github/.typos.toml

Cargo.toml

@@ -11,7 +11,7 @@ license = "MIT"
 name = "nu"
 repository = "https://github.com/nushell/nushell"
 rust-version = "1.72.1"
-version = "0.88.2"
+version = "0.90.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -53,30 +53,32 @@ members = [
 ]
 
 [dependencies]
-nu-cli = { path = "./crates/nu-cli", version = "0.88.2" }
-nu-color-config = { path = "./crates/nu-color-config", version = "0.88.2" }
-nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.88.2" }
-nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.88.2" }
-nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.88.2", features = ["dataframe"], optional = true }
-nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.88.2", optional = true }
-nu-command = { path = "./crates/nu-command", version = "0.88.2" }
-nu-engine = { path = "./crates/nu-engine", version = "0.88.2" }
-nu-explore = { path = "./crates/nu-explore", version = "0.88.2" }
-nu-json = { path = "./crates/nu-json", version = "0.88.2" }
-nu-lsp = { path = "./crates/nu-lsp/", version = "0.88.2" }
-nu-parser = { path = "./crates/nu-parser", version = "0.88.2" }
-nu-path = { path = "./crates/nu-path", version = "0.88.2" }
-nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.88.2" }
-nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.88.2" }
-nu-protocol = { path = "./crates/nu-protocol", version = "0.88.2" }
-nu-system = { path = "./crates/nu-system", version = "0.88.2" }
-nu-table = { path = "./crates/nu-table", version = "0.88.2" }
-nu-term-grid = { path = "./crates/nu-term-grid", version = "0.88.2" }
-nu-std = { path = "./crates/nu-std", version = "0.88.2" }
-nu-utils = { path = "./crates/nu-utils", version = "0.88.2" }
+nu-cli = { path = "./crates/nu-cli", version = "0.90.0" }
+nu-color-config = { path = "./crates/nu-color-config", version = "0.90.0" }
+nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.90.0" }
+nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.90.0" }
+nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.90.0", features = [
+	"dataframe",
+], optional = true }
+nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.90.0", optional = true }
+nu-command = { path = "./crates/nu-command", version = "0.90.0" }
+nu-engine = { path = "./crates/nu-engine", version = "0.90.0" }
+nu-explore = { path = "./crates/nu-explore", version = "0.90.0" }
+nu-json = { path = "./crates/nu-json", version = "0.90.0" }
+nu-lsp = { path = "./crates/nu-lsp/", version = "0.90.0" }
+nu-parser = { path = "./crates/nu-parser", version = "0.90.0" }
+nu-path = { path = "./crates/nu-path", version = "0.90.0" }
+nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.90.0" }
+nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.90.0" }
+nu-protocol = { path = "./crates/nu-protocol", version = "0.90.0" }
+nu-system = { path = "./crates/nu-system", version = "0.90.0" }
+nu-table = { path = "./crates/nu-table", version = "0.90.0" }
+nu-term-grid = { path = "./crates/nu-term-grid", version = "0.90.0" }
+nu-std = { path = "./crates/nu-std", version = "0.90.0" }
+nu-utils = { path = "./crates/nu-utils", version = "0.90.0" }
 
-nu-ansi-term = "0.49.0"
-reedline = { version = "0.27.0", features = ["bashisms", "sqlite"] }
+nu-ansi-term = "0.50.0"
+reedline = { version = "0.29.0", features = ["bashisms", "sqlite"] }
 
 crossterm = "0.27"
 ctrlc = "3.4"
@@ -90,7 +92,6 @@ time = "0.3"
 [target.'cfg(not(target_os = "windows"))'.dependencies]
 # Our dependencies don't use OpenSSL on Windows
 openssl = { version = "0.10", features = ["vendored"], optional = true }
-signal-hook = { version = "0.3", default-features = false }
 
 [target.'cfg(windows)'.build-dependencies]
 winresource = "0.1"
@@ -104,7 +105,7 @@ nix = { version = "0.27", default-features = false, features = [
 ] }
 
 [dev-dependencies]
-nu-test-support = { path = "./crates/nu-test-support", version = "0.88.2" }
+nu-test-support = { path = "./crates/nu-test-support", version = "0.90.0" }
 assert_cmd = "2.0"
 criterion = "0.5"
 pretty_assertions = "1.4"
@@ -121,7 +122,16 @@ plugin = [
 	"nu-protocol/plugin",
 	"nu-engine/plugin",
 ]
-default = ["plugin", "which-support", "trash-support", "sqlite", "mimalloc"]
+default = ["default-no-clipboard", "system-clipboard"]
+# Enables convenient omitting of the system-clipboard feature, as it leads to problems in ci on linux
+# See https://github.com/nushell/nushell/pull/11535
+default-no-clipboard = [
+	"plugin",
+	"which-support",
+	"trash-support",
+	"sqlite",
+	"mimalloc",
+]
 stable = ["default"]
 wasi = ["nu-cmd-lang/wasi"]
 # NOTE: individual features are also passed to `nu-cmd-lang` that uses them to generate the feature matrix in the `version` command
@@ -131,6 +141,7 @@ wasi = ["nu-cmd-lang/wasi"]
 static-link-openssl = ["dep:openssl", "nu-cmd-lang/static-link-openssl"]
 
 mimalloc = ["nu-cmd-lang/mimalloc", "dep:mimalloc"]
+system-clipboard = ["reedline/system_clipboard"]
 
 # Stable (Default)
 which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"]
@@ -172,8 +183,8 @@ bench = false
 
 # To use a development version of a dependency please use a global override here
 # changing versions in each sub-crate of the workspace is tedious
-[patch.crates-io]
-reedline = { git = "https://github.com/nushell/reedline.git", branch = "main" }
+#[patch.crates-io]
+#reedline = { git = "https://github.com/nushell/reedline", branch = "main" }
 # nu-ansi-term = {git = "https://github.com/nushell/nu-ansi-term.git", branch = "main"}
 # uu_cp = { git = "https://github.com/uutils/coreutils.git", branch = "main" }
 

crates/nu-cli/Cargo.toml

@@ -5,27 +5,27 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli"
 edition = "2021"
 license = "MIT"
 name = "nu-cli"
-version = "0.88.2"
+version = "0.90.0"
 
 [lib]
 bench = false
 
 [dev-dependencies]
-nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.88.2" }
-nu-command = { path = "../nu-command", version = "0.88.2" }
-nu-test-support = { path = "../nu-test-support", version = "0.88.2" }
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.90.0" }
+nu-command = { path = "../nu-command", version = "0.90.0" }
+nu-test-support = { path = "../nu-test-support", version = "0.90.0" }
 rstest = { version = "0.18.1", default-features = false }
 
 [dependencies]
-nu-cmd-base = { path = "../nu-cmd-base", version = "0.88.2" }
-nu-engine = { path = "../nu-engine", version = "0.88.2" }
-nu-path = { path = "../nu-path", version = "0.88.2" }
-nu-parser = { path = "../nu-parser", version = "0.88.2" }
-nu-protocol = { path = "../nu-protocol", version = "0.88.2" }
-nu-utils = { path = "../nu-utils", version = "0.88.2" }
-nu-color-config = { path = "../nu-color-config", version = "0.88.2" }
-nu-ansi-term = "0.49.0"
-reedline = { version = "0.27.0", features = ["bashisms", "sqlite"] }
+nu-cmd-base = { path = "../nu-cmd-base", version = "0.90.0" }
+nu-engine = { path = "../nu-engine", version = "0.90.0" }
+nu-path = { path = "../nu-path", version = "0.90.0" }
+nu-parser = { path = "../nu-parser", version = "0.90.0" }
+nu-protocol = { path = "../nu-protocol", version = "0.90.0" }
+nu-utils = { path = "../nu-utils", version = "0.90.0" }
+nu-color-config = { path = "../nu-color-config", version = "0.90.0" }
+nu-ansi-term = "0.50.0"
+reedline = { version = "0.29.0", features = ["bashisms", "sqlite"] }
 
 chrono = { default-features = false, features = ["std"], version = "0.4" }
 crossterm = "0.27"

crates/nu-cli/src/commands/commandline.rs

@@ -71,7 +71,7 @@ impl Command for Commandline {
         if let Some(cmd) = call.opt::<Value>(engine_state, stack, 0)? {
             let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
 
-            if call.has_flag("cursor") {
+            if call.has_flag(engine_state, stack, "cursor")? {
                 let cmd_str = cmd.as_string()?;
                 match cmd_str.parse::<i64>() {
                     Ok(n) => {
@@ -96,9 +96,9 @@ impl Command for Commandline {
                         })
                     }
                 }
-            } else if call.has_flag("append") {
+            } else if call.has_flag(engine_state, stack, "append")? {
                 repl.buffer.push_str(&cmd.as_string()?);
-            } else if call.has_flag("insert") {
+            } else if call.has_flag(engine_state, stack, "insert")? {
                 let cmd_str = cmd.as_string()?;
                 let cursor_pos = repl.cursor_pos;
                 repl.buffer.insert_str(cursor_pos, &cmd_str);
@@ -110,10 +110,10 @@ impl Command for Commandline {
             Ok(Value::nothing(call.head).into_pipeline_data())
         } else {
             let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
-            if call.has_flag("cursor-end") {
-                repl.cursor_pos = repl.buffer.graphemes(true).count();
+            if call.has_flag(engine_state, stack, "cursor-end")? {
+                repl.cursor_pos = repl.buffer.len();
                 Ok(Value::nothing(call.head).into_pipeline_data())
-            } else if call.has_flag("cursor") {
+            } else if call.has_flag(engine_state, stack, "cursor")? {
                 let char_pos = repl
                     .buffer
                     .grapheme_indices(true)

crates/nu-cli/src/commands/history/history_.rs

@@ -1,3 +1,4 @@
+use nu_engine::CallExt;
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
 use nu_protocol::{
@@ -23,10 +24,7 @@ impl Command for History {
 
     fn signature(&self) -> nu_protocol::Signature {
         Signature::build("history")
-            .input_output_types(vec![
-                (Type::Nothing, Type::Table(vec![])),
-                (Type::Nothing, Type::Nothing),
-            ])
+            .input_output_types(vec![(Type::Nothing, Type::Any)])
             .allow_variants_without_examples(true)
             .switch("clear", "Clears out the history entries", Some('c'))
             .switch(
@@ -40,21 +38,25 @@ impl Command for History {
     fn run(
         &self,
         engine_state: &EngineState,
-        _stack: &mut Stack,
+        stack: &mut Stack,
         call: &Call,
         _input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let head = call.head;
 
+        let Some(history) = engine_state.history_config() else {
+            return Ok(PipelineData::empty());
+        };
+
         // todo for sqlite history this command should be an alias to `open ~/.config/nushell/history.sqlite3 | get history`
         if let Some(config_path) = nu_path::config_dir() {
-            let clear = call.has_flag("clear");
-            let long = call.has_flag("long");
+            let clear = call.has_flag(engine_state, stack, "clear")?;
+            let long = call.has_flag(engine_state, stack, "long")?;
             let ctrlc = engine_state.ctrlc.clone();
 
             let mut history_path = config_path;
             history_path.push("nushell");
-            match engine_state.config.history_file_format {
+            match history.file_format {
                 HistoryFileFormat::Sqlite => {
                     history_path.push("history.sqlite3");
                 }
@@ -68,29 +70,27 @@ impl Command for History {
                 // TODO: FIXME also clear the auxiliary files when using sqlite
                 Ok(PipelineData::empty())
             } else {
-                let history_reader: Option<Box<dyn ReedlineHistory>> =
-                    match engine_state.config.history_file_format {
-                        HistoryFileFormat::Sqlite => {
-                            SqliteBackedHistory::with_file(history_path, None, None)
-                                .map(|inner| {
-                                    let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
-                                    boxed
-                                })
-                                .ok()
-                        }
+                let history_reader: Option<Box<dyn ReedlineHistory>> = match history.file_format {
+                    HistoryFileFormat::Sqlite => {
+                        SqliteBackedHistory::with_file(history_path, None, None)
+                            .map(|inner| {
+                                let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
+                                boxed
+                            })
+                            .ok()
+                    }
 
-                        HistoryFileFormat::PlainText => FileBackedHistory::with_file(
-                            engine_state.config.max_history_size as usize,
-                            history_path,
-                        )
-                        .map(|inner| {
-                            let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
-                            boxed
-                        })
-                        .ok(),
-                    };
+                    HistoryFileFormat::PlainText => {
+                        FileBackedHistory::with_file(history.max_size as usize, history_path)
+                            .map(|inner| {
+                                let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
+                                boxed
+                            })
+                            .ok()
+                    }
+                };
 
-                match engine_state.config.history_file_format {
+                match history.file_format {
                     HistoryFileFormat::PlainText => Ok(history_reader
                         .and_then(|h| {
                             h.search(SearchQuery::everything(SearchDirection::Forward, None))

crates/nu-cli/src/completions/command_completions.rs

@@ -94,6 +94,7 @@ impl CommandCompletion {
             .map(move |x| Suggestion {
                 value: String::from_utf8_lossy(&x.0).to_string(),
                 description: x.1,
+                style: None,
                 extra: None,
                 span: reedline::Span::new(span.start - offset, span.end - offset),
                 append_whitespace: true,
@@ -110,6 +111,7 @@ impl CommandCompletion {
                 .map(move |x| Suggestion {
                     value: x,
                     description: None,
+                    style: None,
                     extra: None,
                     span: reedline::Span::new(span.start - offset, span.end - offset),
                     append_whitespace: true,
@@ -123,6 +125,7 @@ impl CommandCompletion {
                     results.push(Suggestion {
                         value: format!("^{}", external.value),
                         description: None,
+                        style: None,
                         extra: None,
                         span: external.span,
                         append_whitespace: true,

crates/nu-cli/src/completions/completer.rs

@@ -2,6 +2,7 @@ use crate::completions::{
     CommandCompletion, Completer, CompletionOptions, CustomCompletion, DirectoryCompletion,
     DotNuCompletion, FileCompletion, FlagCompletion, VariableCompletion,
 };
+use nu_color_config::{color_record_to_nustyle, lookup_ansi_color_style};
 use nu_engine::eval_block;
 use nu_parser::{flatten_expression, parse, FlatShape};
 use nu_protocol::{
@@ -110,10 +111,16 @@ impl NuCompleter {
     fn completion_helper(&mut self, line: &str, pos: usize) -> Vec<Suggestion> {
         let mut working_set = StateWorkingSet::new(&self.engine_state);
         let offset = working_set.next_span_start();
+        // TODO: Callers should be trimming the line themselves
+        let line = if line.len() > pos { &line[..pos] } else { line };
+        // Adjust offset so that the spans of the suggestions will start at the right
+        // place even with `only_buffer_difference: true`
+        let fake_offset = offset + line.len() - pos;
+        let pos = offset + line.len();
         let initial_line = line.to_string();
         let mut line = line.to_string();
-        line.insert(pos, 'a');
-        let pos = offset + pos;
+        line.push('a');
+
         let config = self.engine_state.get_config();
 
         let output = parse(&mut working_set, Some("completer"), line.as_bytes(), false);
@@ -186,7 +193,7 @@ impl NuCompleter {
                                         &working_set,
                                         prefix,
                                         new_span,
-                                        offset,
+                                        fake_offset,
                                         pos,
                                     );
                                 }
@@ -200,7 +207,7 @@ impl NuCompleter {
                                         &working_set,
                                         prefix.clone(),
                                         new_span,
-                                        offset,
+                                        fake_offset,
                                         pos,
                                     );
 
@@ -211,9 +218,12 @@ impl NuCompleter {
                                     // We got no results for internal completion
                                     // now we can check if external completer is set and use it
                                     if let Some(block_id) = config.external_completer {
-                                        if let Some(external_result) = self
-                                            .external_completion(block_id, &spans, offset, new_span)
-                                        {
+                                        if let Some(external_result) = self.external_completion(
+                                            block_id,
+                                            &spans,
+                                            fake_offset,
+                                            new_span,
+                                        ) {
                                             return external_result;
                                         }
                                     }
@@ -237,7 +247,7 @@ impl NuCompleter {
                                         &working_set,
                                         prefix,
                                         new_span,
-                                        offset,
+                                        fake_offset,
                                         pos,
                                     );
                                 }
@@ -262,7 +272,7 @@ impl NuCompleter {
                                                 &working_set,
                                                 prefix,
                                                 new_span,
-                                                offset,
+                                                fake_offset,
                                                 pos,
                                             );
                                         } else if prev_expr_str == b"ls" {
@@ -274,7 +284,7 @@ impl NuCompleter {
                                                 &working_set,
                                                 prefix,
                                                 new_span,
-                                                offset,
+                                                fake_offset,
                                                 pos,
                                             );
                                         }
@@ -296,7 +306,7 @@ impl NuCompleter {
                                             &working_set,
                                             prefix,
                                             new_span,
-                                            offset,
+                                            fake_offset,
                                             pos,
                                         );
                                     }
@@ -309,7 +319,7 @@ impl NuCompleter {
                                             &working_set,
                                             prefix,
                                             new_span,
-                                            offset,
+                                            fake_offset,
                                             pos,
                                         );
                                     }
@@ -322,7 +332,7 @@ impl NuCompleter {
                                             &working_set,
                                             prefix,
                                             new_span,
-                                            offset,
+                                            fake_offset,
                                             pos,
                                         );
                                     }
@@ -341,7 +351,7 @@ impl NuCompleter {
                                             &working_set,
                                             prefix.clone(),
                                             new_span,
-                                            offset,
+                                            fake_offset,
                                             pos,
                                         );
 
@@ -352,7 +362,10 @@ impl NuCompleter {
                                         // Try to complete using an external completer (if set)
                                         if let Some(block_id) = config.external_completer {
                                             if let Some(external_result) = self.external_completion(
-                                                block_id, &spans, offset, new_span,
+                                                block_id,
+                                                &spans,
+                                                fake_offset,
+                                                new_span,
                                             ) {
                                                 return external_result;
                                             }
@@ -366,7 +379,7 @@ impl NuCompleter {
                                             &working_set,
                                             prefix,
                                             new_span,
-                                            offset,
+                                            fake_offset,
                                             pos,
                                         );
 
@@ -453,6 +466,7 @@ pub fn map_value_completions<'a>(
             return Some(Suggestion {
                 value: s,
                 description: None,
+                style: None,
                 extra: None,
                 span: reedline::Span {
                     start: span.start - offset,
@@ -467,6 +481,7 @@ pub fn map_value_completions<'a>(
             let mut suggestion = Suggestion {
                 value: String::from(""), // Initialize with empty string
                 description: None,
+                style: None,
                 extra: None,
                 span: reedline::Span {
                     start: span.start - offset,
@@ -494,6 +509,16 @@ pub fn map_value_completions<'a>(
                         suggestion.description = Some(desc_str);
                     }
                 }
+
+                // Match `style` column
+                if it.0 == "style" {
+                    // Convert the value to string
+                    suggestion.style = match it.1 {
+                        Value::String { val, .. } => Some(lookup_ansi_color_style(val)),
+                        Value::Record { .. } => Some(color_record_to_nustyle(it.1)),
+                        _ => None,
+                    };
+                }
             });
 
             return Some(suggestion);

crates/nu-cli/src/completions/directory_completions.rs

@@ -44,6 +44,7 @@ impl Completer for DirectoryCompletion {
         .map(move |x| Suggestion {
             value: x.1,
             description: None,
+            style: None,
             extra: None,
             span: reedline::Span {
                 start: x.0.start - offset,

crates/nu-cli/src/completions/dotnu_completions.rs

@@ -111,6 +111,7 @@ impl Completer for DotNuCompletion {
                     .map(move |x| Suggestion {
                         value: x.1,
                         description: None,
+                        style: None,
                         extra: None,
                         span: reedline::Span {
                             start: x.0.start - offset,

crates/nu-cli/src/completions/file_completions.rs

@@ -49,6 +49,7 @@ impl Completer for FileCompletion {
         .map(move |x| Suggestion {
             value: x.1,
             description: None,
+            style: None,
             extra: None,
             span: reedline::Span {
                 start: x.0.start - offset,

crates/nu-cli/src/completions/flag_completions.rs

@@ -46,6 +46,7 @@ impl Completer for FlagCompletion {
                         output.push(Suggestion {
                             value: String::from_utf8_lossy(&named).to_string(),
                             description: Some(flag_desc.to_string()),
+                            style: None,
                             extra: None,
                             span: reedline::Span {
                                 start: span.start - offset,
@@ -68,6 +69,7 @@ impl Completer for FlagCompletion {
                     output.push(Suggestion {
                         value: String::from_utf8_lossy(&named).to_string(),
                         description: Some(flag_desc.to_string()),
+                        style: None,
                         extra: None,
                         span: reedline::Span {
                             start: span.start - offset,

crates/nu-cli/src/completions/variable_completions.rs

@@ -95,6 +95,7 @@ impl Completer for VariableCompletion {
                             output.push(Suggestion {
                                 value: env_var.0,
                                 description: None,
+                                style: None,
                                 extra: None,
                                 span: current_span,
                                 append_whitespace: false,
@@ -165,6 +166,7 @@ impl Completer for VariableCompletion {
                 output.push(Suggestion {
                     value: builtin.to_string(),
                     description: None,
+                    style: None,
                     extra: None,
                     span: current_span,
                     append_whitespace: false,
@@ -187,6 +189,7 @@ impl Completer for VariableCompletion {
                         output.push(Suggestion {
                             value: String::from_utf8_lossy(v.0).to_string(),
                             description: None,
+                            style: None,
                             extra: None,
                             span: current_span,
                             append_whitespace: false,
@@ -208,6 +211,7 @@ impl Completer for VariableCompletion {
                     output.push(Suggestion {
                         value: String::from_utf8_lossy(v.0).to_string(),
                         description: None,
+                        style: None,
                         extra: None,
                         span: current_span,
                         append_whitespace: false,
@@ -239,6 +243,7 @@ fn nested_suggestions(
                 output.push(Suggestion {
                     value: col,
                     description: None,
+                    style: None,
                     extra: None,
                     span: current_span,
                     append_whitespace: false,
@@ -253,6 +258,7 @@ fn nested_suggestions(
                 output.push(Suggestion {
                     value: column_name.to_string(),
                     description: None,
+                    style: None,
                     extra: None,
                     span: current_span,
                     append_whitespace: false,
@@ -266,6 +272,7 @@ fn nested_suggestions(
                 output.push(Suggestion {
                     value: column_name,
                     description: None,
+                    style: None,
                     extra: None,
                     span: current_span,
                     append_whitespace: false,

crates/nu-cli/src/lib.rs

@@ -19,7 +19,7 @@ pub use completions::{FileCompletion, NuCompleter};
 pub use config_files::eval_config_contents;
 pub use eval_cmds::evaluate_commands;
 pub use eval_file::evaluate_file;
-pub use menus::{DescriptionMenu, NuHelpCompleter};
+pub use menus::NuHelpCompleter;
 pub use nu_cmd_base::util::get_init_cwd;
 pub use nu_highlight::NuHighlight;
 pub use print::Print;

crates/nu-cli/src/menus/description_menu.rs

@@ -1,730 +0,0 @@
-use {
-    nu_ansi_term::{ansi::RESET, Style},
-    reedline::{
-        menu_functions::string_difference, Completer, Editor, Menu, MenuEvent, MenuTextStyle,
-        Painter, Suggestion, UndoBehavior,
-    },
-};
-
-/// Default values used as reference for the menu. These values are set during
-/// the initial declaration of the menu and are always kept as reference for the
-/// changeable [`WorkingDetails`]
-struct DefaultMenuDetails {
-    /// Number of columns that the menu will have
-    pub columns: u16,
-    /// Column width
-    pub col_width: Option<usize>,
-    /// Column padding
-    pub col_padding: usize,
-    /// Number of rows for commands
-    pub selection_rows: u16,
-    /// Number of rows allowed to display the description
-    pub description_rows: usize,
-}
-
-impl Default for DefaultMenuDetails {
-    fn default() -> Self {
-        Self {
-            columns: 4,
-            col_width: None,
-            col_padding: 2,
-            selection_rows: 4,
-            description_rows: 10,
-        }
-    }
-}
-
-/// Represents the actual column conditions of the menu. These conditions change
-/// since they need to accommodate possible different line sizes for the column values
-#[derive(Default)]
-struct WorkingDetails {
-    /// Number of columns that the menu will have
-    pub columns: u16,
-    /// Column width
-    pub col_width: usize,
-    /// Number of rows for description
-    pub description_rows: usize,
-}
-
-/// Completion menu definition
-pub struct DescriptionMenu {
-    /// Menu name
-    name: String,
-    /// Menu status
-    active: bool,
-    /// Menu coloring
-    color: MenuTextStyle,
-    /// Default column details that are set when creating the menu
-    /// These values are the reference for the working details
-    default_details: DefaultMenuDetails,
-    /// Number of minimum rows that are displayed when
-    /// the required lines is larger than the available lines
-    min_rows: u16,
-    /// Working column details keep changing based on the collected values
-    working_details: WorkingDetails,
-    /// Menu cached values
-    values: Vec<Suggestion>,
-    /// column position of the cursor. Starts from 0
-    col_pos: u16,
-    /// row position in the menu. Starts from 0
-    row_pos: u16,
-    /// Menu marker when active
-    marker: String,
-    /// Event sent to the menu
-    event: Option<MenuEvent>,
-    /// String collected after the menu is activated
-    input: Option<String>,
-    /// Examples to select
-    examples: Vec<String>,
-    /// Example index
-    example_index: Option<usize>,
-    /// Examples may not be shown if there is not enough space in the screen
-    show_examples: bool,
-    /// Skipped description rows
-    skipped_rows: usize,
-    /// Calls the completer using only the line buffer difference difference
-    /// after the menu was activated
-    only_buffer_difference: bool,
-}
-
-impl Default for DescriptionMenu {
-    fn default() -> Self {
-        Self {
-            name: "description_menu".to_string(),
-            active: false,
-            color: MenuTextStyle::default(),
-            default_details: DefaultMenuDetails::default(),
-            min_rows: 3,
-            working_details: WorkingDetails::default(),
-            values: Vec::new(),
-            col_pos: 0,
-            row_pos: 0,
-            marker: "? ".to_string(),
-            event: None,
-            input: None,
-            examples: Vec::new(),
-            example_index: None,
-            show_examples: true,
-            skipped_rows: 0,
-            only_buffer_difference: true,
-        }
-    }
-}
-
-// Menu configuration
-impl DescriptionMenu {
-    /// Menu builder with new name
-    pub fn with_name(mut self, name: &str) -> Self {
-        self.name = name.into();
-        self
-    }
-
-    /// Menu builder with new value for text style
-    pub fn with_text_style(mut self, text_style: Style) -> Self {
-        self.color.text_style = text_style;
-        self
-    }
-
-    /// Menu builder with new value for text style
-    pub fn with_selected_text_style(mut self, selected_text_style: Style) -> Self {
-        self.color.selected_text_style = selected_text_style;
-        self
-    }
-
-    /// Menu builder with new value for text style
-    pub fn with_description_text_style(mut self, description_text_style: Style) -> Self {
-        self.color.description_style = description_text_style;
-        self
-    }
-
-    /// Menu builder with new columns value
-    pub fn with_columns(mut self, columns: u16) -> Self {
-        self.default_details.columns = columns;
-        self
-    }
-
-    /// Menu builder with new column width value
-    pub fn with_column_width(mut self, col_width: Option<usize>) -> Self {
-        self.default_details.col_width = col_width;
-        self
-    }
-
-    /// Menu builder with new column width value
-    pub fn with_column_padding(mut self, col_padding: usize) -> Self {
-        self.default_details.col_padding = col_padding;
-        self
-    }
-
-    /// Menu builder with new selection rows value
-    pub fn with_selection_rows(mut self, selection_rows: u16) -> Self {
-        self.default_details.selection_rows = selection_rows;
-        self
-    }
-
-    /// Menu builder with new description rows value
-    pub fn with_description_rows(mut self, description_rows: usize) -> Self {
-        self.default_details.description_rows = description_rows;
-        self
-    }
-
-    /// Menu builder with marker
-    pub fn with_marker(mut self, marker: String) -> Self {
-        self.marker = marker;
-        self
-    }
-
-    /// Menu builder with new only buffer difference
-    pub fn with_only_buffer_difference(mut self, only_buffer_difference: bool) -> Self {
-        self.only_buffer_difference = only_buffer_difference;
-        self
-    }
-}
-
-// Menu functionality
-impl DescriptionMenu {
-    /// Move menu cursor to the next element
-    fn move_next(&mut self) {
-        let mut new_col = self.col_pos + 1;
-        let mut new_row = self.row_pos;
-
-        if new_col >= self.get_cols() {
-            new_row += 1;
-            new_col = 0;
-        }
-
-        if new_row >= self.get_rows() {
-            new_row = 0;
-            new_col = 0;
-        }
-
-        let position = new_row * self.get_cols() + new_col;
-        if position >= self.get_values().len() as u16 {
-            self.reset_position();
-        } else {
-            self.col_pos = new_col;
-            self.row_pos = new_row;
-        }
-    }
-
-    /// Move menu cursor to the previous element
-    fn move_previous(&mut self) {
-        let new_col = self.col_pos.checked_sub(1);
-
-        let (new_col, new_row) = match new_col {
-            Some(col) => (col, self.row_pos),
-            None => match self.row_pos.checked_sub(1) {
-                Some(row) => (self.get_cols().saturating_sub(1), row),
-                None => (
-                    self.get_cols().saturating_sub(1),
-                    self.get_rows().saturating_sub(1),
-                ),
-            },
-        };
-
-        let position = new_row * self.get_cols() + new_col;
-        if position >= self.get_values().len() as u16 {
-            self.col_pos = (self.get_values().len() as u16 % self.get_cols()).saturating_sub(1);
-            self.row_pos = self.get_rows().saturating_sub(1);
-        } else {
-            self.col_pos = new_col;
-            self.row_pos = new_row;
-        }
-    }
-
-    /// Menu index based on column and row position
-    fn index(&self) -> usize {
-        let index = self.row_pos * self.get_cols() + self.col_pos;
-        index as usize
-    }
-
-    /// Get selected value from the menu
-    fn get_value(&self) -> Option<Suggestion> {
-        self.get_values().get(self.index()).cloned()
-    }
-
-    /// Calculates how many rows the Menu will use
-    fn get_rows(&self) -> u16 {
-        let values = self.get_values().len() as u16;
-
-        if values == 0 {
-            // When the values are empty the no_records_msg is shown, taking 1 line
-            return 1;
-        }
-
-        let rows = values / self.get_cols();
-        if values % self.get_cols() != 0 {
-            rows + 1
-        } else {
-            rows
-        }
-    }
-
-    /// Returns working details col width
-    fn get_width(&self) -> usize {
-        self.working_details.col_width
-    }
-
-    /// Reset menu position
-    fn reset_position(&mut self) {
-        self.col_pos = 0;
-        self.row_pos = 0;
-        self.skipped_rows = 0;
-    }
-
-    fn no_records_msg(&self, use_ansi_coloring: bool) -> String {
-        let msg = "TYPE TO START SEARCH";
-        if use_ansi_coloring {
-            format!(
-                "{}{}{}",
-                self.color.selected_text_style.prefix(),
-                msg,
-                RESET
-            )
-        } else {
-            msg.to_string()
-        }
-    }
-
-    /// Returns working details columns
-    fn get_cols(&self) -> u16 {
-        self.working_details.columns.max(1)
-    }
-
-    /// End of line for menu
-    fn end_of_line(&self, column: u16, index: usize) -> &str {
-        let is_last = index == self.values.len().saturating_sub(1);
-        if column == self.get_cols().saturating_sub(1) || is_last {
-            "\r\n"
-        } else {
-            ""
-        }
-    }
-
-    /// Update list of examples from the actual value
-    fn update_examples(&mut self) {
-        self.examples = self
-            .get_value()
-            .and_then(|suggestion| suggestion.extra)
-            .unwrap_or_default();
-
-        self.example_index = None;
-    }
-
-    /// Creates default string that represents one suggestion from the menu
-    fn create_entry_string(
-        &self,
-        suggestion: &Suggestion,
-        index: usize,
-        column: u16,
-        empty_space: usize,
-        use_ansi_coloring: bool,
-    ) -> String {
-        if use_ansi_coloring {
-            if index == self.index() {
-                format!(
-                    "{}{}{}{:>empty$}{}",
-                    self.color.selected_text_style.prefix(),
-                    &suggestion.value,
-                    RESET,
-                    "",
-                    self.end_of_line(column, index),
-                    empty = empty_space,
-                )
-            } else {
-                format!(
-                    "{}{}{}{:>empty$}{}",
-                    self.color.text_style.prefix(),
-                    &suggestion.value,
-                    RESET,
-                    "",
-                    self.end_of_line(column, index),
-                    empty = empty_space,
-                )
-            }
-        } else {
-            // If no ansi coloring is found, then the selection word is
-            // the line in uppercase
-            let (marker, empty_space) = if index == self.index() {
-                (">", empty_space.saturating_sub(1))
-            } else {
-                ("", empty_space)
-            };
-
-            let line = format!(
-                "{}{}{:>empty$}{}",
-                marker,
-                &suggestion.value,
-                "",
-                self.end_of_line(column, index),
-                empty = empty_space,
-            );
-
-            if index == self.index() {
-                line.to_uppercase()
-            } else {
-                line
-            }
-        }
-    }
-
-    /// Description string with color
-    fn create_description_string(&self, use_ansi_coloring: bool) -> String {
-        let description = self
-            .get_value()
-            .and_then(|suggestion| suggestion.description)
-            .unwrap_or_default()
-            .lines()
-            .skip(self.skipped_rows)
-            .take(self.working_details.description_rows)
-            .collect::<Vec<&str>>()
-            .join("\r\n");
-
-        if use_ansi_coloring && !description.is_empty() {
-            format!(
-                "{}{}{}",
-                self.color.description_style.prefix(),
-                description,
-                RESET,
-            )
-        } else {
-            description
-        }
-    }
-
-    /// Selectable list of examples from the actual value
-    fn create_example_string(&self, use_ansi_coloring: bool) -> String {
-        if !self.show_examples {
-            return "".into();
-        }
-
-        let examples: String = self
-            .examples
-            .iter()
-            .enumerate()
-            .map(|(index, example)| {
-                if let Some(example_index) = self.example_index {
-                    if index == example_index {
-                        format!(
-                            "  {}{}{}\r\n",
-                            self.color.selected_text_style.prefix(),
-                            example,
-                            RESET
-                        )
-                    } else {
-                        format!("  {example}\r\n")
-                    }
-                } else {
-                    format!("  {example}\r\n")
-                }
-            })
-            .collect();
-
-        if examples.is_empty() {
-            "".into()
-        } else if use_ansi_coloring {
-            format!(
-                "{}\r\n\r\nExamples:\r\n{}{}",
-                self.color.description_style.prefix(),
-                RESET,
-                examples,
-            )
-        } else {
-            format!("\r\n\r\nExamples:\r\n{examples}",)
-        }
-    }
-}
-
-impl Menu for DescriptionMenu {
-    /// Menu name
-    fn name(&self) -> &str {
-        self.name.as_str()
-    }
-
-    /// Menu indicator
-    fn indicator(&self) -> &str {
-        self.marker.as_str()
-    }
-
-    /// Deactivates context menu
-    fn is_active(&self) -> bool {
-        self.active
-    }
-
-    /// The menu stays active even with one record
-    fn can_quick_complete(&self) -> bool {
-        false
-    }
-
-    /// The menu does not need to partially complete
-    fn can_partially_complete(
-        &mut self,
-        _values_updated: bool,
-        _editor: &mut Editor,
-        _completer: &mut dyn Completer,
-    ) -> bool {
-        false
-    }
-
-    /// Selects what type of event happened with the menu
-    fn menu_event(&mut self, event: MenuEvent) {
-        match &event {
-            MenuEvent::Activate(_) => self.active = true,
-            MenuEvent::Deactivate => {
-                self.active = false;
-                self.input = None;
-                self.values = Vec::new();
-            }
-            _ => {}
-        };
-
-        self.event = Some(event);
-    }
-
-    /// Updates menu values
-    fn update_values(&mut self, editor: &mut Editor, completer: &mut dyn Completer) {
-        if self.only_buffer_difference {
-            if let Some(old_string) = &self.input {
-                let (start, input) = string_difference(editor.get_buffer(), old_string);
-                if !input.is_empty() {
-                    self.reset_position();
-                    self.values = completer.complete(input, start);
-                }
-            }
-        } else {
-            let trimmed_buffer = editor.get_buffer().replace('\n', " ");
-            self.values = completer.complete(
-                trimmed_buffer.as_str(),
-                editor.line_buffer().insertion_point(),
-            );
-            self.reset_position();
-        }
-    }
-
-    /// The working details for the menu changes based on the size of the lines
-    /// collected from the completer
-    fn update_working_details(
-        &mut self,
-        editor: &mut Editor,
-        completer: &mut dyn Completer,
-        painter: &Painter,
-    ) {
-        if let Some(event) = self.event.take() {
-            // Updating all working parameters from the menu before executing any of the
-            // possible event
-            let max_width = self.get_values().iter().fold(0, |acc, suggestion| {
-                let str_len = suggestion.value.len() + self.default_details.col_padding;
-                if str_len > acc {
-                    str_len
-                } else {
-                    acc
-                }
-            });
-
-            // If no default width is found, then the total screen width is used to estimate
-            // the column width based on the default number of columns
-            let default_width = if let Some(col_width) = self.default_details.col_width {
-                col_width
-            } else {
-                let col_width = painter.screen_width() / self.default_details.columns;
-                col_width as usize
-            };
-
-            // Adjusting the working width of the column based the max line width found
-            // in the menu values
-            if max_width > default_width {
-                self.working_details.col_width = max_width;
-            } else {
-                self.working_details.col_width = default_width;
-            };
-
-            // The working columns is adjusted based on possible number of columns
-            // that could be fitted in the screen with the calculated column width
-            let possible_cols = painter.screen_width() / self.working_details.col_width as u16;
-            if possible_cols > self.default_details.columns {
-                self.working_details.columns = self.default_details.columns.max(1);
-            } else {
-                self.working_details.columns = possible_cols;
-            }
-
-            // Updating the working rows to display the description
-            if self.menu_required_lines(painter.screen_width()) <= painter.remaining_lines() {
-                self.working_details.description_rows = self.default_details.description_rows;
-                self.show_examples = true;
-            } else {
-                self.working_details.description_rows = painter
-                    .remaining_lines()
-                    .saturating_sub(self.default_details.selection_rows + 1)
-                    as usize;
-
-                self.show_examples = false;
-            }
-
-            match event {
-                MenuEvent::Activate(_) => {
-                    self.reset_position();
-                    self.input = Some(editor.get_buffer().to_string());
-                    self.update_values(editor, completer);
-                }
-                MenuEvent::Deactivate => self.active = false,
-                MenuEvent::Edit(_) => {
-                    self.reset_position();
-                    self.update_values(editor, completer);
-                    self.update_examples()
-                }
-                MenuEvent::NextElement => {
-                    self.skipped_rows = 0;
-                    self.move_next();
-                    self.update_examples();
-                }
-                MenuEvent::PreviousElement => {
-                    self.skipped_rows = 0;
-                    self.move_previous();
-                    self.update_examples();
-                }
-                MenuEvent::MoveUp => {
-                    if let Some(example_index) = self.example_index {
-                        if let Some(index) = example_index.checked_sub(1) {
-                            self.example_index = Some(index);
-                        } else {
-                            self.example_index = Some(self.examples.len().saturating_sub(1));
-                        }
-                    } else if !self.examples.is_empty() {
-                        self.example_index = Some(0);
-                    }
-                }
-                MenuEvent::MoveDown => {
-                    if let Some(example_index) = self.example_index {
-                        let index = example_index + 1;
-                        if index < self.examples.len() {
-                            self.example_index = Some(index);
-                        } else {
-                            self.example_index = Some(0);
-                        }
-                    } else if !self.examples.is_empty() {
-                        self.example_index = Some(0);
-                    }
-                }
-                MenuEvent::MoveLeft => self.skipped_rows = self.skipped_rows.saturating_sub(1),
-                MenuEvent::MoveRight => {
-                    let skipped = self.skipped_rows + 1;
-                    let description_rows = self
-                        .get_value()
-                        .and_then(|suggestion| suggestion.description)
-                        .unwrap_or_default()
-                        .lines()
-                        .count();
-
-                    let allowed_skips =
-                        description_rows.saturating_sub(self.working_details.description_rows);
-
-                    if skipped < allowed_skips {
-                        self.skipped_rows = skipped;
-                    } else {
-                        self.skipped_rows = allowed_skips;
-                    }
-                }
-                MenuEvent::PreviousPage | MenuEvent::NextPage => {}
-            }
-        }
-    }
-
-    /// The buffer gets replaced in the Span location
-    fn replace_in_buffer(&self, editor: &mut Editor) {
-        if let Some(Suggestion { value, span, .. }) = self.get_value() {
-            let start = span.start.min(editor.line_buffer().len());
-            let end = span.end.min(editor.line_buffer().len());
-
-            let replacement = if let Some(example_index) = self.example_index {
-                self.examples
-                    .get(example_index)
-                    .expect("the example index is always checked")
-            } else {
-                &value
-            };
-
-            editor.edit_buffer(
-                |lb| {
-                    lb.replace_range(start..end, replacement);
-                    let mut offset = lb.insertion_point();
-                    offset += lb
-                        .len()
-                        .saturating_sub(end.saturating_sub(start))
-                        .saturating_sub(start);
-                    lb.set_insertion_point(offset);
-                },
-                UndoBehavior::CreateUndoPoint,
-            );
-        }
-    }
-
-    /// Minimum rows that should be displayed by the menu
-    fn min_rows(&self) -> u16 {
-        self.get_rows().min(self.min_rows)
-    }
-
-    /// Gets values from filler that will be displayed in the menu
-    fn get_values(&self) -> &[Suggestion] {
-        &self.values
-    }
-
-    fn menu_required_lines(&self, _terminal_columns: u16) -> u16 {
-        let example_lines = self
-            .examples
-            .iter()
-            .fold(0, |acc, example| example.lines().count() + acc);
-
-        self.default_details.selection_rows
-            + self.default_details.description_rows as u16
-            + example_lines as u16
-            + 3
-    }
-
-    fn menu_string(&self, _available_lines: u16, use_ansi_coloring: bool) -> String {
-        if self.get_values().is_empty() {
-            self.no_records_msg(use_ansi_coloring)
-        } else {
-            // The skip values represent the number of lines that should be skipped
-            // while printing the menu
-            let available_lines = self.default_details.selection_rows;
-            let skip_values = if self.row_pos >= available_lines {
-                let skip_lines = self.row_pos.saturating_sub(available_lines) + 1;
-                (skip_lines * self.get_cols()) as usize
-            } else {
-                0
-            };
-
-            // It seems that crossterm prefers to have a complete string ready to be printed
-            // rather than looping through the values and printing multiple things
-            // This reduces the flickering when printing the menu
-            let available_values = (available_lines * self.get_cols()) as usize;
-            let selection_values: String = self
-                .get_values()
-                .iter()
-                .skip(skip_values)
-                .take(available_values)
-                .enumerate()
-                .map(|(index, suggestion)| {
-                    // Correcting the enumerate index based on the number of skipped values
-                    let index = index + skip_values;
-                    let column = index as u16 % self.get_cols();
-                    let empty_space = self.get_width().saturating_sub(suggestion.value.len());
-
-                    self.create_entry_string(
-                        suggestion,
-                        index,
-                        column,
-                        empty_space,
-                        use_ansi_coloring,
-                    )
-                })
-                .collect();
-
-            format!(
-                "{}{}{}",
-                selection_values,
-                self.create_description_string(use_ansi_coloring),
-                self.create_example_string(use_ansi_coloring)
-            )
-        }
-    }
-}

crates/nu-cli/src/menus/help_completions.rs

@@ -102,10 +102,11 @@ impl NuHelpCompleter {
                 Suggestion {
                     value: sig.name.clone(),
                     description: Some(long_desc),
+                    style: None,
                     extra: Some(extra),
                     span: reedline::Span {
-                        start: pos,
-                        end: pos + line.len(),
+                        start: pos - line.len(),
+                        end: pos,
                     },
                     append_whitespace: false,
                 }
@@ -119,3 +120,42 @@ impl Completer for NuHelpCompleter {
         self.completion_helper(line, pos)
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use rstest::rstest;
+
+    #[rstest]
+    #[case("who", 5, 8, &["whoami"])]
+    #[case("hash", 1, 5, &["hash", "hash md5", "hash sha256"])]
+    #[case("into f", 0, 6, &["into float", "into filesize"])]
+    #[case("into nonexistent", 0, 16, &[])]
+    fn test_help_completer(
+        #[case] line: &str,
+        #[case] start: usize,
+        #[case] end: usize,
+        #[case] expected: &[&str],
+    ) {
+        let engine_state =
+            nu_command::add_shell_command_context(nu_cmd_lang::create_default_context());
+        let mut completer = NuHelpCompleter::new(engine_state.into());
+        let suggestions = completer.complete(line, end);
+
+        assert_eq!(
+            expected.len(),
+            suggestions.len(),
+            "expected {:?}, got {:?}",
+            expected,
+            suggestions
+                .iter()
+                .map(|s| s.value.clone())
+                .collect::<Vec<_>>()
+        );
+
+        for (exp, actual) in expected.iter().zip(suggestions) {
+            assert_eq!(exp, &actual.value);
+            assert_eq!(reedline::Span::new(start, end), actual.span);
+        }
+    }
+}

crates/nu-cli/src/menus/menu_completions.rs

@@ -101,9 +101,13 @@ fn convert_to_suggestions(
                             }
                         }
                         _ => reedline::Span {
-                            start: if only_buffer_difference { pos } else { 0 },
+                            start: if only_buffer_difference {
+                                pos - line.len()
+                            } else {
+                                0
+                            },
                             end: if only_buffer_difference {
-                                pos + line.len()
+                                pos
                             } else {
                                 line.len()
                             },
@@ -111,9 +115,13 @@ fn convert_to_suggestions(
                     }
                 }
                 _ => reedline::Span {
-                    start: if only_buffer_difference { pos } else { 0 },
+                    start: if only_buffer_difference {
+                        pos - line.len()
+                    } else {
+                        0
+                    },
                     end: if only_buffer_difference {
-                        pos + line.len()
+                        pos
                     } else {
                         line.len()
                     },
@@ -138,6 +146,7 @@ fn convert_to_suggestions(
             vec![Suggestion {
                 value: text,
                 description,
+                style: None,
                 extra,
                 span,
                 append_whitespace: false,
@@ -150,10 +159,19 @@ fn convert_to_suggestions(
         _ => vec![Suggestion {
             value: format!("Not a record: {value:?}"),
             description: None,
+            style: None,
             extra: None,
             span: reedline::Span {
-                start: 0,
-                end: line.len(),
+                start: if only_buffer_difference {
+                    pos - line.len()
+                } else {
+                    0
+                },
+                end: if only_buffer_difference {
+                    pos
+                } else {
+                    line.len()
+                },
             },
             append_whitespace: false,
         }],

crates/nu-cli/src/menus/mod.rs

@@ -1,7 +1,5 @@
-mod description_menu;
 mod help_completions;
 mod menu_completions;
 
-pub use description_menu::DescriptionMenu;
 pub use help_completions::NuHelpCompleter;
 pub use menu_completions::NuMenuCompleter;

crates/nu-cli/src/nu_highlight.rs

@@ -28,7 +28,7 @@ impl Command for NuHighlight {
     fn run(
         &self,
         engine_state: &EngineState,
-        _stack: &mut Stack,
+        stack: &mut Stack,
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
@@ -40,6 +40,7 @@ impl Command for NuHighlight {
 
         let highlighter = crate::NuHighlighter {
             engine_state,
+            stack: std::sync::Arc::new(stack.clone()),
             config,
         };
 

crates/nu-cli/src/print.rs

@@ -54,8 +54,8 @@ Since this command has no output, there is no point in piping it with other comm
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let args: Vec<Value> = call.rest(engine_state, stack, 0)?;
-        let no_newline = call.has_flag("no-newline");
-        let to_stderr = call.has_flag("stderr");
+        let no_newline = call.has_flag(engine_state, stack, "no-newline")?;
+        let to_stderr = call.has_flag(engine_state, stack, "stderr")?;
 
         // This will allow for easy printing of pipelines as well
         if !args.is_empty() {

crates/nu-cli/src/reedline_config.rs

@@ -1,4 +1,3 @@
-use super::DescriptionMenu;
 use crate::{menus::NuMenuCompleter, NuHelpCompleter};
 use crossterm::event::{KeyCode, KeyModifiers};
 use nu_color_config::{color_record_to_nustyle, lookup_ansi_color_style};
@@ -12,7 +11,8 @@ use nu_protocol::{
 };
 use reedline::{
     default_emacs_keybindings, default_vi_insert_keybindings, default_vi_normal_keybindings,
-    ColumnarMenu, EditCommand, Keybindings, ListMenu, Reedline, ReedlineEvent, ReedlineMenu,
+    ColumnarMenu, DescriptionMenu, DescriptionMode, EditCommand, IdeMenu, Keybindings, ListMenu,
+    MenuBuilder, Reedline, ReedlineEvent, ReedlineMenu,
 };
 use std::sync::Arc;
 
@@ -138,9 +138,10 @@ fn add_menu(
         match layout.as_str() {
             "columnar" => add_columnar_menu(line_editor, menu, engine_state, stack, config),
             "list" => add_list_menu(line_editor, menu, engine_state, stack, config),
+            "ide" => add_ide_menu(line_editor, menu, engine_state, stack, config),
             "description" => add_description_menu(line_editor, menu, engine_state, stack, config),
             _ => Err(ShellError::UnsupportedConfigValue {
-                expected: "columnar, list or description".to_string(),
+                expected: "columnar, list, ide or description".to_string(),
                 value: menu.menu_type.into_abbreviated_string(config),
                 span: menu.menu_type.span(),
             }),
@@ -235,10 +236,26 @@ pub(crate) fn add_columnar_menu(
             columnar_menu,
             ColumnarMenu::with_description_text_style
         );
+        add_style!(
+            "match_text",
+            val,
+            span,
+            config,
+            columnar_menu,
+            ColumnarMenu::with_match_text_style
+        );
+        add_style!(
+            "selected_match_text",
+            val,
+            span,
+            config,
+            columnar_menu,
+            ColumnarMenu::with_selected_match_text_style
+        );
     }
 
     let marker = menu.marker.into_string("", config);
-    columnar_menu = columnar_menu.with_marker(marker);
+    columnar_menu = columnar_menu.with_marker(&marker);
 
     let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
     columnar_menu = columnar_menu.with_only_buffer_difference(only_buffer_difference);
@@ -320,7 +337,7 @@ pub(crate) fn add_list_menu(
     }
 
     let marker = menu.marker.into_string("", config);
-    list_menu = list_menu.with_marker(marker);
+    list_menu = list_menu.with_marker(&marker);
 
     let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
     list_menu = list_menu.with_only_buffer_difference(only_buffer_difference);
@@ -351,6 +368,232 @@ pub(crate) fn add_list_menu(
     }
 }
 
+// Adds an IDE menu to the line editor
+pub(crate) fn add_ide_menu(
+    line_editor: Reedline,
+    menu: &ParsedMenu,
+    engine_state: Arc<EngineState>,
+    stack: &Stack,
+    config: &Config,
+) -> Result<Reedline, ShellError> {
+    let span = menu.menu_type.span();
+    let name = menu.name.into_string("", config);
+    let mut ide_menu = IdeMenu::default().with_name(&name);
+
+    if let Value::Record { val, .. } = &menu.menu_type {
+        ide_menu = match extract_value("min_completion_width", val, span) {
+            Ok(min_completion_width) => {
+                let min_completion_width = min_completion_width.as_int()?;
+                ide_menu.with_min_completion_width(min_completion_width as u16)
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("max_completion_width", val, span) {
+            Ok(max_completion_width) => {
+                let max_completion_width = max_completion_width.as_int()?;
+                ide_menu.with_max_completion_width(max_completion_width as u16)
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("max_completion_height", val, span) {
+            Ok(max_completion_height) => {
+                let max_completion_height = max_completion_height.as_int()?;
+                ide_menu.with_max_completion_height(max_completion_height as u16)
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("padding", val, span) {
+            Ok(padding) => {
+                let padding = padding.as_int()?;
+                ide_menu.with_padding(padding as u16)
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("border", val, span) {
+            Ok(border) => {
+                if let Ok(border) = border.as_bool() {
+                    if border {
+                        ide_menu.with_default_border()
+                    } else {
+                        ide_menu
+                    }
+                } else if let Ok(border_chars) = border.as_record() {
+                    let top_right = extract_value("top_right", border_chars, span)?.as_char()?;
+                    let top_left = extract_value("top_left", border_chars, span)?.as_char()?;
+                    let bottom_right =
+                        extract_value("bottom_right", border_chars, span)?.as_char()?;
+                    let bottom_left =
+                        extract_value("bottom_left", border_chars, span)?.as_char()?;
+                    let horizontal = extract_value("horizontal", border_chars, span)?.as_char()?;
+                    let vertical = extract_value("vertical", border_chars, span)?.as_char()?;
+
+                    ide_menu.with_border(
+                        top_right,
+                        top_left,
+                        bottom_right,
+                        bottom_left,
+                        horizontal,
+                        vertical,
+                    )
+                } else {
+                    return Err(ShellError::UnsupportedConfigValue {
+                        expected: "bool or record".to_string(),
+                        value: border.into_abbreviated_string(config),
+                        span: border.span(),
+                    });
+                }
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("cursor_offset", val, span) {
+            Ok(cursor_offset) => {
+                let cursor_offset = cursor_offset.as_int()?;
+                ide_menu.with_cursor_offset(cursor_offset as i16)
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("description_mode", val, span) {
+            Ok(description_mode) => {
+                let description_mode_str = description_mode.as_string()?;
+                match description_mode_str.as_str() {
+                    "left" => ide_menu.with_description_mode(DescriptionMode::Left),
+                    "right" => ide_menu.with_description_mode(DescriptionMode::Right),
+                    "prefer_right" => ide_menu.with_description_mode(DescriptionMode::PreferRight),
+                    _ => {
+                        return Err(ShellError::UnsupportedConfigValue {
+                            expected: "\"left\", \"right\" or \"prefer_right\"".to_string(),
+                            value: description_mode.into_abbreviated_string(config),
+                            span: description_mode.span(),
+                        });
+                    }
+                }
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("min_description_width", val, span) {
+            Ok(min_description_width) => {
+                let min_description_width = min_description_width.as_int()?;
+                ide_menu.with_min_description_width(min_description_width as u16)
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("max_description_width", val, span) {
+            Ok(max_description_width) => {
+                let max_description_width = max_description_width.as_int()?;
+                ide_menu.with_max_description_width(max_description_width as u16)
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("max_description_height", val, span) {
+            Ok(max_description_height) => {
+                let max_description_height = max_description_height.as_int()?;
+                ide_menu.with_max_description_height(max_description_height as u16)
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("description_offset", val, span) {
+            Ok(description_padding) => {
+                let description_padding = description_padding.as_int()?;
+                ide_menu.with_description_offset(description_padding as u16)
+            }
+            Err(_) => ide_menu,
+        };
+
+        ide_menu = match extract_value("correct_cursor_pos", val, span) {
+            Ok(correct_cursor_pos) => {
+                let correct_cursor_pos = correct_cursor_pos.as_bool()?;
+                ide_menu.with_correct_cursor_pos(correct_cursor_pos)
+            }
+            Err(_) => ide_menu,
+        };
+    }
+
+    let span = menu.style.span();
+    if let Value::Record { val, .. } = &menu.style {
+        add_style!(
+            "text",
+            val,
+            span,
+            config,
+            ide_menu,
+            IdeMenu::with_text_style
+        );
+        add_style!(
+            "selected_text",
+            val,
+            span,
+            config,
+            ide_menu,
+            IdeMenu::with_selected_text_style
+        );
+        add_style!(
+            "description_text",
+            val,
+            span,
+            config,
+            ide_menu,
+            IdeMenu::with_description_text_style
+        );
+        add_style!(
+            "match_text",
+            val,
+            span,
+            config,
+            ide_menu,
+            IdeMenu::with_match_text_style
+        );
+        add_style!(
+            "selected_match_text",
+            val,
+            span,
+            config,
+            ide_menu,
+            IdeMenu::with_selected_match_text_style
+        );
+    }
+
+    let marker = menu.marker.into_string("", config);
+    ide_menu = ide_menu.with_marker(&marker);
+
+    let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
+    ide_menu = ide_menu.with_only_buffer_difference(only_buffer_difference);
+
+    let span = menu.source.span();
+    match &menu.source {
+        Value::Nothing { .. } => {
+            Ok(line_editor.with_menu(ReedlineMenu::EngineCompleter(Box::new(ide_menu))))
+        }
+        Value::Closure { val, .. } => {
+            let menu_completer = NuMenuCompleter::new(
+                val.block_id,
+                span,
+                stack.captures_to_stack(val.captures.clone()),
+                engine_state,
+                only_buffer_difference,
+            );
+            Ok(line_editor.with_menu(ReedlineMenu::WithCompleter {
+                menu: Box::new(ide_menu),
+                completer: Box::new(menu_completer),
+            }))
+        }
+        _ => Err(ShellError::UnsupportedConfigValue {
+            expected: "block or omitted value".to_string(),
+            value: menu.source.into_abbreviated_string(config),
+            span,
+        }),
+    }
+}
+
 // Adds a description menu to the line editor
 pub(crate) fn add_description_menu(
     line_editor: Reedline,
@@ -434,7 +677,7 @@ pub(crate) fn add_description_menu(
     }
 
     let marker = menu.marker.into_string("", config);
-    description_menu = description_menu.with_marker(marker);
+    description_menu = description_menu.with_marker(&marker);
 
     let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
     description_menu = description_menu.with_only_buffer_difference(only_buffer_difference);
@@ -861,22 +1104,82 @@ fn edit_from_record(
     span: Span,
 ) -> Result<EditCommand, ShellError> {
     let edit = match name {
-        "movetostart" => EditCommand::MoveToStart,
-        "movetolinestart" => EditCommand::MoveToLineStart,
-        "movetoend" => EditCommand::MoveToEnd,
-        "movetolineend" => EditCommand::MoveToLineEnd,
-        "moveleft" => EditCommand::MoveLeft,
-        "moveright" => EditCommand::MoveRight,
-        "movewordleft" => EditCommand::MoveWordLeft,
-        "movebigwordleft" => EditCommand::MoveBigWordLeft,
-        "movewordright" => EditCommand::MoveWordRight,
-        "movewordrightend" => EditCommand::MoveWordRightEnd,
-        "movebigwordrightend" => EditCommand::MoveBigWordRightEnd,
-        "movewordrightstart" => EditCommand::MoveWordRightStart,
-        "movebigwordrightstart" => EditCommand::MoveBigWordRightStart,
+        "movetostart" => EditCommand::MoveToStart {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "movetolinestart" => EditCommand::MoveToLineStart {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+
+        "movetoend" => EditCommand::MoveToEnd {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "movetolineend" => EditCommand::MoveToLineEnd {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "moveleft" => EditCommand::MoveLeft {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "moveright" => EditCommand::MoveRight {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "movewordleft" => EditCommand::MoveWordLeft {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "movebigwordleft" => EditCommand::MoveBigWordLeft {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "movewordright" => EditCommand::MoveWordRight {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "movewordrightend" => EditCommand::MoveWordRightEnd {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "movebigwordrightend" => EditCommand::MoveBigWordRightEnd {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "movewordrightstart" => EditCommand::MoveWordRightStart {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
+        "movebigwordrightstart" => EditCommand::MoveBigWordRightStart {
+            select: extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false),
+        },
         "movetoposition" => {
             let value = extract_value("value", record, span)?;
-            EditCommand::MoveToPosition(value.as_int()? as usize)
+            let select = extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false);
+
+            EditCommand::MoveToPosition {
+                position: value.as_int()? as usize,
+                select,
+            }
         }
         "insertchar" => {
             let value = extract_value("value", record, span)?;
@@ -928,12 +1231,18 @@ fn edit_from_record(
         "moverightuntil" => {
             let value = extract_value("value", record, span)?;
             let char = extract_char(value, config)?;
-            EditCommand::MoveRightUntil(char)
+            let select = extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false);
+            EditCommand::MoveRightUntil { c: char, select }
         }
         "moverightbefore" => {
             let value = extract_value("value", record, span)?;
             let char = extract_char(value, config)?;
-            EditCommand::MoveRightBefore(char)
+            let select = extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false);
+            EditCommand::MoveRightBefore { c: char, select }
         }
         "cutleftuntil" => {
             let value = extract_value("value", record, span)?;
@@ -948,14 +1257,23 @@ fn edit_from_record(
         "moveleftuntil" => {
             let value = extract_value("value", record, span)?;
             let char = extract_char(value, config)?;
-            EditCommand::MoveLeftUntil(char)
+            let select = extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false);
+            EditCommand::MoveLeftUntil { c: char, select }
         }
         "moveleftbefore" => {
             let value = extract_value("value", record, span)?;
             let char = extract_char(value, config)?;
-            EditCommand::MoveLeftBefore(char)
+            let select = extract_value("select", record, span)
+                .and_then(|value| value.as_bool())
+                .unwrap_or(false);
+            EditCommand::MoveLeftBefore { c: char, select }
         }
         "complete" => EditCommand::Complete,
+        "cutselection" => EditCommand::CutSelection,
+        "copyselection" => EditCommand::CopySelection,
+        "selectall" => EditCommand::SelectAll,
         e => {
             return Err(ShellError::UnsupportedConfigValue {
                 expected: "reedline EditCommand".to_string(),
@@ -1109,4 +1427,57 @@ mod test {
         let b = EventType::try_from_record(&event, span);
         assert!(matches!(b, Err(ShellError::MissingConfigValue { .. })));
     }
+
+    #[test]
+    fn test_move_without_optional_select() {
+        let event = record! {
+            "edit" => Value::test_string("moveleft")
+        };
+        let event = Value::test_record(event);
+        let config = Config::default();
+
+        let parsed_event = parse_event(&event, &config).unwrap();
+        assert_eq!(
+            parsed_event,
+            Some(ReedlineEvent::Edit(vec![EditCommand::MoveLeft {
+                select: false
+            }]))
+        );
+    }
+
+    #[test]
+    fn test_move_with_select_false() {
+        let event = record! {
+            "edit" => Value::test_string("moveleft"),
+            "select" => Value::test_bool(false)
+        };
+        let event = Value::test_record(event);
+        let config = Config::default();
+
+        let parsed_event = parse_event(&event, &config).unwrap();
+        assert_eq!(
+            parsed_event,
+            Some(ReedlineEvent::Edit(vec![EditCommand::MoveLeft {
+                select: false
+            }]))
+        );
+    }
+
+    #[test]
+    fn test_move_with_select_true() {
+        let event = record! {
+            "edit" => Value::test_string("moveleft"),
+            "select" => Value::test_bool(true)
+        };
+        let event = Value::test_record(event);
+        let config = Config::default();
+
+        let parsed_event = parse_event(&event, &config).unwrap();
+        assert_eq!(
+            parsed_event,
+            Some(ReedlineEvent::Edit(vec![EditCommand::MoveLeft {
+                select: true
+            }]))
+        );
+    }
 }

crates/nu-cli/src/repl.rs

@@ -11,13 +11,13 @@ use miette::{ErrReport, IntoDiagnostic, Result};
 use nu_cmd_base::util::get_guaranteed_cwd;
 use nu_cmd_base::{hook::eval_hook, util::get_editor};
 use nu_color_config::StyleComputer;
-use nu_engine::convert_env_values;
+use nu_engine::{convert_env_values, env_to_strings};
 use nu_parser::{lex, parse, trim_quotes_str};
 use nu_protocol::{
     config::NuCursorShape,
     engine::{EngineState, Stack, StateWorkingSet},
     eval_const::create_nu_constant,
-    report_error, report_error_new, HistoryFileFormat, PipelineData, ShellError, Span, Spanned,
+    report_error_new, HistoryConfig, HistoryFileFormat, PipelineData, ShellError, Span, Spanned,
     Value, NU_VARIABLE_ID,
 };
 use nu_utils::utils::perf;
@@ -28,9 +28,9 @@ use reedline::{
 use std::{
     env::temp_dir,
     io::{self, IsTerminal, Write},
-    path::Path,
+    path::PathBuf,
     sync::atomic::Ordering,
-    time::Instant,
+    time::{Duration, Instant},
 };
 use sysinfo::System;
 
@@ -44,6 +44,9 @@ const PRE_EXECUTE_MARKER: &str = "\x1b]133;C\x1b\\";
 // const CMD_FINISHED_MARKER: &str = "\x1b]133;D;{}\x1b\\";
 const RESET_APPLICATION_MODE: &str = "\x1b[?1l";
 
+///
+/// The main REPL loop, including spinning up the prompt itself.
+///
 pub fn evaluate_repl(
     engine_state: &mut EngineState,
     stack: &mut Stack,
@@ -57,15 +60,7 @@ pub fn evaluate_repl(
     let config = engine_state.get_config();
     let use_color = config.use_ansi_coloring;
 
-    // Guard against invocation without a connected terminal.
-    // reedline / crossterm event polling will fail without a connected tty
-    if !std::io::stdin().is_terminal() {
-        return Err(std::io::Error::new(
-            std::io::ErrorKind::NotFound,
-            "Nushell launched as a REPL, but STDIN is not a TTY; either launch in a valid terminal or provide arguments to invoke a script!",
-        ))
-        .into_diagnostic();
-    }
+    confirm_stdin_is_terminal()?;
 
     let mut entry_num = 0;
 
@@ -74,8 +69,7 @@ pub fn evaluate_repl(
     let start_time = std::time::Instant::now();
     // Translate environment variables from Strings to Values
     if let Some(e) = convert_env_values(engine_state, stack) {
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(&working_set, &e);
+        report_error_new(engine_state, &e);
     }
     perf(
         "translate env vars",
@@ -109,31 +103,20 @@ pub fn evaluate_repl(
         use_color,
     );
 
-    // Setup history_isolation aka "history per session"
-    let history_isolation = engine_state.get_config().history_isolation;
-    let history_session_id = if history_isolation {
-        Reedline::create_history_session_id()
-    } else {
-        None
-    };
+    if let Some(history) = engine_state.history_config() {
+        start_time = std::time::Instant::now();
 
-    start_time = std::time::Instant::now();
-    let history_path = crate::config_files::get_history_path(
-        nushell_path,
-        engine_state.config.history_file_format,
-    );
-    if let Some(history_path) = history_path.as_deref() {
-        line_editor =
-            update_line_editor_history(engine_state, history_path, line_editor, history_session_id)?
-    };
-    perf(
-        "setup history",
-        start_time,
-        file!(),
-        line!(),
-        column!(),
-        use_color,
-    );
+        line_editor = setup_history(nushell_path, engine_state, line_editor, history)?;
+
+        perf(
+            "setup history",
+            start_time,
+            file!(),
+            line!(),
+            column!(),
+            use_color,
+        );
+    }
 
     if let Some(s) = prerun_command {
         eval_source(
@@ -164,9 +147,7 @@ pub fn evaluate_repl(
         );
     }
 
-    if engine_state.get_config().use_kitty_protocol && !reedline::kitty_protocol_available() {
-        warn!("Terminal doesn't support use_kitty_protocol config");
-    }
+    kitty_protocol_healthcheck(engine_state);
 
     loop {
         let loop_start_time = std::time::Instant::now();
@@ -231,6 +212,7 @@ pub fn evaluate_repl(
             .use_bracketed_paste(cfg!(not(target_os = "windows")) && config.bracketed_paste)
             .with_highlighter(Box::new(NuHighlighter {
                 engine_state: engine_reference.clone(),
+                stack: std::sync::Arc::new(stack.clone()),
                 config: config.clone(),
             }))
             .with_validator(Box::new(NuValidator {
@@ -276,8 +258,7 @@ pub fn evaluate_repl(
 
         start_time = std::time::Instant::now();
         line_editor = add_menus(line_editor, engine_reference, stack, config).unwrap_or_else(|e| {
-            let working_set = StateWorkingSet::new(engine_state);
-            report_error(&working_set, &e);
+            report_error_new(engine_state, &e);
             Reedline::create()
         });
         perf(
@@ -294,12 +275,9 @@ pub fn evaluate_repl(
 
         line_editor = if let Ok((cmd, args)) = buffer_editor {
             let mut command = std::process::Command::new(&cmd);
-            command.args(args).envs(
-                engine_state
-                    .render_env_vars()
-                    .into_iter()
-                    .filter_map(|(k, v)| v.as_string().ok().map(|v| (k, v))),
-            );
+            command
+                .args(args)
+                .envs(env_to_strings(engine_state, stack)?);
             line_editor.with_buffer_editor(command, temp_file.clone())
         } else {
             line_editor
@@ -313,43 +291,26 @@ pub fn evaluate_repl(
             use_color,
         );
 
-        start_time = std::time::Instant::now();
-        if config.sync_history_on_enter {
-            if let Err(e) = line_editor.sync_history() {
-                warn!("Failed to sync history: {}", e);
+        if let Some(history) = engine_state.history_config() {
+            start_time = std::time::Instant::now();
+            if history.sync_on_enter {
+                if let Err(e) = line_editor.sync_history() {
+                    warn!("Failed to sync history: {}", e);
+                }
             }
+            perf(
+                "sync_history",
+                start_time,
+                file!(),
+                line!(),
+                column!(),
+                use_color,
+            );
         }
-        perf(
-            "sync_history",
-            start_time,
-            file!(),
-            line!(),
-            column!(),
-            use_color,
-        );
 
         start_time = std::time::Instant::now();
         // Changing the line editor based on the found keybindings
-        line_editor = match create_keybindings(config) {
-            Ok(keybindings) => match keybindings {
-                KeybindingsMode::Emacs(keybindings) => {
-                    let edit_mode = Box::new(Emacs::new(keybindings));
-                    line_editor.with_edit_mode(edit_mode)
-                }
-                KeybindingsMode::Vi {
-                    insert_keybindings,
-                    normal_keybindings,
-                } => {
-                    let edit_mode = Box::new(Vi::new(insert_keybindings, normal_keybindings));
-                    line_editor.with_edit_mode(edit_mode)
-                }
-            },
-            Err(e) => {
-                let working_set = StateWorkingSet::new(engine_state);
-                report_error(&working_set, &e);
-                line_editor
-            }
-        };
+        line_editor = setup_keybindings(engine_state, line_editor);
         perf(
             "keybindings",
             start_time,
@@ -418,19 +379,13 @@ pub fn evaluate_repl(
         match input {
             Ok(Signal::Success(s)) => {
                 let hostname = System::host_name();
-                let history_supports_meta =
-                    matches!(config.history_file_format, HistoryFileFormat::Sqlite);
-                if history_supports_meta && !s.is_empty() && line_editor.has_last_command_context()
-                {
-                    line_editor
-                        .update_last_command_context(&|mut c| {
-                            c.start_timestamp = Some(chrono::Utc::now());
-                            c.hostname = hostname.clone();
+                let history_supports_meta = matches!(
+                    engine_state.history_config().map(|h| h.file_format),
+                    Some(HistoryFileFormat::Sqlite)
+                );
 
-                            c.cwd = Some(StateWorkingSet::new(engine_state).get_cwd());
-                            c
-                        })
-                        .into_diagnostic()?; // todo: don't stop repl if error here?
+                if history_supports_meta {
+                    prepare_history_metadata(&s, &hostname, engine_state, &mut line_editor)?;
                 }
 
                 // Right before we start running the code the user gave us, fire the `pre_execution`
@@ -457,107 +412,27 @@ pub fn evaluate_repl(
                     run_ansi_sequence(PRE_EXECUTE_MARKER)?;
                 }
 
+                // Actual command execution logic starts from here
                 let start_time = Instant::now();
-                let tokens = lex(s.as_bytes(), 0, &[], &[], false);
-                // Check if this is a single call to a directory, if so auto-cd
-                let cwd = nu_engine::env::current_dir_str(engine_state, stack)?;
 
-                let mut orig = s.clone();
-                if orig.starts_with('`') {
-                    orig = trim_quotes_str(&orig).to_string()
-                }
-
-                let path = nu_path::expand_path_with(&orig, &cwd);
-
-                if looks_like_path(&orig) && path.is_dir() && tokens.0.len() == 1 {
-                    // We have an auto-cd
-                    let (path, span) = {
-                        if !path.exists() {
-                            let working_set = StateWorkingSet::new(engine_state);
-
-                            report_error(
-                                &working_set,
-                                &ShellError::DirectoryNotFound {
-                                    dir: path.to_string_lossy().to_string(),
-                                    span: tokens.0[0].span,
-                                },
-                            );
-                        }
-                        let path = nu_path::canonicalize_with(path, &cwd)
-                            .expect("internal error: cannot canonicalize known path");
-                        (path.to_string_lossy().to_string(), tokens.0[0].span)
-                    };
-
-                    stack.add_env_var("OLDPWD".into(), Value::string(cwd.clone(), Span::unknown()));
-
-                    //FIXME: this only changes the current scope, but instead this environment variable
-                    //should probably be a block that loads the information from the state in the overlay
-                    stack.add_env_var("PWD".into(), Value::string(path.clone(), Span::unknown()));
-                    let cwd = Value::string(cwd, span);
-
-                    let shells = stack.get_env_var(engine_state, "NUSHELL_SHELLS");
-                    let mut shells = if let Some(v) = shells {
-                        v.as_list()
-                            .map(|x| x.to_vec())
-                            .unwrap_or_else(|_| vec![cwd])
-                    } else {
-                        vec![cwd]
-                    };
-
-                    let current_shell = stack.get_env_var(engine_state, "NUSHELL_CURRENT_SHELL");
-                    let current_shell = if let Some(v) = current_shell {
-                        v.as_int().unwrap_or_default() as usize
-                    } else {
-                        0
-                    };
-
-                    let last_shell = stack.get_env_var(engine_state, "NUSHELL_LAST_SHELL");
-                    let last_shell = if let Some(v) = last_shell {
-                        v.as_int().unwrap_or_default() as usize
-                    } else {
-                        0
-                    };
-
-                    shells[current_shell] = Value::string(path, span);
-
-                    stack.add_env_var("NUSHELL_SHELLS".into(), Value::list(shells, span));
-                    stack.add_env_var(
-                        "NUSHELL_LAST_SHELL".into(),
-                        Value::int(last_shell as i64, span),
-                    );
-                } else if !s.trim().is_empty() {
-                    trace!("eval source: {}", s);
-
-                    let mut cmds = s.split_whitespace();
-                    if let Some("exit") = cmds.next() {
-                        let mut working_set = StateWorkingSet::new(engine_state);
-                        let _ = parse(&mut working_set, None, s.as_bytes(), false);
-
-                        if working_set.parse_errors.is_empty() {
-                            match cmds.next() {
-                                Some(s) => {
-                                    if let Ok(n) = s.parse::<i32>() {
-                                        drop(line_editor);
-                                        std::process::exit(n);
-                                    }
-                                }
-                                None => {
-                                    drop(line_editor);
-                                    std::process::exit(0);
-                                }
-                            }
-                        }
+                match parse_operation(s.clone(), engine_state, stack)? {
+                    ReplOperation::AutoCd { cwd, target, span } => {
+                        do_auto_cd(target, cwd, stack, engine_state, span);
                     }
-
-                    eval_source(
-                        engine_state,
-                        stack,
-                        s.as_bytes(),
-                        &format!("entry #{entry_num}"),
-                        PipelineData::empty(),
-                        false,
-                    );
+                    ReplOperation::RunCommand(cmd) => {
+                        line_editor = do_run_cmd(
+                            &cmd,
+                            stack,
+                            engine_state,
+                            line_editor,
+                            shell_integration,
+                            entry_num,
+                        )?;
+                    }
+                    // as the name implies, we do nothing in this case
+                    ReplOperation::DoNothing => {}
                 }
+
                 let cmd_duration = start_time.elapsed();
 
                 stack.add_env_var(
@@ -565,74 +440,21 @@ pub fn evaluate_repl(
                     Value::string(format!("{}", cmd_duration.as_millis()), Span::unknown()),
                 );
 
-                if history_supports_meta && !s.is_empty() && line_editor.has_last_command_context()
-                {
-                    line_editor
-                        .update_last_command_context(&|mut c| {
-                            c.duration = Some(cmd_duration);
-                            c.exit_status = stack
-                                .get_env_var(engine_state, "LAST_EXIT_CODE")
-                                .and_then(|e| e.as_i64().ok());
-                            c
-                        })
-                        .into_diagnostic()?; // todo: don't stop repl if error here?
+                if history_supports_meta {
+                    fill_in_result_related_history_metadata(
+                        &s,
+                        engine_state,
+                        cmd_duration,
+                        stack,
+                        &mut line_editor,
+                    )?;
                 }
 
                 if shell_integration {
-                    run_ansi_sequence(&get_command_finished_marker(stack, engine_state))?;
-                    if let Some(cwd) = stack.get_env_var(engine_state, "PWD") {
-                        let path = cwd.as_string()?;
-
-                        // Supported escape sequences of Microsoft's Visual Studio Code (vscode)
-                        // https://code.visualstudio.com/docs/terminal/shell-integration#_supported-escape-sequences
-                        if stack.get_env_var(engine_state, "TERM_PROGRAM")
-                            == Some(Value::test_string("vscode"))
-                        {
-                            // If we're in vscode, run their specific ansi escape sequence.
-                            // This is helpful for ctrl+g to change directories in the terminal.
-                            run_ansi_sequence(&format!("\x1b]633;P;Cwd={}\x1b\\", path))?;
-                        } else {
-                            // Otherwise, communicate the path as OSC 7 (often used for spawning new tabs in the same dir)
-                            run_ansi_sequence(&format!(
-                                "\x1b]7;file://{}{}{}\x1b\\",
-                                percent_encoding::utf8_percent_encode(
-                                    &hostname.unwrap_or_else(|| "localhost".to_string()),
-                                    percent_encoding::CONTROLS
-                                ),
-                                if path.starts_with('/') { "" } else { "/" },
-                                percent_encoding::utf8_percent_encode(
-                                    &path,
-                                    percent_encoding::CONTROLS
-                                )
-                            ))?;
-                        }
-
-                        // Try to abbreviate string for windows title
-                        let maybe_abbrev_path = if let Some(p) = nu_path::home_dir() {
-                            path.replace(&p.as_path().display().to_string(), "~")
-                        } else {
-                            path
-                        };
-
-                        // Set window title too
-                        // https://tldp.org/HOWTO/Xterm-Title-3.html
-                        // ESC]0;stringBEL -- Set icon name and window title to string
-                        // ESC]1;stringBEL -- Set icon name to string
-                        // ESC]2;stringBEL -- Set window title to string
-                        run_ansi_sequence(&format!("\x1b]2;{maybe_abbrev_path}\x07"))?;
-                    }
-                    run_ansi_sequence(RESET_APPLICATION_MODE)?;
+                    do_shell_integration_finalize_command(hostname, engine_state, stack)?;
                 }
 
-                let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
-                line_editor.run_edit_commands(&[
-                    EditCommand::Clear,
-                    EditCommand::InsertString(repl.buffer.to_string()),
-                    EditCommand::MoveToPosition(repl.cursor_pos),
-                ]);
-                repl.buffer = "".to_string();
-                repl.cursor_pos = 0;
-                drop(repl);
+                flush_engine_state_repl_buffer(engine_state, &mut line_editor);
             }
             Ok(Signal::CtrlC) => {
                 // `Reedline` clears the line content. New prompt is shown
@@ -684,6 +506,364 @@ pub fn evaluate_repl(
     Ok(())
 }
 
+///
+/// Put in history metadata not related to the result of running the command
+///
+fn prepare_history_metadata(
+    s: &str,
+    hostname: &Option<String>,
+    engine_state: &EngineState,
+    line_editor: &mut Reedline,
+) -> Result<()> {
+    if !s.is_empty() && line_editor.has_last_command_context() {
+        line_editor
+            .update_last_command_context(&|mut c| {
+                c.start_timestamp = Some(chrono::Utc::now());
+                c.hostname = hostname.clone();
+
+                c.cwd = Some(StateWorkingSet::new(engine_state).get_cwd());
+                c
+            })
+            .into_diagnostic()?; // todo: don't stop repl if error here?
+    }
+    Ok(())
+}
+
+///
+/// Fills in history item metadata based on the execution result (notably duration and exit code)
+///
+fn fill_in_result_related_history_metadata(
+    s: &str,
+    engine_state: &EngineState,
+    cmd_duration: Duration,
+    stack: &mut Stack,
+    line_editor: &mut Reedline,
+) -> Result<()> {
+    if !s.is_empty() && line_editor.has_last_command_context() {
+        line_editor
+            .update_last_command_context(&|mut c| {
+                c.duration = Some(cmd_duration);
+                c.exit_status = stack
+                    .get_env_var(engine_state, "LAST_EXIT_CODE")
+                    .and_then(|e| e.as_i64().ok());
+                c
+            })
+            .into_diagnostic()?; // todo: don't stop repl if error here?
+    }
+    Ok(())
+}
+
+/// The kinds of operations you can do in a single loop iteration of the REPL
+enum ReplOperation {
+    /// "auto-cd": change directory by typing it in directly
+    AutoCd {
+        /// the current working directory
+        cwd: String,
+        /// the target
+        target: PathBuf,
+        /// span information for debugging
+        span: Span,
+    },
+    /// run a command
+    RunCommand(String),
+    /// do nothing (usually through an empty string)
+    DoNothing,
+}
+
+///
+/// Parses one "REPL line" of input, to try and derive intent.
+/// Notably, this is where we detect whether the user is attempting an
+/// "auto-cd" (writing a relative path directly instead of `cd path`)
+///
+/// Returns the ReplOperation we believe the user wants to do
+///
+fn parse_operation(
+    s: String,
+    engine_state: &EngineState,
+    stack: &Stack,
+) -> Result<ReplOperation, ErrReport> {
+    let tokens = lex(s.as_bytes(), 0, &[], &[], false);
+    // Check if this is a single call to a directory, if so auto-cd
+    let cwd = nu_engine::env::current_dir_str(engine_state, stack)?;
+    let mut orig = s.clone();
+    if orig.starts_with('`') {
+        orig = trim_quotes_str(&orig).to_string()
+    }
+
+    let path = nu_path::expand_path_with(&orig, &cwd);
+    if looks_like_path(&orig) && path.is_dir() && tokens.0.len() == 1 {
+        Ok(ReplOperation::AutoCd {
+            cwd,
+            target: path,
+            span: tokens.0[0].span,
+        })
+    } else if !s.trim().is_empty() {
+        Ok(ReplOperation::RunCommand(s))
+    } else {
+        Ok(ReplOperation::DoNothing)
+    }
+}
+
+///
+/// Execute an "auto-cd" operation, changing the current working directory.
+///
+fn do_auto_cd(
+    path: PathBuf,
+    cwd: String,
+    stack: &mut Stack,
+    engine_state: &mut EngineState,
+    span: Span,
+) {
+    let path = {
+        if !path.exists() {
+            report_error_new(
+                engine_state,
+                &ShellError::DirectoryNotFound {
+                    dir: path.to_string_lossy().to_string(),
+                    span,
+                },
+            );
+        }
+        let path = nu_path::canonicalize_with(path, &cwd)
+            .expect("internal error: cannot canonicalize known path");
+        path.to_string_lossy().to_string()
+    };
+
+    stack.add_env_var("OLDPWD".into(), Value::string(cwd.clone(), Span::unknown()));
+
+    //FIXME: this only changes the current scope, but instead this environment variable
+    //should probably be a block that loads the information from the state in the overlay
+    stack.add_env_var("PWD".into(), Value::string(path.clone(), Span::unknown()));
+    let cwd = Value::string(cwd, span);
+
+    let shells = stack.get_env_var(engine_state, "NUSHELL_SHELLS");
+    let mut shells = if let Some(v) = shells {
+        v.as_list()
+            .map(|x| x.to_vec())
+            .unwrap_or_else(|_| vec![cwd])
+    } else {
+        vec![cwd]
+    };
+
+    let current_shell = stack.get_env_var(engine_state, "NUSHELL_CURRENT_SHELL");
+    let current_shell = if let Some(v) = current_shell {
+        v.as_int().unwrap_or_default() as usize
+    } else {
+        0
+    };
+
+    let last_shell = stack.get_env_var(engine_state, "NUSHELL_LAST_SHELL");
+    let last_shell = if let Some(v) = last_shell {
+        v.as_int().unwrap_or_default() as usize
+    } else {
+        0
+    };
+
+    shells[current_shell] = Value::string(path, span);
+
+    stack.add_env_var("NUSHELL_SHELLS".into(), Value::list(shells, span));
+    stack.add_env_var(
+        "NUSHELL_LAST_SHELL".into(),
+        Value::int(last_shell as i64, span),
+    );
+}
+
+///
+/// Run a command as received from reedline. This is where we are actually
+/// running a thing!
+///
+fn do_run_cmd(
+    s: &str,
+    stack: &mut Stack,
+    engine_state: &mut EngineState,
+    // we pass in the line editor so it can be dropped in the case of a process exit
+    // (in the normal case we don't want to drop it so return it as-is otherwise)
+    line_editor: Reedline,
+    shell_integration: bool,
+    entry_num: usize,
+) -> Result<Reedline> {
+    trace!("eval source: {}", s);
+
+    let mut cmds = s.split_whitespace();
+    if let Some("exit") = cmds.next() {
+        let mut working_set = StateWorkingSet::new(engine_state);
+        let _ = parse(&mut working_set, None, s.as_bytes(), false);
+
+        if working_set.parse_errors.is_empty() {
+            match cmds.next() {
+                Some(s) => {
+                    if let Ok(n) = s.parse::<i32>() {
+                        drop(line_editor);
+                        std::process::exit(n);
+                    }
+                }
+                None => {
+                    drop(line_editor);
+                    std::process::exit(0);
+                }
+            }
+        }
+    }
+
+    if shell_integration {
+        if let Some(cwd) = stack.get_env_var(engine_state, "PWD") {
+            let path = cwd.as_string()?;
+
+            // Try to abbreviate string for windows title
+            let maybe_abbrev_path = if let Some(p) = nu_path::home_dir() {
+                path.replace(&p.as_path().display().to_string(), "~")
+            } else {
+                path
+            };
+            let binary_name = s.split_whitespace().next();
+
+            if let Some(binary_name) = binary_name {
+                run_ansi_sequence(&format!("\x1b]2;{maybe_abbrev_path}> {binary_name}\x07"))?;
+            }
+        }
+    }
+
+    eval_source(
+        engine_state,
+        stack,
+        s.as_bytes(),
+        &format!("entry #{entry_num}"),
+        PipelineData::empty(),
+        false,
+    );
+
+    Ok(line_editor)
+}
+
+///
+/// Output some things and set environment variables so shells with the right integration
+/// can have more information about what is going on (after we have run a command)
+///
+fn do_shell_integration_finalize_command(
+    hostname: Option<String>,
+    engine_state: &EngineState,
+    stack: &mut Stack,
+) -> Result<()> {
+    run_ansi_sequence(&get_command_finished_marker(stack, engine_state))?;
+    if let Some(cwd) = stack.get_env_var(engine_state, "PWD") {
+        let path = cwd.as_string()?;
+
+        // Supported escape sequences of Microsoft's Visual Studio Code (vscode)
+        // https://code.visualstudio.com/docs/terminal/shell-integration#_supported-escape-sequences
+        if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) {
+            // If we're in vscode, run their specific ansi escape sequence.
+            // This is helpful for ctrl+g to change directories in the terminal.
+            run_ansi_sequence(&format!("\x1b]633;P;Cwd={}\x1b\\", path))?;
+        } else {
+            // Otherwise, communicate the path as OSC 7 (often used for spawning new tabs in the same dir)
+            run_ansi_sequence(&format!(
+                "\x1b]7;file://{}{}{}\x1b\\",
+                percent_encoding::utf8_percent_encode(
+                    &hostname.unwrap_or_else(|| "localhost".to_string()),
+                    percent_encoding::CONTROLS
+                ),
+                if path.starts_with('/') { "" } else { "/" },
+                percent_encoding::utf8_percent_encode(&path, percent_encoding::CONTROLS)
+            ))?;
+        }
+
+        // Try to abbreviate string for windows title
+        let maybe_abbrev_path = if let Some(p) = nu_path::home_dir() {
+            path.replace(&p.as_path().display().to_string(), "~")
+        } else {
+            path
+        };
+
+        // Set window title too
+        // https://tldp.org/HOWTO/Xterm-Title-3.html
+        // ESC]0;stringBEL -- Set icon name and window title to string
+        // ESC]1;stringBEL -- Set icon name to string
+        // ESC]2;stringBEL -- Set window title to string
+        run_ansi_sequence(&format!("\x1b]2;{maybe_abbrev_path}\x07"))?;
+    }
+    run_ansi_sequence(RESET_APPLICATION_MODE)?;
+    Ok(())
+}
+
+///
+/// Clear the screen and output anything remaining in the EngineState buffer.
+///
+fn flush_engine_state_repl_buffer(engine_state: &mut EngineState, line_editor: &mut Reedline) {
+    let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
+    line_editor.run_edit_commands(&[
+        EditCommand::Clear,
+        EditCommand::InsertString(repl.buffer.to_string()),
+        EditCommand::MoveToPosition {
+            position: repl.cursor_pos,
+            select: false,
+        },
+    ]);
+    repl.buffer = "".to_string();
+    repl.cursor_pos = 0;
+}
+
+///
+/// Setup history management for Reedline
+///
+fn setup_history(
+    nushell_path: &str,
+    engine_state: &mut EngineState,
+    line_editor: Reedline,
+    history: HistoryConfig,
+) -> Result<Reedline> {
+    // Setup history_isolation aka "history per session"
+    let history_session_id = if history.isolation {
+        Reedline::create_history_session_id()
+    } else {
+        None
+    };
+
+    if let Some(path) = crate::config_files::get_history_path(nushell_path, history.file_format) {
+        return update_line_editor_history(
+            engine_state,
+            path,
+            history,
+            line_editor,
+            history_session_id,
+        );
+    };
+    Ok(line_editor)
+}
+
+///
+/// Setup Reedline keybindingds based on the provided config
+///
+fn setup_keybindings(engine_state: &EngineState, line_editor: Reedline) -> Reedline {
+    return match create_keybindings(engine_state.get_config()) {
+        Ok(keybindings) => match keybindings {
+            KeybindingsMode::Emacs(keybindings) => {
+                let edit_mode = Box::new(Emacs::new(keybindings));
+                line_editor.with_edit_mode(edit_mode)
+            }
+            KeybindingsMode::Vi {
+                insert_keybindings,
+                normal_keybindings,
+            } => {
+                let edit_mode = Box::new(Vi::new(insert_keybindings, normal_keybindings));
+                line_editor.with_edit_mode(edit_mode)
+            }
+        },
+        Err(e) => {
+            report_error_new(engine_state, &e);
+            line_editor
+        }
+    };
+}
+
+///
+/// Make sure that the terminal supports the kitty protocol if the config is asking for it
+///
+fn kitty_protocol_healthcheck(engine_state: &EngineState) {
+    if engine_state.get_config().use_kitty_protocol && !reedline::kitty_protocol_available() {
+        warn!("Terminal doesn't support use_kitty_protocol config");
+    }
+}
+
 fn store_history_id_in_engine(engine_state: &mut EngineState, line_editor: &Reedline) {
     let session_id = line_editor
         .get_history_session_id()
@@ -695,18 +875,15 @@ fn store_history_id_in_engine(engine_state: &mut EngineState, line_editor: &Reed
 
 fn update_line_editor_history(
     engine_state: &mut EngineState,
-    history_path: &Path,
+    history_path: PathBuf,
+    history: HistoryConfig,
     line_editor: Reedline,
     history_session_id: Option<HistorySessionId>,
 ) -> Result<Reedline, ErrReport> {
-    let config = engine_state.get_config();
-    let history: Box<dyn reedline::History> = match engine_state.config.history_file_format {
+    let history: Box<dyn reedline::History> = match history.file_format {
         HistoryFileFormat::PlainText => Box::new(
-            FileBackedHistory::with_file(
-                config.max_history_size as usize,
-                history_path.to_path_buf(),
-            )
-            .into_diagnostic()?,
+            FileBackedHistory::with_file(history.max_size as usize, history_path)
+                .into_diagnostic()?,
         ),
         HistoryFileFormat::Sqlite => Box::new(
             SqliteBackedHistory::with_file(
@@ -727,6 +904,18 @@ fn update_line_editor_history(
     Ok(line_editor)
 }
 
+fn confirm_stdin_is_terminal() -> Result<()> {
+    // Guard against invocation without a connected terminal.
+    // reedline / crossterm event polling will fail without a connected tty
+    if !std::io::stdin().is_terminal() {
+        return Err(std::io::Error::new(
+            std::io::ErrorKind::NotFound,
+            "Nushell launched as a REPL, but STDIN is not a TTY; either launch in a valid terminal or provide arguments to invoke a script!",
+        ))
+        .into_diagnostic();
+    }
+    Ok(())
+}
 fn map_nucursorshape_to_cursorshape(shape: NuCursorShape) -> Option<SetCursorStyle> {
     match shape {
         NuCursorShape::Block => Some(SetCursorStyle::SteadyBlock),
@@ -814,14 +1003,18 @@ fn trailing_slash_looks_like_path() {
 #[test]
 fn are_session_ids_in_sync() {
     let engine_state = &mut EngineState::new();
-    let history_path_o =
-        crate::config_files::get_history_path("nushell", engine_state.config.history_file_format);
-    assert!(history_path_o.is_some());
-    let history_path = history_path_o.as_deref().unwrap();
+    let history = engine_state.history_config().unwrap();
+    let history_path =
+        crate::config_files::get_history_path("nushell", history.file_format).unwrap();
     let line_editor = reedline::Reedline::create();
     let history_session_id = reedline::Reedline::create_history_session_id();
-    let line_editor =
-        update_line_editor_history(engine_state, history_path, line_editor, history_session_id);
+    let line_editor = update_line_editor_history(
+        engine_state,
+        history_path,
+        history,
+        line_editor,
+        history_session_id,
+    );
     assert_eq!(
         i64::from(line_editor.unwrap().get_history_session_id().unwrap()),
         engine_state.history_session_id

crates/nu-cli/src/syntax_highlight.rs

@@ -1,15 +1,17 @@
 use log::trace;
 use nu_ansi_term::Style;
 use nu_color_config::{get_matching_brackets_style, get_shape_color};
+use nu_engine::env;
 use nu_parser::{flatten_block, parse, FlatShape};
 use nu_protocol::ast::{Argument, Block, Expr, Expression, PipelineElement, RecordItem};
-use nu_protocol::engine::{EngineState, StateWorkingSet};
+use nu_protocol::engine::{EngineState, Stack, StateWorkingSet};
 use nu_protocol::{Config, Span};
 use reedline::{Highlighter, StyledText};
 use std::sync::Arc;
 
 pub struct NuHighlighter {
     pub engine_state: Arc<EngineState>,
+    pub stack: Arc<Stack>,
     pub config: Config,
 }
 
@@ -32,7 +34,17 @@ impl Highlighter for NuHighlighter {
                             working_set.get_span_contents(Span::new(span.start, span.end));
 
                         let str_word = String::from_utf8_lossy(str_contents).to_string();
-                        if which::which(str_word).ok().is_some() {
+                        let paths = env::path_str(&self.engine_state, &self.stack, *span).ok();
+                        let res = if let Ok(cwd) =
+                            env::current_dir_str(&self.engine_state, &self.stack)
+                        {
+                            which::which_in(str_word, paths.as_ref(), cwd).ok()
+                        } else {
+                            which::which_in_global(str_word, paths.as_ref())
+                                .ok()
+                                .and_then(|mut i| i.next())
+                        };
+                        if res.is_some() {
                             *shape = FlatShape::ExternalResolved;
                         }
                     }
@@ -321,9 +333,9 @@ fn find_matching_block_end_in_expr(
             Expr::Keyword(..) => None,
             Expr::ValueWithUnit(..) => None,
             Expr::DateTime(_) => None,
-            Expr::Filepath(_) => None,
-            Expr::Directory(_) => None,
-            Expr::GlobPattern(_) => None,
+            Expr::Filepath(_, _) => None,
+            Expr::Directory(_, _) => None,
+            Expr::GlobPattern(_, _) => None,
             Expr::String(_) => None,
             Expr::CellPath(_) => None,
             Expr::ImportPattern(_) => None,

crates/nu-cli/tests/completions.rs

@@ -66,7 +66,7 @@ fn custom_completer() -> NuCompleter {
 
     // Add record value as example
     let record = r#"
-        let external_completer = {|spans| 
+        let external_completer = {|spans|
             $spans
         }
 
@@ -684,13 +684,14 @@ fn variables_completions() {
     // Test completions for $nu
     let suggestions = completer.complete("$nu.", 4);
 
-    assert_eq!(14, suggestions.len());
+    assert_eq!(15, suggestions.len());
 
     let expected: Vec<String> = vec![
         "config-path".into(),
         "current-exe".into(),
         "default-config-dir".into(),
         "env-path".into(),
+        "history-enabled".into(),
         "history-path".into(),
         "home-path".into(),
         "is-interactive".into(),
@@ -709,9 +710,13 @@ fn variables_completions() {
     // Test completions for $nu.h (filter)
     let suggestions = completer.complete("$nu.h", 5);
 
-    assert_eq!(2, suggestions.len());
+    assert_eq!(3, suggestions.len());
 
-    let expected: Vec<String> = vec!["history-path".into(), "home-path".into()];
+    let expected: Vec<String> = vec![
+        "history-enabled".into(),
+        "history-path".into(),
+        "home-path".into(),
+    ];
 
     // Match results
     match_suggestions(expected, suggestions);

crates/nu-cmd-base/Cargo.toml

@@ -5,21 +5,21 @@ edition = "2021"
 license = "MIT"
 name = "nu-cmd-base"
 repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-base"
-version = "0.88.2"
+version = "0.90.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nu-engine = { path = "../nu-engine", version = "0.88.2" }
-nu-glob = { path = "../nu-glob", version = "0.88.2" }
-nu-parser = { path = "../nu-parser", version = "0.88.2" }
-nu-path = { path = "../nu-path", version = "0.88.2" }
-nu-protocol = { path = "../nu-protocol", version = "0.88.2" }
-nu-utils = { path = "../nu-utils", version = "0.88.2" }
+nu-engine = { path = "../nu-engine", version = "0.90.0" }
+nu-glob = { path = "../nu-glob", version = "0.90.0" }
+nu-parser = { path = "../nu-parser", version = "0.90.0" }
+nu-path = { path = "../nu-path", version = "0.90.0" }
+nu-protocol = { path = "../nu-protocol", version = "0.90.0" }
+nu-utils = { path = "../nu-utils", version = "0.90.0" }
 
 indexmap = "2.1"
 miette = "5.10.0"
 
 [dev-dependencies]
-nu-test-support = { path = "../nu-test-support", version = "0.88.2" }
+nu-test-support = { path = "../nu-test-support", version = "0.90.0" }
 rstest = "0.18.2"

crates/nu-cmd-base/src/arg_glob.rs

@@ -1,205 +0,0 @@
-// utilities for expanding globs in command arguments
-
-use nu_glob::{glob_with_parent, MatchOptions, Paths};
-use nu_protocol::{ShellError, Spanned};
-use std::fs;
-use std::path::{Path, PathBuf};
-
-// standard glob options to use for filesystem command arguments
-
-const GLOB_PARAMS: MatchOptions = MatchOptions {
-    case_sensitive: true,
-    require_literal_separator: false,
-    require_literal_leading_dot: false,
-    recursive_match_hidden_dir: true,
-};
-
-// handle an argument that could be a literal path or a glob.
-// if literal path, return just that (whether user can access it or not).
-// if glob, expand into matching paths, using GLOB_PARAMS options.
-pub fn arg_glob(
-    pattern: &Spanned<String>, // alleged path or glob
-    cwd: &Path,                // current working directory
-) -> Result<Paths, ShellError> {
-    arg_glob_opt(pattern, cwd, GLOB_PARAMS)
-}
-
-// variant of [arg_glob] that requires literal dot prefix in pattern to match dot-prefixed path.
-pub fn arg_glob_leading_dot(pattern: &Spanned<String>, cwd: &Path) -> Result<Paths, ShellError> {
-    arg_glob_opt(
-        pattern,
-        cwd,
-        MatchOptions {
-            require_literal_leading_dot: true,
-            ..GLOB_PARAMS
-        },
-    )
-}
-
-fn arg_glob_opt(
-    pattern: &Spanned<String>,
-    cwd: &Path,
-    options: MatchOptions,
-) -> Result<Paths, ShellError> {
-    // remove ansi coloring (?)
-    let pattern = {
-        Spanned {
-            item: nu_utils::strip_ansi_string_unlikely(pattern.item.clone()),
-            span: pattern.span,
-        }
-    };
-
-    // if there's a file with same path as the pattern, just return that.
-    let pp = cwd.join(&pattern.item);
-    let md = fs::metadata(pp);
-    #[allow(clippy::single_match)]
-    match md {
-        Ok(_metadata) => {
-            return Ok(Paths::single(&PathBuf::from(pattern.item), cwd));
-        }
-        // file not found, but also "invalid chars in file" (e.g * on Windows).  Fall through and glob
-        Err(_) => {}
-    }
-
-    // user wasn't referring to a specific thing in filesystem, try to glob it.
-    match glob_with_parent(&pattern.item, options, cwd) {
-        Ok(p) => Ok(p),
-        Err(pat_err) => Err(ShellError::InvalidGlobPattern {
-            msg: pat_err.msg.into(),
-            span: pattern.span,
-        }),
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use nu_glob::GlobResult;
-    use nu_protocol::{Span, Spanned};
-    use nu_test_support::fs::Stub::EmptyFile;
-    use nu_test_support::playground::Playground;
-    use rstest::rstest;
-
-    fn spanned_string(str: &str) -> Spanned<String> {
-        Spanned {
-            item: str.to_string(),
-            span: Span::test_data(),
-        }
-    }
-
-    #[test]
-    fn does_something() {
-        let act = arg_glob(&spanned_string("*"), &PathBuf::from("."));
-        assert!(act.is_ok());
-        for f in act.expect("checked ok") {
-            match f {
-                Ok(p) => {
-                    assert!(!p.to_str().unwrap().is_empty());
-                }
-                Err(e) => panic!("unexpected error {:?}", e),
-            };
-        }
-    }
-
-    #[test]
-    fn glob_format_error() {
-        let act = arg_glob(&spanned_string(r#"ab]c[def"#), &PathBuf::from("."));
-        assert!(act.is_err());
-    }
-
-    #[rstest]
-    #[case("*", 4, "no dirs")]
-    #[case("**/*", 7, "incl dirs")]
-    fn glob_subdirs(#[case] pat: &str, #[case] exp_count: usize, #[case] case: &str) {
-        Playground::setup("glob_subdirs", |dirs, sandbox| {
-            sandbox.with_files(vec![
-                EmptyFile("yehuda.txt"),
-                EmptyFile("jttxt"),
-                EmptyFile("andres.txt"),
-            ]);
-            sandbox.mkdir(".children");
-            sandbox.within(".children").with_files(vec![
-                EmptyFile("timothy.txt"),
-                EmptyFile("tiffany.txt"),
-                EmptyFile("trish.txt"),
-            ]);
-
-            let p: Vec<GlobResult> = arg_glob(&spanned_string(pat), &dirs.test)
-                .expect("no error")
-                .collect();
-            assert_eq!(
-                exp_count,
-                p.iter().filter(|i| i.is_ok()).count(),
-                " case: {case} ",
-            );
-
-            // expected behavior -- that directories are included in results (if name matches pattern)
-            let t = p
-                .iter()
-                .any(|i| i.as_ref().unwrap().to_string_lossy().contains(".children"));
-            assert!(t, "check for dir, case {case}");
-        })
-    }
-
-    #[rstest]
-    #[case("yehuda.txt", true, 1, "matches literal path")]
-    #[case("*", false, 3, "matches glob")]
-    #[case(r#"bad[glob.foo"#, true, 1, "matches literal, would be bad glob pat")]
-    fn exact_vs_glob(
-        #[case] pat: &str,
-        #[case] exp_matches_input: bool,
-        #[case] exp_count: usize,
-        #[case] case: &str,
-    ) {
-        Playground::setup("exact_vs_glob", |dirs, sandbox| {
-            sandbox.with_files(vec![
-                EmptyFile("yehuda.txt"),
-                EmptyFile("jttxt"),
-                EmptyFile("bad[glob.foo"),
-            ]);
-
-            let res = arg_glob(&spanned_string(pat), &dirs.test)
-                .expect("no error")
-                .collect::<Vec<GlobResult>>();
-
-            eprintln!("res: {:?}", res);
-            if exp_matches_input {
-                assert_eq!(
-                    exp_count,
-                    res.len(),
-                    " case {case}: matches input, but count not 1? "
-                );
-                assert_eq!(
-                    &res[0].as_ref().unwrap().to_string_lossy(),
-                    pat, // todo: is it OK for glob to return relative paths (not to current cwd, but to arg cwd of arg_glob)?
-                );
-            } else {
-                assert_eq!(exp_count, res.len(), " case: {}: matched glob", case);
-            }
-        })
-    }
-
-    #[rstest]
-    #[case(r#"realbad[glob.foo"#, true, 1, "error, bad glob")]
-    fn exact_vs_bad_glob(
-        // if path doesn't exist but pattern is not valid glob, should get error.
-        #[case] pat: &str,
-        #[case] _exp_matches_input: bool,
-        #[case] _exp_count: usize,
-        #[case] _tag: &str,
-    ) {
-        Playground::setup("exact_vs_bad_glob", |dirs, sandbox| {
-            sandbox.with_files(vec![
-                EmptyFile("yehuda.txt"),
-                EmptyFile("jttxt"),
-                EmptyFile("bad[glob.foo"),
-            ]);
-
-            let res = arg_glob(&spanned_string(pat), &dirs.test);
-            assert!(res
-                .expect_err("expected error")
-                .to_string()
-                .contains("Invalid glob pattern"));
-        })
-    }
-}

crates/nu-cmd-base/src/lib.rs

@@ -1,7 +1,4 @@
-mod arg_glob;
 pub mod formats;
 pub mod hook;
 pub mod input_handler;
 pub mod util;
-pub use arg_glob::arg_glob;
-pub use arg_glob::arg_glob_leading_dot;

crates/nu-cmd-dataframe/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 license = "MIT"
 name = "nu-cmd-dataframe"
 repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-dataframe"
-version = "0.88.2"
+version = "0.90.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -13,9 +13,9 @@ version = "0.88.2"
 bench = false
 
 [dependencies]
-nu-engine = { path = "../nu-engine", version = "0.88.2" }
-nu-parser = { path = "../nu-parser", version = "0.88.2" }
-nu-protocol = { path = "../nu-protocol", version = "0.88.2" }
+nu-engine = { path = "../nu-engine", version = "0.90.0" }
+nu-parser = { path = "../nu-parser", version = "0.90.0" }
+nu-protocol = { path = "../nu-protocol", version = "0.90.0" }
 
 # Potential dependencies for extras
 chrono = { version = "0.4", features = ["std", "unstable-locales"], default-features = false }
@@ -24,11 +24,12 @@ fancy-regex = "0.12"
 indexmap = { version = "2.1" }
 num = { version = "0.4", optional = true }
 serde = { version = "1.0", features = ["derive"] }
-sqlparser = { version = "0.39", optional = true }
-polars-io = { version = "0.35", features = ["avro"], optional = true }
-polars-arrow = { version = "0.35", optional = true }
-polars-ops = { version = "0.35", optional = true }
-polars-plan = { version = "0.35", optional = true }
+sqlparser = { version = "0.43", optional = true }
+polars-io = { version = "0.36", features = ["avro"], optional = true }
+polars-arrow = { version = "0.36", optional = true }
+polars-ops = { version = "0.36", optional = true }
+polars-plan = { version = "0.36", optional = true }
+polars-utils = { version = "0.36", optional = true }
 
 [dependencies.polars]
 features = [
@@ -62,12 +63,12 @@ features = [
 	"to_dummies",
 ]
 optional = true
-version = "0.35"
+version = "0.36"
 
 [features]
-dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "sqlparser"]
+dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"]
 default = []
 
 [dev-dependencies]
-nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.88.2" }
-nu-test-support = { path = "../nu-test-support", version = "0.88.2" }
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.90.0" }
+nu-test-support = { path = "../nu-test-support", version = "0.90.0" }

crates/nu-cmd-dataframe/src/dataframe/eager/append.rs

@@ -37,24 +37,27 @@ impl Command for AppendDF {
                 example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
     $a | dfr append $a"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "a_x".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3)],
-                        ),
-                        Column::new(
-                            "b_x".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(1), Value::test_int(3)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "a_x".to_string(),
+                                vec![Value::test_int(1), Value::test_int(3)],
+                            ),
+                            Column::new(
+                                "b_x".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -64,26 +67,29 @@ impl Command for AppendDF {
                 example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
     $a | dfr append $a --col"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![
-                                Value::test_int(1),
-                                Value::test_int(3),
-                                Value::test_int(1),
-                                Value::test_int(3),
-                            ],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![
-                                Value::test_int(2),
-                                Value::test_int(4),
-                                Value::test_int(2),
-                                Value::test_int(4),
-                            ],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![
+                                    Value::test_int(1),
+                                    Value::test_int(3),
+                                    Value::test_int(1),
+                                    Value::test_int(3),
+                                ],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![
+                                    Value::test_int(2),
+                                    Value::test_int(4),
+                                    Value::test_int(2),
+                                    Value::test_int(4),
+                                ],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -110,7 +116,7 @@ fn command(
 ) -> Result<PipelineData, ShellError> {
     let other: Value = call.req(engine_state, stack, 0)?;
 
-    let axis = if call.has_flag("col") {
+    let axis = if call.has_flag(engine_state, stack, "col")? {
         Axis::Column
     } else {
         Axis::Row

crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs

@@ -35,10 +35,13 @@ impl Command for DropDF {
             description: "drop column a",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr drop a",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "b".to_string(),
-                    vec![Value::test_int(2), Value::test_int(4)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "b".to_string(),
+                        vec![Value::test_int(2), Value::test_int(4)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs

@@ -46,16 +46,19 @@ impl Command for DropDuplicates {
             description: "drop duplicates",
             example: "[[a b]; [1 2] [3 4] [1 2]] | dfr into-df | dfr drop-duplicates",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_int(3), Value::test_int(1)],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_int(4), Value::test_int(2)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_int(3), Value::test_int(1)],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_int(4), Value::test_int(2)],
+                        ),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -92,7 +95,7 @@ fn command(
 
     let subset_slice = subset.as_ref().map(|cols| &cols[..]);
 
-    let keep_strategy = if call.has_flag("last") {
+    let keep_strategy = if call.has_flag(engine_state, stack, "last")? {
         UniqueKeepStrategy::Last
     } else {
         UniqueKeepStrategy::First

crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs

@@ -43,20 +43,23 @@ impl Command for DropNulls {
     let a = ($df | dfr with-column $res --name res);
     $a | dfr drop-nulls"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(1)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(2)],
-                        ),
-                        Column::new(
-                            "res".to_string(),
-                            vec![Value::test_int(1), Value::test_int(1)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(1), Value::test_int(1)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(2), Value::test_int(2)],
+                            ),
+                            Column::new(
+                                "res".to_string(),
+                                vec![Value::test_int(1), Value::test_int(1)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -66,15 +69,18 @@ impl Command for DropNulls {
                 example: r#"let s = ([1 2 0 0 3 4] | dfr into-df);
     ($s / $s) | dfr drop-nulls"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "div_0_0".to_string(),
-                        vec![
-                            Value::test_int(1),
-                            Value::test_int(1),
-                            Value::test_int(1),
-                            Value::test_int(1),
-                        ],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "div_0_0".to_string(),
+                            vec![
+                                Value::test_int(1),
+                                Value::test_int(1),
+                                Value::test_int(1),
+                                Value::test_int(1),
+                            ],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs

@@ -31,16 +31,19 @@ impl Command for DataTypes {
             description: "Dataframe dtypes",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dtypes",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "column".to_string(),
-                        vec![Value::test_string("a"), Value::test_string("b")],
-                    ),
-                    Column::new(
-                        "dtype".to_string(),
-                        vec![Value::test_string("i64"), Value::test_string("i64")],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "column".to_string(),
+                            vec![Value::test_string("a"), Value::test_string("b")],
+                        ),
+                        Column::new(
+                            "dtype".to_string(),
+                            vec![Value::test_string("i64"), Value::test_string("i64")],
+                        ),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -79,6 +82,7 @@ fn command(
                 .dtype();
 
             let dtype_str = dtype.to_string();
+
             dtypes.push(Value::string(dtype_str, call.head));
 
             Value::string(*v, call.head)
@@ -88,7 +92,7 @@ fn command(
     let names_col = Column::new("column".to_string(), names);
     let dtypes_col = Column::new("dtype".to_string(), dtypes);
 
-    NuDataFrame::try_from_columns(vec![names_col, dtypes_col])
+    NuDataFrame::try_from_columns(vec![names_col, dtypes_col], None)
         .map(|df| PipelineData::Value(df.into_value(call.head), None))
 }
 

crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs

@@ -1,4 +1,5 @@
 use super::super::values::NuDataFrame;
+use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
@@ -78,12 +79,12 @@ impl Command for Dummies {
 }
 
 fn command(
-    _engine_state: &EngineState,
-    _stack: &mut Stack,
+    engine_state: &EngineState,
+    stack: &mut Stack,
     call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let drop_first: bool = call.has_flag("drop-first");
+    let drop_first: bool = call.has_flag(engine_state, stack, "drop-first")?;
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     df.as_ref()

crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs

@@ -43,10 +43,13 @@ impl Command for FilterWith {
                 example: r#"let mask = ([true false] | dfr into-df);
     [[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with $mask"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new("a".to_string(), vec![Value::test_int(1)]),
-                        Column::new("b".to_string(), vec![Value::test_int(2)]),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new("a".to_string(), vec![Value::test_int(1)]),
+                            Column::new("b".to_string(), vec![Value::test_int(2)]),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -55,10 +58,13 @@ impl Command for FilterWith {
                 description: "Filter dataframe using an expression",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with ((dfr col a) > 1)",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new("a".to_string(), vec![Value::test_int(3)]),
-                        Column::new("b".to_string(), vec![Value::test_int(4)]),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new("a".to_string(), vec![Value::test_int(3)]),
+                            Column::new("b".to_string(), vec![Value::test_int(4)]),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/first.rs

@@ -44,10 +44,13 @@ impl Command for FirstDF {
                 description: "Return the first row of a dataframe",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new("a".to_string(), vec![Value::test_int(1)]),
-                        Column::new("b".to_string(), vec![Value::test_int(2)]),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new("a".to_string(), vec![Value::test_int(1)]),
+                            Column::new("b".to_string(), vec![Value::test_int(2)]),
+                        ],
+                        None,
+                    )
                     .expect("should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -56,16 +59,19 @@ impl Command for FirstDF {
                 description: "Return the first two rows of a dataframe",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first 2",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(1), Value::test_int(3)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/get.rs

@@ -36,10 +36,13 @@ impl Command for GetDF {
             description: "Returns the selected column",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr get a",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "a".to_string(),
-                    vec![Value::test_int(1), Value::test_int(3)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "a".to_string(),
+                        vec![Value::test_int(1), Value::test_int(3)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/eager/last.rs

@@ -40,10 +40,13 @@ impl Command for LastDF {
                 description: "Create new dataframe with last rows",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr last 1",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new("a".to_string(), vec![Value::test_int(3)]),
-                        Column::new("b".to_string(), vec![Value::test_int(4)]),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new("a".to_string(), vec![Value::test_int(3)]),
+                            Column::new("b".to_string(), vec![Value::test_int(4)]),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs

@@ -106,7 +106,7 @@ impl Command for MeltDF {
                             Value::test_string("c"),
                         ],
                     ),
-                ])
+                ], None)
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs

@@ -15,6 +15,7 @@ mod open;
 mod query_df;
 mod rename;
 mod sample;
+mod schema;
 mod shape;
 mod slice;
 mod sql_context;
@@ -49,10 +50,10 @@ pub use melt::MeltDF;
 pub use query_df::QueryDf;
 pub use rename::RenameDF;
 pub use sample::SampleDF;
+pub use schema::SchemaDF;
 pub use shape::ShapeDF;
 pub use slice::SliceDF;
 pub use sql_context::SQLContext;
-pub use sql_expr::parse_sql_expr;
 pub use summary::Summary;
 pub use take::TakeDF;
 pub use to_arrow::ToArrow;
@@ -94,6 +95,7 @@ pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
         QueryDf,
         RenameDF,
         SampleDF,
+        SchemaDF,
         ShapeDF,
         SliceDF,
         TakeDF,

crates/nu-cmd-dataframe/src/dataframe/eager/open.rs

@@ -1,3 +1,5 @@
+use crate::dataframe::values::NuSchema;
+
 use super::super::values::{NuDataFrame, NuLazyFrame};
 use nu_engine::CallExt;
 use nu_protocol::{
@@ -70,6 +72,12 @@ impl Command for OpenDataFrame {
                 "Columns to be selected from csv file. CSV and Parquet file",
                 None,
             )
+            .named(
+                "schema",
+                SyntaxShape::Record(vec![]),
+                r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
+                Some('s')
+            )
             .input_output_type(Type::Any, Type::Custom("dataframe".into()))
             .category(Category::Custom("dataframe".into()))
     }
@@ -139,7 +147,7 @@ fn from_parquet(
     stack: &mut Stack,
     call: &Call,
 ) -> Result<Value, ShellError> {
-    if call.has_flag("lazy") {
+    if call.has_flag(engine_state, stack, "lazy")? {
         let file: String = call.req(engine_state, stack, 0)?;
         let args = ScanArgsParquet {
             n_rows: None,
@@ -238,7 +246,7 @@ fn from_ipc(
     stack: &mut Stack,
     call: &Call,
 ) -> Result<Value, ShellError> {
-    if call.has_flag("lazy") {
+    if call.has_flag(engine_state, stack, "lazy")? {
         let file: String = call.req(engine_state, stack, 0)?;
         let args = ScanArgsIpc {
             n_rows: None,
@@ -305,10 +313,19 @@ fn from_json(
         help: None,
         inner: vec![],
     })?;
+    let maybe_schema = call
+        .get_flag(engine_state, stack, "schema")?
+        .map(|schema| NuSchema::try_from(&schema))
+        .transpose()?;
 
     let buf_reader = BufReader::new(file);
     let reader = JsonReader::new(buf_reader);
 
+    let reader = match maybe_schema {
+        Some(schema) => reader.with_schema(schema.into()),
+        None => reader,
+    };
+
     let df: NuDataFrame = reader
         .finish()
         .map_err(|e| ShellError::GenericError {
@@ -329,6 +346,10 @@ fn from_jsonl(
     call: &Call,
 ) -> Result<Value, ShellError> {
     let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
+    let maybe_schema = call
+        .get_flag(engine_state, stack, "schema")?
+        .map(|schema| NuSchema::try_from(&schema))
+        .transpose()?;
     let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
     let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
         error: "Error opening file".into(),
@@ -343,6 +364,11 @@ fn from_jsonl(
         .with_json_format(JsonFormat::JsonLines)
         .infer_schema_len(infer_schema);
 
+    let reader = match maybe_schema {
+        Some(schema) => reader.with_schema(schema.into()),
+        None => reader,
+    };
+
     let df: NuDataFrame = reader
         .finish()
         .map_err(|e| ShellError::GenericError {
@@ -363,12 +389,17 @@ fn from_csv(
     call: &Call,
 ) -> Result<Value, ShellError> {
     let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
-    let no_header: bool = call.has_flag("no-header");
+    let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
     let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
     let skip_rows: Option<usize> = call.get_flag(engine_state, stack, "skip-rows")?;
     let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
 
-    if call.has_flag("lazy") {
+    let maybe_schema = call
+        .get_flag(engine_state, stack, "schema")?
+        .map(|schema| NuSchema::try_from(&schema))
+        .transpose()?;
+
+    if call.has_flag(engine_state, stack, "lazy")? {
         let file: String = call.req(engine_state, stack, 0)?;
         let csv_reader = LazyCsvReader::new(file);
 
@@ -395,6 +426,11 @@ fn from_csv(
 
         let csv_reader = csv_reader.has_header(!no_header);
 
+        let csv_reader = match maybe_schema {
+            Some(schema) => csv_reader.with_schema(Some(schema.into())),
+            None => csv_reader,
+        };
+
         let csv_reader = match infer_schema {
             None => csv_reader,
             Some(r) => csv_reader.with_infer_schema_length(Some(r)),
@@ -452,6 +488,11 @@ fn from_csv(
 
         let csv_reader = csv_reader.has_header(!no_header);
 
+        let csv_reader = match maybe_schema {
+            Some(schema) => csv_reader.with_schema(Some(schema.into())),
+            None => csv_reader,
+        };
+
         let csv_reader = match infer_schema {
             None => csv_reader,
             Some(r) => csv_reader.infer_schema(Some(r)),

crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs

@@ -44,10 +44,13 @@ impl Command for QueryDf {
             description: "Query dataframe using SQL",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr query 'select a from df'",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "a".to_string(),
-                    vec![Value::test_int(1), Value::test_int(3)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "a".to_string(),
+                        vec![Value::test_int(1), Value::test_int(3)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs

@@ -46,15 +46,18 @@ impl Command for RenameDF {
                 description: "Renames a series",
                 example: "[5 6 7 8] | dfr into-df | dfr rename '0' new_name",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "new_name".to_string(),
-                        vec![
-                            Value::test_int(5),
-                            Value::test_int(6),
-                            Value::test_int(7),
-                            Value::test_int(8),
-                        ],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "new_name".to_string(),
+                            vec![
+                                Value::test_int(5),
+                                Value::test_int(6),
+                                Value::test_int(7),
+                                Value::test_int(8),
+                            ],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -63,16 +66,19 @@ impl Command for RenameDF {
                 description: "Renames a dataframe column",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename a a_new",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a_new".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a_new".to_string(),
+                                vec![Value::test_int(1), Value::test_int(3)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -81,16 +87,19 @@ impl Command for RenameDF {
                 description: "Renames two dataframe columns",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename [a b] [a_new b_new]",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a_new".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3)],
-                        ),
-                        Column::new(
-                            "b_new".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a_new".to_string(),
+                                vec![Value::test_int(1), Value::test_int(3)],
+                            ),
+                            Column::new(
+                                "b_new".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs

@@ -88,8 +88,8 @@ fn command(
     let seed: Option<u64> = call
         .get_flag::<i64>(engine_state, stack, "seed")?
         .map(|val| val as u64);
-    let replace: bool = call.has_flag("replace");
-    let shuffle: bool = call.has_flag("shuffle");
+    let replace: bool = call.has_flag(engine_state, stack, "replace")?;
+    let shuffle: bool = call.has_flag(engine_state, stack, "shuffle")?;
 
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 

crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs

@@ -0,0 +1,119 @@
+use super::super::values::NuDataFrame;
+use nu_engine::CallExt;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, Example, PipelineData, Record, ShellError, Signature, Span, Type, Value,
+};
+
+#[derive(Clone)]
+pub struct SchemaDF;
+
+impl Command for SchemaDF {
+    fn name(&self) -> &str {
+        "dfr schema"
+    }
+
+    fn usage(&self) -> &str {
+        "Show schema for a dataframe."
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .switch("datatype-list", "creates a lazy dataframe", Some('l'))
+            .input_output_type(
+                Type::Custom("dataframe".into()),
+                Type::Custom("dataframe".into()),
+            )
+            .category(Category::Custom("dataframe".into()))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Dataframe schema",
+            example: r#"[[a b]; [1 "foo"] [3 "bar"]] | dfr into-df | dfr schema"#,
+            result: Some(Value::record(
+                Record::from_raw_cols_vals_unchecked(
+                    vec!["a".to_string(), "b".to_string()],
+                    vec![
+                        Value::string("i64", Span::test_data()),
+                        Value::string("str", Span::test_data()),
+                    ],
+                ),
+                Span::test_data(),
+            )),
+        }]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        if call.has_flag(engine_state, stack, "datatype-list")? {
+            Ok(PipelineData::Value(datatype_list(Span::unknown()), None))
+        } else {
+            command(engine_state, stack, call, input)
+        }
+    }
+}
+
+fn command(
+    _engine_state: &EngineState,
+    _stack: &mut Stack,
+    call: &Call,
+    input: PipelineData,
+) -> Result<PipelineData, ShellError> {
+    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
+    let schema = df.schema();
+    let value: Value = schema.into();
+    Ok(PipelineData::Value(value, None))
+}
+
+fn datatype_list(span: Span) -> Value {
+    let types: Vec<Value> = [
+        ("null", ""),
+        ("bool", ""),
+        ("u8", ""),
+        ("u16", ""),
+        ("u32", ""),
+        ("u64", ""),
+        ("i8", ""),
+        ("i16", ""),
+        ("i32", ""),
+        ("i64", ""),
+        ("f32", ""),
+        ("f64", ""),
+        ("str", ""),
+        ("binary", ""),
+        ("date", ""),
+        ("datetime<time_unit: (ms, us, ns) timezone (optional)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns. Timezone wildcard is *. Other Timezone examples: UTC, America/Los_Angeles."),
+        ("duration<time_unit: (ms, us, ns)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns."),
+        ("time", ""),
+        ("object", ""),
+        ("unknown", ""),
+        ("list<dtype>", ""),
+    ]
+    .iter()
+    .map(|(dtype, note)| {
+        Value::record(Record::from_raw_cols_vals_unchecked(
+            vec!["dtype".to_string(), "note".to_string()],
+            vec![Value::string(*dtype, span), Value::string(*note, span)],
+        ),span)
+    })
+    .collect();
+    Value::list(types, span)
+}
+
+#[cfg(test)]
+mod test {
+    use super::super::super::test_dataframe::test_dataframe;
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        test_dataframe(vec![Box::new(SchemaDF {})])
+    }
+}

crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs

@@ -34,10 +34,13 @@ impl Command for ShapeDF {
             description: "Shows row and column shape",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr shape",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new("rows".to_string(), vec![Value::test_int(2)]),
-                    Column::new("columns".to_string(), vec![Value::test_int(2)]),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new("rows".to_string(), vec![Value::test_int(2)]),
+                        Column::new("columns".to_string(), vec![Value::test_int(2)]),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -70,7 +73,7 @@ fn command(
     let rows_col = Column::new("rows".to_string(), vec![rows]);
     let cols_col = Column::new("columns".to_string(), vec![cols]);
 
-    NuDataFrame::try_from_columns(vec![rows_col, cols_col])
+    NuDataFrame::try_from_columns(vec![rows_col, cols_col], None)
         .map(|df| PipelineData::Value(df.into_value(call.head), None))
 }
 

crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs

@@ -37,10 +37,13 @@ impl Command for SliceDF {
             description: "Create new dataframe from a slice of the rows",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr slice 0 1",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new("a".to_string(), vec![Value::test_int(1)]),
-                    Column::new("b".to_string(), vec![Value::test_int(2)]),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new("a".to_string(), vec![Value::test_int(1)]),
+                        Column::new("b".to_string(), vec![Value::test_int(2)]),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs

@@ -13,7 +13,7 @@ fn map_sql_polars_datatype(data_type: &SQLDataType) -> Result<DataType> {
         | SQLDataType::Uuid
         | SQLDataType::Clob(_)
         | SQLDataType::Text
-        | SQLDataType::String(_) => DataType::Utf8,
+        | SQLDataType::String(_) => DataType::String,
         SQLDataType::Float(_) => DataType::Float32,
         SQLDataType::Real => DataType::Float32,
         SQLDataType::Double => DataType::Float64,
@@ -62,7 +62,9 @@ fn binary_op_(left: Expr, right: Expr, op: &SQLBinaryOperator) -> Result<Expr> {
         SQLBinaryOperator::Multiply => left * right,
         SQLBinaryOperator::Divide => left / right,
         SQLBinaryOperator::Modulo => left % right,
-        SQLBinaryOperator::StringConcat => left.cast(DataType::Utf8) + right.cast(DataType::Utf8),
+        SQLBinaryOperator::StringConcat => {
+            left.cast(DataType::String) + right.cast(DataType::String)
+        }
         SQLBinaryOperator::Gt => left.gt(right),
         SQLBinaryOperator::Lt => left.lt(right),
         SQLBinaryOperator::GtEq => left.gt_eq(right),

crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs

@@ -10,7 +10,7 @@ use polars::{
     chunked_array::ChunkedArray,
     prelude::{
         AnyValue, DataFrame, DataType, Float64Type, IntoSeries, NewChunkedArray,
-        QuantileInterpolOptions, Series, Utf8Type,
+        QuantileInterpolOptions, Series, StringType,
     },
 };
 
@@ -46,53 +46,56 @@ impl Command for Summary {
             description: "list dataframe descriptives",
             example: "[[a b]; [1 1] [1 1]] | dfr into-df | dfr summary",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "descriptor".to_string(),
-                        vec![
-                            Value::test_string("count"),
-                            Value::test_string("sum"),
-                            Value::test_string("mean"),
-                            Value::test_string("median"),
-                            Value::test_string("std"),
-                            Value::test_string("min"),
-                            Value::test_string("25%"),
-                            Value::test_string("50%"),
-                            Value::test_string("75%"),
-                            Value::test_string("max"),
-                        ],
-                    ),
-                    Column::new(
-                        "a (i64)".to_string(),
-                        vec![
-                            Value::test_float(2.0),
-                            Value::test_float(2.0),
-                            Value::test_float(1.0),
-                            Value::test_float(1.0),
-                            Value::test_float(0.0),
-                            Value::test_float(1.0),
-                            Value::test_float(1.0),
-                            Value::test_float(1.0),
-                            Value::test_float(1.0),
-                            Value::test_float(1.0),
-                        ],
-                    ),
-                    Column::new(
-                        "b (i64)".to_string(),
-                        vec![
-                            Value::test_float(2.0),
-                            Value::test_float(2.0),
-                            Value::test_float(1.0),
-                            Value::test_float(1.0),
-                            Value::test_float(0.0),
-                            Value::test_float(1.0),
-                            Value::test_float(1.0),
-                            Value::test_float(1.0),
-                            Value::test_float(1.0),
-                            Value::test_float(1.0),
-                        ],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "descriptor".to_string(),
+                            vec![
+                                Value::test_string("count"),
+                                Value::test_string("sum"),
+                                Value::test_string("mean"),
+                                Value::test_string("median"),
+                                Value::test_string("std"),
+                                Value::test_string("min"),
+                                Value::test_string("25%"),
+                                Value::test_string("50%"),
+                                Value::test_string("75%"),
+                                Value::test_string("max"),
+                            ],
+                        ),
+                        Column::new(
+                            "a (i64)".to_string(),
+                            vec![
+                                Value::test_float(2.0),
+                                Value::test_float(2.0),
+                                Value::test_float(1.0),
+                                Value::test_float(1.0),
+                                Value::test_float(0.0),
+                                Value::test_float(1.0),
+                                Value::test_float(1.0),
+                                Value::test_float(1.0),
+                                Value::test_float(1.0),
+                                Value::test_float(1.0),
+                            ],
+                        ),
+                        Column::new(
+                            "b (i64)".to_string(),
+                            vec![
+                                Value::test_float(2.0),
+                                Value::test_float(2.0),
+                                Value::test_float(1.0),
+                                Value::test_float(1.0),
+                                Value::test_float(0.0),
+                                Value::test_float(1.0),
+                                Value::test_float(1.0),
+                                Value::test_float(1.0),
+                                Value::test_float(1.0),
+                                Value::test_float(1.0),
+                            ],
+                        ),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -171,7 +174,7 @@ fn command(
 
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
-    let names = ChunkedArray::<Utf8Type>::from_slice_options("descriptor", &labels).into_series();
+    let names = ChunkedArray::<StringType>::from_slice_options("descriptor", &labels).into_series();
 
     let head = std::iter::once(names);
 
@@ -179,42 +182,50 @@ fn command(
         .as_ref()
         .get_columns()
         .iter()
-        .filter(|col| col.dtype() != &DataType::Object("object"))
+        .filter(|col| !matches!(col.dtype(), &DataType::Object("object", _)))
         .map(|col| {
             let count = col.len() as f64;
 
-            let sum = col
-                .sum_as_series()
-                .cast(&DataType::Float64)
-                .ok()
-                .and_then(|ca| match ca.get(0) {
-                    Ok(AnyValue::Float64(v)) => Some(v),
-                    _ => None,
-                });
+            let sum = col.sum_as_series().ok().and_then(|series| {
+                series
+                    .cast(&DataType::Float64)
+                    .ok()
+                    .and_then(|ca| match ca.get(0) {
+                        Ok(AnyValue::Float64(v)) => Some(v),
+                        _ => None,
+                    })
+            });
 
             let mean = match col.mean_as_series().get(0) {
                 Ok(AnyValue::Float64(v)) => Some(v),
                 _ => None,
             };
 
-            let median = match col.median_as_series().get(0) {
-                Ok(AnyValue::Float64(v)) => Some(v),
-                _ => None,
-            };
-
-            let std = match col.std_as_series(0).get(0) {
-                Ok(AnyValue::Float64(v)) => Some(v),
-                _ => None,
-            };
-
-            let min = col
-                .min_as_series()
-                .cast(&DataType::Float64)
-                .ok()
-                .and_then(|ca| match ca.get(0) {
+            let median = match col.median_as_series() {
+                Ok(v) => match v.get(0) {
                     Ok(AnyValue::Float64(v)) => Some(v),
                     _ => None,
-                });
+                },
+                _ => None,
+            };
+
+            let std = match col.std_as_series(0) {
+                Ok(v) => match v.get(0) {
+                    Ok(AnyValue::Float64(v)) => Some(v),
+                    _ => None,
+                },
+                _ => None,
+            };
+
+            let min = col.min_as_series().ok().and_then(|series| {
+                series
+                    .cast(&DataType::Float64)
+                    .ok()
+                    .and_then(|ca| match ca.get(0) {
+                        Ok(AnyValue::Float64(v)) => Some(v),
+                        _ => None,
+                    })
+            });
 
             let mut quantiles = quantiles
                 .clone()
@@ -230,14 +241,15 @@ fn command(
                 })
                 .collect::<Vec<Option<f64>>>();
 
-            let max = col
-                .max_as_series()
-                .cast(&DataType::Float64)
-                .ok()
-                .and_then(|ca| match ca.get(0) {
-                    Ok(AnyValue::Float64(v)) => Some(v),
-                    _ => None,
-                });
+            let max = col.max_as_series().ok().and_then(|series| {
+                series
+                    .cast(&DataType::Float64)
+                    .ok()
+                    .and_then(|ca| match ca.get(0) {
+                        Ok(AnyValue::Float64(v)) => Some(v),
+                        _ => None,
+                    })
+            });
 
             let mut descriptors = vec![Some(count), sum, mean, median, std, min];
             descriptors.append(&mut quantiles);

crates/nu-cmd-dataframe/src/dataframe/eager/take.rs

@@ -44,16 +44,19 @@ impl Command for TakeDF {
     let indices = ([0 2] | dfr into-df);
     $df | dfr take $indices"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(4), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(4), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(1), Value::test_int(3)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -64,10 +67,13 @@ impl Command for TakeDF {
     let indices = ([0 2] | dfr into-df);
     $series | dfr take $indices"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "0".to_string(),
-                        vec![Value::test_int(4), Value::test_int(5)],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "0".to_string(),
+                            vec![Value::test_int(4), Value::test_int(5)],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs

@@ -70,7 +70,7 @@ fn command(
 ) -> Result<PipelineData, ShellError> {
     let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
     let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
-    let no_header: bool = call.has_flag("no-header");
+    let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
 
     let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
 

crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs

@@ -1,10 +1,14 @@
+use crate::dataframe::values::NuSchema;
+
 use super::super::values::{Column, NuDataFrame};
 
+use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Category, Example, PipelineData, ShellError, Signature, Span, Type, Value,
+    Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
 };
+use polars::prelude::*;
 
 #[derive(Clone)]
 pub struct ToDataFrame;
@@ -20,6 +24,12 @@ impl Command for ToDataFrame {
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
+            .named(
+                "schema",
+                SyntaxShape::Record(vec![]),
+                r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
+                Some('s'),
+            )
             .input_output_type(Type::Any, Type::Custom("dataframe".into()))
             .category(Category::Custom("dataframe".into()))
     }
@@ -30,16 +40,19 @@ impl Command for ToDataFrame {
                 description: "Takes a dictionary and creates a dataframe",
                 example: "[[a b];[1 2] [3 4]] | dfr into-df",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(1), Value::test_int(3)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -48,24 +61,27 @@ impl Command for ToDataFrame {
                 description: "Takes a list of tables and creates a dataframe",
                 example: "[[1 2 a] [3 4 b] [5 6 c]] | dfr into-df",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "0".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3), Value::test_int(5)],
-                        ),
-                        Column::new(
-                            "1".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4), Value::test_int(6)],
-                        ),
-                        Column::new(
-                            "2".to_string(),
-                            vec![
-                                Value::test_string("a"),
-                                Value::test_string("b"),
-                                Value::test_string("c"),
-                            ],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "0".to_string(),
+                                vec![Value::test_int(1), Value::test_int(3), Value::test_int(5)],
+                            ),
+                            Column::new(
+                                "1".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4), Value::test_int(6)],
+                            ),
+                            Column::new(
+                                "2".to_string(),
+                                vec![
+                                    Value::test_string("a"),
+                                    Value::test_string("b"),
+                                    Value::test_string("c"),
+                                ],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -74,14 +90,17 @@ impl Command for ToDataFrame {
                 description: "Takes a list and creates a dataframe",
                 example: "[a b c] | dfr into-df",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "0".to_string(),
-                        vec![
-                            Value::test_string("a"),
-                            Value::test_string("b"),
-                            Value::test_string("c"),
-                        ],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "0".to_string(),
+                            vec![
+                                Value::test_string("a"),
+                                Value::test_string("b"),
+                                Value::test_string("c"),
+                            ],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -90,14 +109,41 @@ impl Command for ToDataFrame {
                 description: "Takes a list of booleans and creates a dataframe",
                 example: "[true true false] | dfr into-df",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "0".to_string(),
-                        vec![
-                            Value::test_bool(true),
-                            Value::test_bool(true),
-                            Value::test_bool(false),
-                        ],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "0".to_string(),
+                            vec![
+                                Value::test_bool(true),
+                                Value::test_bool(true),
+                                Value::test_bool(false),
+                            ],
+                        )],
+                        None,
+                    )
+                    .expect("simple df for test should not fail")
+                    .into_value(Span::test_data()),
+                ),
+            },
+            Example {
+                description: "Convert to a dataframe and provide a schema",
+                example: "{a: 1, b: {a: [1 2 3]}, c: [a b c]}| dfr into-df -s {a: u8, b: {a: list<u64>}, c: list<str>}",
+                result: Some(
+                    NuDataFrame::try_from_series(vec![
+                        Series::new("a", &[1u8]),
+                        {
+                            let dtype = DataType::Struct(vec![Field::new("a", DataType::List(Box::new(DataType::UInt64)))]);
+                            let vals = vec![AnyValue::StructOwned(
+                                Box::new((vec![AnyValue::List(Series::new("a", &[1u64, 2, 3]))], vec![Field::new("a", DataType::String)]))); 1];
+                            Series::from_any_values_and_dtype("b", &vals, &dtype, false)
+                                .expect("Struct series should not fail")
+                        },
+                        {
+                            let dtype = DataType::List(Box::new(DataType::String));
+                            let vals = vec![AnyValue::List(Series::new("c", &["a", "b", "c"]))];
+                            Series::from_any_values_and_dtype("c", &vals, &dtype, false)
+                                .expect("List series should not fail")
+                        }
+                    ], Span::test_data())
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -107,12 +153,17 @@ impl Command for ToDataFrame {
 
     fn run(
         &self,
-        _engine_state: &EngineState,
-        _stack: &mut Stack,
+        engine_state: &EngineState,
+        stack: &mut Stack,
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        NuDataFrame::try_from_iter(input.into_iter())
+        let maybe_schema = call
+            .get_flag(engine_state, stack, "schema")?
+            .map(|schema| NuSchema::try_from(&schema))
+            .transpose()?;
+
+        NuDataFrame::try_from_iter(input.into_iter(), maybe_schema)
             .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
     }
 }

crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs

@@ -100,7 +100,7 @@ fn dataframe_command(
     input: Value,
 ) -> Result<PipelineData, ShellError> {
     let rows: Option<usize> = call.get_flag(engine_state, stack, "rows")?;
-    let tail: bool = call.has_flag("tail");
+    let tail: bool = call.has_flag(engine_state, stack, "tail")?;
 
     let df = NuDataFrame::try_from_value(input)?;
 

crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs

@@ -42,20 +42,23 @@ impl Command for WithColumn {
     | dfr into-df
     | dfr with-column ([5 6] | dfr into-df) --name c"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "c".to_string(),
-                            vec![Value::test_int(5), Value::test_int(6)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(1), Value::test_int(3)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "c".to_string(),
+                                vec![Value::test_int(5), Value::test_int(6)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -70,24 +73,27 @@ impl Command for WithColumn {
       ]
     | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(3)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "c".to_string(),
-                            vec![Value::test_int(2), Value::test_int(6)],
-                        ),
-                        Column::new(
-                            "d".to_string(),
-                            vec![Value::test_int(3), Value::test_int(9)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(1), Value::test_int(3)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "c".to_string(),
+                                vec![Value::test_int(2), Value::test_int(6)],
+                            ),
+                            Column::new(
+                                "d".to_string(),
+                                vec![Value::test_int(3), Value::test_int(9)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs

@@ -32,10 +32,13 @@ impl Command for ExprArgWhere {
             example: "let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df);
     $df | dfr select (dfr arg-where ((dfr col b) >= 2) | dfr as b_arg)",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "b_arg".to_string(),
-                    vec![Value::test_int(1), Value::test_int(2)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "b_arg".to_string(),
+                        vec![Value::test_int(1), Value::test_int(2)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs

@@ -41,27 +41,30 @@ impl Command for ExprConcatStr {
             example: r#"let df = ([[a b c]; [one two 1] [three four 2]] | dfr into-df);
     $df | dfr with-column ((dfr concat-str "-" [(dfr col a) (dfr col b) ((dfr col c) * 2)]) | dfr as concat)"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_string("one"), Value::test_string("three")],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_string("two"), Value::test_string("four")],
-                    ),
-                    Column::new(
-                        "c".to_string(),
-                        vec![Value::test_int(1), Value::test_int(2)],
-                    ),
-                    Column::new(
-                        "concat".to_string(),
-                        vec![
-                            Value::test_string("one-two-2"),
-                            Value::test_string("three-four-4"),
-                        ],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_string("one"), Value::test_string("three")],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_string("two"), Value::test_string("four")],
+                        ),
+                        Column::new(
+                            "c".to_string(),
+                            vec![Value::test_int(1), Value::test_int(2)],
+                        ),
+                        Column::new(
+                            "concat".to_string(),
+                            vec![
+                                Value::test_string("one-two-2"),
+                                Value::test_string("three-four-4"),
+                            ],
+                        ),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs

@@ -9,6 +9,11 @@ use nu_protocol::{
     Category, Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Type,
     Value,
 };
+use polars::{
+    datatypes::{DataType, TimeUnit},
+    prelude::NamedFrom,
+    series::Series,
+};
 
 #[derive(Clone)]
 pub struct ExprDatePart;
@@ -47,10 +52,13 @@ impl Command for ExprDatePart {
                 description: "Creates an expression to capture the year date part",
                 example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" | dfr with-column [(dfr col datetime | dfr datepart year | dfr as datetime_year )]"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new("datetime".to_string(), vec![Value::test_date(dt)]),
-                        Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new("datetime".to_string(), vec![Value::test_date(dt)]),
+                            Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -66,16 +74,21 @@ impl Command for ExprDatePart {
                 (dfr col datetime | dfr datepart second | dfr as datetime_second ),
                 (dfr col datetime | dfr datepart nanosecond | dfr as datetime_ns ) ]"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new("datetime".to_string(), vec![Value::test_date(dt)]),
-                        Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]),
-                        Column::new("datetime_month".to_string(), vec![Value::test_int(12)]),
-                        Column::new("datetime_day".to_string(), vec![Value::test_int(30)]),
-                        Column::new("datetime_hour".to_string(), vec![Value::test_int(1)]),
-                        Column::new("datetime_minute".to_string(), vec![Value::test_int(2)]),
-                        Column::new("datetime_second".to_string(), vec![Value::test_int(3)]),
-                        Column::new("datetime_ns".to_string(), vec![Value::test_int(123456789)]),
-                    ])
+                    NuDataFrame::try_from_series(
+                        vec![
+                            Series::new("datetime", &[dt.timestamp_nanos_opt()])
+                                .cast(&DataType::Datetime(TimeUnit::Nanoseconds, None))
+                                .expect("Error casting to datetime type"),
+                            Series::new("datetime_year", &[2021_i64]), // i32 was coerced to i64
+                            Series::new("datetime_month", &[12_i8]),
+                            Series::new("datetime_day", &[30_i8]),
+                            Series::new("datetime_hour", &[1_i8]),
+                            Series::new("datetime_minute", &[2_i8]),
+                            Series::new("datetime_second", &[3_i8]),
+                            Series::new("datetime_ns", &[123456789_i64]), // i32 was coerced to i64
+                        ],
+                        Span::test_data(),
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs

@@ -179,7 +179,18 @@ macro_rules! lazy_expr_command {
                 let value = input.into_value(call.head);
                 if NuDataFrame::can_downcast(&value) {
                     let lazy = NuLazyFrame::try_from_value(value)?;
-                    let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func());
+                    let lazy = NuLazyFrame::new(
+                        lazy.from_eager,
+                        lazy.into_polars()
+                            .$func()
+                            .map_err(|e| ShellError::GenericError {
+                                error: "Dataframe Error".into(),
+                                msg: e.to_string(),
+                                help: None,
+                                span: None,
+                                inner: vec![],
+                            })?,
+                    );
 
                     Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
                 } else {
@@ -267,7 +278,18 @@ macro_rules! lazy_expr_command {
                 let value = input.into_value(call.head);
                 if NuDataFrame::can_downcast(&value) {
                     let lazy = NuLazyFrame::try_from_value(value)?;
-                    let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func($ddof));
+                    let lazy = NuLazyFrame::new(
+                        lazy.from_eager,
+                        lazy.into_polars()
+                            .$func($ddof)
+                            .map_err(|e| ShellError::GenericError {
+                                error: "Dataframe Error".into(),
+                                msg: e.to_string(),
+                                help: None,
+                                span: None,
+                                inner: vec![],
+                            })?,
+                    );
 
                     Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
                 } else {
@@ -385,10 +407,13 @@ lazy_expr_command!(
             description: "Max value from columns in a dataframe",
             example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr max",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new("a".to_string(), vec![Value::test_int(6)],),
-                    Column::new("b".to_string(), vec![Value::test_int(4)],),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new("a".to_string(), vec![Value::test_int(6)],),
+                        Column::new("b".to_string(), vec![Value::test_int(4)],),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -400,16 +425,19 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr max)"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_string("one"), Value::test_string("two")],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_int(4), Value::test_int(1)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_string("one"), Value::test_string("two")],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_int(4), Value::test_int(1)],
+                        ),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -430,10 +458,13 @@ lazy_expr_command!(
             description: "Min value from columns in a dataframe",
             example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr min",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new("a".to_string(), vec![Value::test_int(1)],),
-                    Column::new("b".to_string(), vec![Value::test_int(1)],),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new("a".to_string(), vec![Value::test_int(1)],),
+                        Column::new("b".to_string(), vec![Value::test_int(1)],),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -445,16 +476,19 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr min)"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_string("one"), Value::test_string("two")],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_int(2), Value::test_int(1)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_string("one"), Value::test_string("two")],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(1)],
+                        ),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -475,10 +509,13 @@ lazy_expr_command!(
             description: "Sums all columns in a dataframe",
             example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr sum",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new("a".to_string(), vec![Value::test_int(11)],),
-                    Column::new("b".to_string(), vec![Value::test_int(7)],),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new("a".to_string(), vec![Value::test_int(11)],),
+                        Column::new("b".to_string(), vec![Value::test_int(7)],),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -490,16 +527,19 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr sum)"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_string("one"), Value::test_string("two")],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_int(6), Value::test_int(1)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_string("one"), Value::test_string("two")],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_int(6), Value::test_int(1)],
+                        ),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -520,10 +560,13 @@ lazy_expr_command!(
             description: "Mean value from columns in a dataframe",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr mean",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new("a".to_string(), vec![Value::test_float(4.0)],),
-                    Column::new("b".to_string(), vec![Value::test_float(2.0)],),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new("a".to_string(), vec![Value::test_float(4.0)],),
+                        Column::new("b".to_string(), vec![Value::test_float(2.0)],),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -535,16 +578,19 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr mean)"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_string("one"), Value::test_string("two")],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_float(3.0), Value::test_float(1.0)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_string("one"), Value::test_string("two")],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_float(3.0), Value::test_float(1.0)],
+                        ),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -567,16 +613,19 @@ expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr median)"#,
         result: Some(
-            NuDataFrame::try_from_columns(vec![
-                Column::new(
-                    "a".to_string(),
-                    vec![Value::test_string("one"), Value::test_string("two")],
-                ),
-                Column::new(
-                    "b".to_string(),
-                    vec![Value::test_float(3.0), Value::test_float(1.0)],
-                ),
-            ])
+            NuDataFrame::try_from_columns(
+                vec![
+                    Column::new(
+                        "a".to_string(),
+                        vec![Value::test_string("one"), Value::test_string("two")],
+                    ),
+                    Column::new(
+                        "b".to_string(),
+                        vec![Value::test_float(3.0), Value::test_float(1.0)],
+                    ),
+                ],
+                None
+            )
             .expect("simple df for test should not fail")
             .into_value(Span::test_data()),
         ),
@@ -596,10 +645,13 @@ lazy_expr_command!(
             description: "Std value from columns in a dataframe",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr std",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new("a".to_string(), vec![Value::test_float(2.0)],),
-                    Column::new("b".to_string(), vec![Value::test_float(0.0)],),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new("a".to_string(), vec![Value::test_float(2.0)],),
+                        Column::new("b".to_string(), vec![Value::test_float(0.0)],),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -611,16 +663,19 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr std)"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_string("one"), Value::test_string("two")],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_float(0.0), Value::test_float(0.0)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_string("one"), Value::test_string("two")],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_float(0.0), Value::test_float(0.0)],
+                        ),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -643,10 +698,13 @@ lazy_expr_command!(
                 "Var value from columns in a dataframe or aggregates columns to their var value",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr var",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new("a".to_string(), vec![Value::test_float(4.0)],),
-                    Column::new("b".to_string(), vec![Value::test_float(0.0)],),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new("a".to_string(), vec![Value::test_float(4.0)],),
+                        Column::new("b".to_string(), vec![Value::test_float(0.0)],),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -658,16 +716,19 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr var)"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_string("one"), Value::test_string("two")],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_float(0.0), Value::test_float(0.0)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_string("one"), Value::test_string("two")],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_float(0.0), Value::test_float(0.0)],
+                        ),
+                    ],
+                    None
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/expressions/is_in.rs

@@ -39,28 +39,31 @@ impl Command for ExprIsIn {
             example: r#"let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df);
     $df | dfr with-column (dfr col a | dfr is-in [one two] | dfr as a_in)"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![
-                            Value::test_string("one"),
-                            Value::test_string("two"),
-                            Value::test_string("three"),
-                        ],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)],
-                    ),
-                    Column::new(
-                        "a_in".to_string(),
-                        vec![
-                            Value::test_bool(true),
-                            Value::test_bool(true),
-                            Value::test_bool(false),
-                        ],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![
+                                Value::test_string("one"),
+                                Value::test_string("two"),
+                                Value::test_string("three"),
+                            ],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)],
+                        ),
+                        Column::new(
+                            "a_in".to_string(),
+                            vec![
+                                Value::test_bool(true),
+                                Value::test_bool(true),
+                                Value::test_bool(false),
+                            ],
+                        ),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -81,7 +84,8 @@ impl Command for ExprIsIn {
         let list: Vec<Value> = call.req(engine_state, stack, 0)?;
         let expr = NuExpression::try_from_pipeline(input, call.head)?;
 
-        let values = NuDataFrame::try_from_columns(vec![Column::new("list".to_string(), list)])?;
+        let values =
+            NuDataFrame::try_from_columns(vec![Column::new("list".to_string(), list)], None)?;
         let list = values.as_series(call.head)?;
 
         if matches!(list.dtype(), DataType::Object(..)) {

crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs

@@ -54,24 +54,27 @@ impl Command for ExprOtherwise {
      )
    | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
-                        ),
-                        Column::new(
-                            "c".to_string(),
-                            vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "d".to_string(),
-                            vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
+                            ),
+                            Column::new(
+                                "c".to_string(),
+                                vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "d".to_string(),
+                                vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs

@@ -41,16 +41,19 @@ impl Command for ExprQuantile {
     | dfr group-by a
     | dfr agg (dfr col b | dfr quantile 0.5)"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_string("one"), Value::test_string("two")],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_float(4.0), Value::test_float(1.0)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_string("one"), Value::test_string("two")],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_float(4.0), Value::test_float(1.0)],
+                        ),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs

@@ -62,24 +62,27 @@ impl Command for ExprWhen {
      )
    | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
-                        ),
-                        Column::new(
-                            "c".to_string(),
-                            vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "d".to_string(),
-                            vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
+                            ),
+                            Column::new(
+                                "c".to_string(),
+                                vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "d".to_string(),
+                                vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs

@@ -47,24 +47,27 @@ impl Command for LazyAggregate {
         (dfr col b | dfr sum | dfr as "b_sum")
      ]"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(2)],
-                        ),
-                        Column::new(
-                            "b_min".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "b_max".to_string(),
-                            vec![Value::test_int(4), Value::test_int(6)],
-                        ),
-                        Column::new(
-                            "b_sum".to_string(),
-                            vec![Value::test_int(6), Value::test_int(10)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(1), Value::test_int(2)],
+                            ),
+                            Column::new(
+                                "b_min".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "b_max".to_string(),
+                                vec![Value::test_int(4), Value::test_int(6)],
+                            ),
+                            Column::new(
+                                "b_sum".to_string(),
+                                vec![Value::test_int(6), Value::test_int(10)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -81,24 +84,27 @@ impl Command for LazyAggregate {
      ]
     | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(2)],
-                        ),
-                        Column::new(
-                            "b_min".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "b_max".to_string(),
-                            vec![Value::test_int(4), Value::test_int(6)],
-                        ),
-                        Column::new(
-                            "b_sum".to_string(),
-                            vec![Value::test_int(6), Value::test_int(10)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(1), Value::test_int(2)],
+                            ),
+                            Column::new(
+                                "b_min".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "b_max".to_string(),
+                                vec![Value::test_int(4), Value::test_int(6)],
+                            ),
+                            Column::new(
+                                "b_sum".to_string(),
+                                vec![Value::test_int(6), Value::test_int(10)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -160,7 +166,7 @@ fn get_col_name(expr: &Expr) -> Option<String> {
             | polars::prelude::AggExpr::Last(e)
             | polars::prelude::AggExpr::Mean(e)
             | polars::prelude::AggExpr::Implode(e)
-            | polars::prelude::AggExpr::Count(e)
+            | polars::prelude::AggExpr::Count(e, _)
             | polars::prelude::AggExpr::Sum(e)
             | polars::prelude::AggExpr::AggGroups(e)
             | polars::prelude::AggExpr::Std(e, _)

crates/nu-cmd-dataframe/src/dataframe/lazy/collect.rs

@@ -33,16 +33,19 @@ impl Command for LazyCollect {
             description: "drop duplicates",
             example: "[[a b]; [1 2] [3 4]] | dfr into-lazy | dfr collect",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_int(1), Value::test_int(3)],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_int(2), Value::test_int(4)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
+                        ),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs

@@ -69,7 +69,7 @@ impl Command for LazyExplode {
                             Value::test_string("Skiing"),
                             Value::test_string("Football"),
                         ]),
-                   ]).expect("simple df for test should not fail")
+                   ], None).expect("simple df for test should not fail")
                    .into_value(Span::test_data()),
                     )
             },
@@ -86,7 +86,7 @@ impl Command for LazyExplode {
                             Value::test_string("Skiing"),
                             Value::test_string("Football"),
                         ]),
-                   ]).expect("simple df for test should not fail")
+                   ], None).expect("simple df for test should not fail")
                    .into_value(Span::test_data()),
                     ),
             },

crates/nu-cmd-dataframe/src/dataframe/lazy/fetch.rs

@@ -38,16 +38,19 @@ impl Command for LazyFetch {
             description: "Fetch a rows from the dataframe",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr fetch 2",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_int(6), Value::test_int(4)],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_int(2), Value::test_int(2)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_int(6), Value::test_int(4)],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(2)],
+                        ),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs

@@ -38,16 +38,19 @@ impl Command for LazyFillNA {
                 description: "Fills the NaN values with 0",
                 example: "[1 2 NaN 3 NaN] | dfr into-df | dfr fill-nan 0",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "0".to_string(),
-                        vec![
-                            Value::test_int(1),
-                            Value::test_int(2),
-                            Value::test_int(0),
-                            Value::test_int(3),
-                            Value::test_int(0),
-                        ],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "0".to_string(),
+                            vec![
+                                Value::test_int(1),
+                                Value::test_int(2),
+                                Value::test_int(0),
+                                Value::test_int(3),
+                                Value::test_int(0),
+                            ],
+                        )],
+                        None,
+                    )
                     .expect("Df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -56,16 +59,19 @@ impl Command for LazyFillNA {
                 description: "Fills the NaN values of a whole dataframe",
                 example: "[[a b]; [0.2 1] [0.1 NaN]] | dfr into-df | dfr fill-nan 0",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_float(0.2), Value::test_float(0.1)],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![Value::test_int(1), Value::test_int(0)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_float(0.2), Value::test_float(0.1)],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![Value::test_int(1), Value::test_int(0)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("Df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -123,7 +129,7 @@ impl Command for LazyFillNA {
                 })
                 .collect::<Vec<Column>>();
             Ok(PipelineData::Value(
-                NuDataFrame::try_from_columns(dataframe)?.into_value(call.head),
+                NuDataFrame::try_from_columns(dataframe, None)?.into_value(call.head),
                 None,
             ))
         }

crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs

@@ -37,16 +37,19 @@ impl Command for LazyFillNull {
             description: "Fills the null values by 0",
             example: "[1 2 2 3 3] | dfr into-df | dfr shift 2 | dfr fill-null 0",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![
-                        Value::test_int(0),
-                        Value::test_int(0),
-                        Value::test_int(1),
-                        Value::test_int(2),
-                        Value::test_int(2),
-                    ],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![
+                            Value::test_int(0),
+                            Value::test_int(0),
+                            Value::test_int(1),
+                            Value::test_int(2),
+                            Value::test_int(2),
+                        ],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/lazy/filter.rs

@@ -38,16 +38,19 @@ impl Command for LazyFilter {
             description: "Filter dataframe using an expression",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr filter ((dfr col a) >= 4)",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_int(6), Value::test_int(4)],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_int(2), Value::test_int(2)],
-                    ),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new(
+                            "a".to_string(),
+                            vec![Value::test_int(6), Value::test_int(4)],
+                        ),
+                        Column::new(
+                            "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(2)],
+                        ),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/lazy/flatten.rs

@@ -71,7 +71,7 @@ Example {
                             Value::test_string("Skiing"),
                             Value::test_string("Football"),
                         ]),
-                   ]).expect("simple df for test should not fail")
+                   ], None).expect("simple df for test should not fail")
                    .into_value(Span::test_data()),
                     )
             },
@@ -88,7 +88,7 @@ Example {
                             Value::test_string("Skiing"),
                             Value::test_string("Football"),
                         ]),
-                   ]).expect("simple df for test should not fail")
+                   ], None).expect("simple df for test should not fail")
                    .into_value(Span::test_data()),
                     ),
             },

crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs

@@ -46,24 +46,27 @@ impl Command for ToLazyGroupBy {
         (dfr col b | dfr sum | dfr as "b_sum")
      ]"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(2)],
-                        ),
-                        Column::new(
-                            "b_min".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "b_max".to_string(),
-                            vec![Value::test_int(4), Value::test_int(6)],
-                        ),
-                        Column::new(
-                            "b_sum".to_string(),
-                            vec![Value::test_int(6), Value::test_int(10)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(1), Value::test_int(2)],
+                            ),
+                            Column::new(
+                                "b_min".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "b_max".to_string(),
+                                vec![Value::test_int(4), Value::test_int(6)],
+                            ),
+                            Column::new(
+                                "b_sum".to_string(),
+                                vec![Value::test_int(6), Value::test_int(10)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -80,24 +83,27 @@ impl Command for ToLazyGroupBy {
      ]
     | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![Value::test_int(1), Value::test_int(2)],
-                        ),
-                        Column::new(
-                            "b_min".to_string(),
-                            vec![Value::test_int(2), Value::test_int(4)],
-                        ),
-                        Column::new(
-                            "b_max".to_string(),
-                            vec![Value::test_int(4), Value::test_int(6)],
-                        ),
-                        Column::new(
-                            "b_sum".to_string(),
-                            vec![Value::test_int(6), Value::test_int(10)],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![Value::test_int(1), Value::test_int(2)],
+                            ),
+                            Column::new(
+                                "b_min".to_string(),
+                                vec![Value::test_int(2), Value::test_int(4)],
+                            ),
+                            Column::new(
+                                "b_max".to_string(),
+                                vec![Value::test_int(4), Value::test_int(6)],
+                            ),
+                            Column::new(
+                                "b_sum".to_string(),
+                                vec![Value::test_int(6), Value::test_int(10)],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs

@@ -53,53 +53,56 @@ impl Command for LazyJoin {
     let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | dfr into-lazy);
     $df_a | dfr join $df_b a foo | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![
-                                Value::test_int(1),
-                                Value::test_int(2),
-                                Value::test_int(1),
-                                Value::test_int(1),
-                            ],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![
-                                Value::test_string("a"),
-                                Value::test_string("b"),
-                                Value::test_string("c"),
-                                Value::test_string("c"),
-                            ],
-                        ),
-                        Column::new(
-                            "c".to_string(),
-                            vec![
-                                Value::test_int(0),
-                                Value::test_int(1),
-                                Value::test_int(2),
-                                Value::test_int(3),
-                            ],
-                        ),
-                        Column::new(
-                            "bar".to_string(),
-                            vec![
-                                Value::test_string("a"),
-                                Value::test_string("c"),
-                                Value::test_string("a"),
-                                Value::test_string("a"),
-                            ],
-                        ),
-                        Column::new(
-                            "ham".to_string(),
-                            vec![
-                                Value::test_string("let"),
-                                Value::test_string("var"),
-                                Value::test_string("let"),
-                                Value::test_string("let"),
-                            ],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![
+                                    Value::test_int(1),
+                                    Value::test_int(2),
+                                    Value::test_int(1),
+                                    Value::test_int(1),
+                                ],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![
+                                    Value::test_string("a"),
+                                    Value::test_string("b"),
+                                    Value::test_string("c"),
+                                    Value::test_string("c"),
+                                ],
+                            ),
+                            Column::new(
+                                "c".to_string(),
+                                vec![
+                                    Value::test_int(0),
+                                    Value::test_int(1),
+                                    Value::test_int(2),
+                                    Value::test_int(3),
+                                ],
+                            ),
+                            Column::new(
+                                "bar".to_string(),
+                                vec![
+                                    Value::test_string("a"),
+                                    Value::test_string("c"),
+                                    Value::test_string("a"),
+                                    Value::test_string("a"),
+                                ],
+                            ),
+                            Column::new(
+                                "ham".to_string(),
+                                vec![
+                                    Value::test_string("let"),
+                                    Value::test_string("var"),
+                                    Value::test_string("let"),
+                                    Value::test_string("let"),
+                                ],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -110,53 +113,56 @@ impl Command for LazyJoin {
     let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | dfr into-lazy);
     $df_a | dfr join $df_b a foo"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new(
-                            "a".to_string(),
-                            vec![
-                                Value::test_int(1),
-                                Value::test_int(2),
-                                Value::test_int(1),
-                                Value::test_int(1),
-                            ],
-                        ),
-                        Column::new(
-                            "b".to_string(),
-                            vec![
-                                Value::test_string("a"),
-                                Value::test_string("b"),
-                                Value::test_string("c"),
-                                Value::test_string("c"),
-                            ],
-                        ),
-                        Column::new(
-                            "c".to_string(),
-                            vec![
-                                Value::test_int(0),
-                                Value::test_int(1),
-                                Value::test_int(2),
-                                Value::test_int(3),
-                            ],
-                        ),
-                        Column::new(
-                            "bar".to_string(),
-                            vec![
-                                Value::test_string("a"),
-                                Value::test_string("c"),
-                                Value::test_string("a"),
-                                Value::test_string("a"),
-                            ],
-                        ),
-                        Column::new(
-                            "ham".to_string(),
-                            vec![
-                                Value::test_string("let"),
-                                Value::test_string("var"),
-                                Value::test_string("let"),
-                                Value::test_string("let"),
-                            ],
-                        ),
-                    ])
+                    NuDataFrame::try_from_columns(
+                        vec![
+                            Column::new(
+                                "a".to_string(),
+                                vec![
+                                    Value::test_int(1),
+                                    Value::test_int(2),
+                                    Value::test_int(1),
+                                    Value::test_int(1),
+                                ],
+                            ),
+                            Column::new(
+                                "b".to_string(),
+                                vec![
+                                    Value::test_string("a"),
+                                    Value::test_string("b"),
+                                    Value::test_string("c"),
+                                    Value::test_string("c"),
+                                ],
+                            ),
+                            Column::new(
+                                "c".to_string(),
+                                vec![
+                                    Value::test_int(0),
+                                    Value::test_int(1),
+                                    Value::test_int(2),
+                                    Value::test_int(3),
+                                ],
+                            ),
+                            Column::new(
+                                "bar".to_string(),
+                                vec![
+                                    Value::test_string("a"),
+                                    Value::test_string("c"),
+                                    Value::test_string("a"),
+                                    Value::test_string("a"),
+                                ],
+                            ),
+                            Column::new(
+                                "ham".to_string(),
+                                vec![
+                                    Value::test_string("let"),
+                                    Value::test_string("var"),
+                                    Value::test_string("let"),
+                                    Value::test_string("let"),
+                                ],
+                            ),
+                        ],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -171,14 +177,14 @@ impl Command for LazyJoin {
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        let left = call.has_flag("left");
-        let outer = call.has_flag("outer");
-        let cross = call.has_flag("cross");
+        let left = call.has_flag(engine_state, stack, "left")?;
+        let outer = call.has_flag(engine_state, stack, "outer")?;
+        let cross = call.has_flag(engine_state, stack, "cross")?;
 
         let how = if left {
             JoinType::Left
         } else if outer {
-            JoinType::Outer
+            JoinType::Outer { coalesce: true }
         } else if cross {
             JoinType::Cross
         } else {

crates/nu-cmd-dataframe/src/dataframe/lazy/macro_commands.rs

@@ -112,6 +112,70 @@ macro_rules! lazy_command {
             }
         }
     };
+
+    ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident?, $test: ident) => {
+        #[derive(Clone)]
+        pub struct $command;
+
+        impl Command for $command {
+            fn name(&self) -> &str {
+                $name
+            }
+
+            fn usage(&self) -> &str {
+                $desc
+            }
+
+            fn signature(&self) -> Signature {
+                Signature::build(self.name())
+                    .input_output_type(
+                        Type::Custom("dataframe".into()),
+                        Type::Custom("dataframe".into()),
+                    )
+                    .category(Category::Custom("lazyframe".into()))
+            }
+
+            fn examples(&self) -> Vec<Example> {
+                $examples
+            }
+
+            fn run(
+                &self,
+                _engine_state: &EngineState,
+                _stack: &mut Stack,
+                call: &Call,
+                input: PipelineData,
+            ) -> Result<PipelineData, ShellError> {
+                let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
+
+                let lazy = NuLazyFrame::new(
+                    lazy.from_eager,
+                    lazy.into_polars()
+                        .$func()
+                        .map_err(|e| ShellError::GenericError {
+                            error: "Dataframe Error".into(),
+                            msg: e.to_string(),
+                            help: None,
+                            span: None,
+                            inner: vec![],
+                        })?,
+                );
+
+                Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
+            }
+        }
+
+        #[cfg(test)]
+        mod $test {
+            use super::super::super::test_dataframe::test_dataframe;
+            use super::*;
+
+            #[test]
+            fn test_examples() {
+                test_dataframe(vec![Box::new($command {})])
+            }
+        }
+    };
 }
 
 // LazyReverse command
@@ -124,16 +188,19 @@ lazy_command!(
         description: "Reverses the dataframe.",
         example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr reverse",
         result: Some(
-            NuDataFrame::try_from_columns(vec![
-                Column::new(
-                    "a".to_string(),
-                    vec![Value::test_int(2), Value::test_int(4), Value::test_int(6),],
-                ),
-                Column::new(
-                    "b".to_string(),
-                    vec![Value::test_int(2), Value::test_int(2), Value::test_int(2),],
-                ),
-            ])
+            NuDataFrame::try_from_columns(
+                vec![
+                    Column::new(
+                        "a".to_string(),
+                        vec![Value::test_int(2), Value::test_int(4), Value::test_int(6),],
+                    ),
+                    Column::new(
+                        "b".to_string(),
+                        vec![Value::test_int(2), Value::test_int(2), Value::test_int(2),],
+                    ),
+                ],
+                None
+            )
             .expect("simple df for test should not fail")
             .into_value(Span::test_data()),
         ),
@@ -167,14 +234,17 @@ lazy_command!(
         description: "Median value from columns in a dataframe",
         example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr median",
         result: Some(
-            NuDataFrame::try_from_columns(vec![
-                Column::new("a".to_string(), vec![Value::test_float(4.0)],),
-                Column::new("b".to_string(), vec![Value::test_float(2.0)],),
-            ])
+            NuDataFrame::try_from_columns(
+                vec![
+                    Column::new("a".to_string(), vec![Value::test_float(4.0)],),
+                    Column::new("b".to_string(), vec![Value::test_float(2.0)],),
+                ],
+                None
+            )
             .expect("simple df for test should not fail")
             .into_value(Span::test_data()),
         ),
     },],
-    median,
+    median?,
     test_median
 );

crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs

@@ -38,10 +38,13 @@ impl Command for LazyQuantile {
             description: "quantile value from columns in a dataframe",
             example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr quantile 0.5",
             result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new("a".to_string(), vec![Value::test_float(4.0)]),
-                    Column::new("b".to_string(), vec![Value::test_float(2.0)]),
-                ])
+                NuDataFrame::try_from_columns(
+                    vec![
+                        Column::new("a".to_string(), vec![Value::test_float(4.0)]),
+                        Column::new("b".to_string(), vec![Value::test_float(2.0)]),
+                    ],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -62,7 +65,14 @@ impl Command for LazyQuantile {
         let lazy = NuLazyFrame::new(
             lazy.from_eager,
             lazy.into_polars()
-                .quantile(lit(quantile), QuantileInterpolOptions::default()),
+                .quantile(lit(quantile), QuantileInterpolOptions::default())
+                .map_err(|e| ShellError::GenericError {
+                    error: "Dataframe Error".into(),
+                    msg: e.to_string(),
+                    help: None,
+                    span: None,
+                    inner: vec![],
+                })?,
         );
 
         Ok(PipelineData::Value(lazy.into_value(call.head)?, None))

crates/nu-cmd-dataframe/src/dataframe/lazy/select.rs

@@ -37,10 +37,13 @@ impl Command for LazySelect {
             description: "Select a column from the dataframe",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr select a",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "a".to_string(),
-                    vec![Value::test_int(6), Value::test_int(4), Value::test_int(2)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "a".to_string(),
+                        vec![Value::test_int(6), Value::test_int(4), Value::test_int(2)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs

@@ -60,7 +60,7 @@ impl Command for LazySortBy {
                             "b".to_string(),
                             vec![Value::test_int(4), Value::test_int(1), Value::test_int(2)],
                         ),
-                    ])
+                    ], None)
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -89,7 +89,7 @@ impl Command for LazySortBy {
                                 Value::test_int(2),
                             ],
                         ),
-                    ])
+                    ], None)
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -107,8 +107,8 @@ impl Command for LazySortBy {
         let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
         let value = Value::list(vals, call.head);
         let expressions = NuExpression::extract_exprs(value)?;
-        let nulls_last = call.has_flag("nulls-last");
-        let maintain_order = call.has_flag("maintain-order");
+        let nulls_last = call.has_flag(engine_state, stack, "nulls-last")?;
+        let maintain_order = call.has_flag(engine_state, stack, "maintain-order")?;
 
         let reverse: Option<Vec<bool>> = call.get_flag(engine_state, stack, "reverse")?;
         let reverse = match reverse {

crates/nu-cmd-dataframe/src/dataframe/lazy/to_lazy.rs

@@ -1,9 +1,12 @@
+use crate::dataframe::values::NuSchema;
+
 use super::super::values::{NuDataFrame, NuLazyFrame};
 
+use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Category, Example, PipelineData, ShellError, Signature, Type, Value,
+    Category, Example, PipelineData, ShellError, Signature, SyntaxShape, Type, Value,
 };
 
 #[derive(Clone)]
@@ -20,6 +23,12 @@ impl Command for ToLazyFrame {
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
+            .named(
+                "schema",
+                SyntaxShape::Record(vec![]),
+                r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
+                Some('s'),
+            )
             .input_output_type(Type::Any, Type::Custom("dataframe".into()))
             .category(Category::Custom("lazyframe".into()))
     }
@@ -34,12 +43,17 @@ impl Command for ToLazyFrame {
 
     fn run(
         &self,
-        _engine_state: &EngineState,
-        _stack: &mut Stack,
+        engine_state: &EngineState,
+        stack: &mut Stack,
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        let df = NuDataFrame::try_from_iter(input.into_iter())?;
+        let maybe_schema = call
+            .get_flag(engine_state, stack, "schema")?
+            .map(|schema| NuSchema::try_from(&schema))
+            .transpose()?;
+
+        let df = NuDataFrame::try_from_iter(input.into_iter(), maybe_schema)?;
         let lazy = NuLazyFrame::from_dataframe(df);
         let value = Value::custom_value(Box::new(lazy), call.head);
 

crates/nu-cmd-dataframe/src/dataframe/series/all_false.rs

@@ -33,10 +33,13 @@ impl Command for AllFalse {
                 description: "Returns true if all values are false",
                 example: "[false false false] | dfr into-df | dfr all-false",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "all_false".to_string(),
-                        vec![Value::test_bool(true)],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "all_false".to_string(),
+                            vec![Value::test_bool(true)],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -47,10 +50,13 @@ impl Command for AllFalse {
     let res = ($s > 9);
     $res | dfr all-false"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "all_false".to_string(),
-                        vec![Value::test_bool(false)],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "all_false".to_string(),
+                            vec![Value::test_bool(false)],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -88,8 +94,11 @@ fn command(
 
     let value = Value::bool(!bool.any(), call.head);
 
-    NuDataFrame::try_from_columns(vec![Column::new("all_false".to_string(), vec![value])])
-        .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
+    NuDataFrame::try_from_columns(
+        vec![Column::new("all_false".to_string(), vec![value])],
+        None,
+    )
+    .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
 }
 
 #[cfg(test)]

crates/nu-cmd-dataframe/src/dataframe/series/all_true.rs

@@ -33,10 +33,13 @@ impl Command for AllTrue {
                 description: "Returns true if all values are true",
                 example: "[true true true] | dfr into-df | dfr all-true",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "all_true".to_string(),
-                        vec![Value::test_bool(true)],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "all_true".to_string(),
+                            vec![Value::test_bool(true)],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -47,10 +50,13 @@ impl Command for AllTrue {
     let res = ($s > 9);
     $res | dfr all-true"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "all_true".to_string(),
-                        vec![Value::test_bool(false)],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "all_true".to_string(),
+                            vec![Value::test_bool(false)],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -88,7 +94,7 @@ fn command(
 
     let value = Value::bool(bool.all(), call.head);
 
-    NuDataFrame::try_from_columns(vec![Column::new("all_true".to_string(), vec![value])])
+    NuDataFrame::try_from_columns(vec![Column::new("all_true".to_string(), vec![value])], None)
         .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
 }
 

crates/nu-cmd-dataframe/src/dataframe/series/arg_max.rs

@@ -37,10 +37,10 @@ impl Command for ArgMax {
             description: "Returns index for max value",
             example: "[1 3 2] | dfr into-df | dfr arg-max",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "arg_max".to_string(),
-                    vec![Value::test_int(1)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new("arg_max".to_string(), vec![Value::test_int(1)])],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/arg_min.rs

@@ -37,10 +37,10 @@ impl Command for ArgMin {
             description: "Returns index for min value",
             example: "[1 3 2] | dfr into-df | dfr arg-min",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "arg_min".to_string(),
-                    vec![Value::test_int(0)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new("arg_min".to_string(), vec![Value::test_int(0)])],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/cumulative.rs

@@ -69,16 +69,19 @@ impl Command for Cumulative {
             description: "Cumulative sum for a series",
             example: "[1 2 3 4 5] | dfr into-df | dfr cumulative sum",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0_cumulative_sum".to_string(),
-                    vec![
-                        Value::test_int(1),
-                        Value::test_int(3),
-                        Value::test_int(6),
-                        Value::test_int(10),
-                        Value::test_int(15),
-                    ],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0_cumulative_sum".to_string(),
+                        vec![
+                            Value::test_int(1),
+                            Value::test_int(3),
+                            Value::test_int(6),
+                            Value::test_int(10),
+                            Value::test_int(15),
+                        ],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -103,12 +106,12 @@ fn command(
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
     let cum_type: Spanned<String> = call.req(engine_state, stack, 0)?;
-    let reverse = call.has_flag("reverse");
+    let reverse = call.has_flag(engine_state, stack, "reverse")?;
 
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
     let series = df.as_series(call.head)?;
 
-    if let DataType::Object(_) = series.dtype() {
+    if let DataType::Object(..) = series.dtype() {
         return Err(ShellError::GenericError {
             error: "Found object series".into(),
             msg: "Series of type object cannot be used for cumulative operation".into(),

crates/nu-cmd-dataframe/src/dataframe/series/date/as_date.rs

@@ -6,7 +6,7 @@ use nu_protocol::{
     engine::{Command, EngineState, Stack},
     Category, Example, PipelineData, ShellError, Signature, SyntaxShape, Type,
 };
-use polars::prelude::{IntoSeries, Utf8Methods};
+use polars::prelude::{IntoSeries, StringMethods};
 
 #[derive(Clone)]
 pub struct AsDate;
@@ -64,11 +64,11 @@ fn command(
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
     let format: String = call.req(engine_state, stack, 0)?;
-    let not_exact = call.has_flag("not-exact");
+    let not_exact = call.has_flag(engine_state, stack, "not-exact")?;
 
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
     let series = df.as_series(call.head)?;
-    let casted = series.utf8().map_err(|e| ShellError::GenericError {
+    let casted = series.str().map_err(|e| ShellError::GenericError {
         error: "Error casting to string".into(),
         msg: e.to_string(),
         span: Some(call.head),

crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs

@@ -7,7 +7,7 @@ use nu_protocol::{
     engine::{Command, EngineState, Stack},
     Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
 };
-use polars::prelude::{IntoSeries, TimeUnit, Utf8Methods};
+use polars::prelude::{IntoSeries, StringMethods, TimeUnit};
 
 #[derive(Clone)]
 pub struct AsDateTime;
@@ -53,27 +53,30 @@ impl Command for AsDateTime {
                 description: "Converts string to datetime",
                 example: r#"["2021-12-30 00:00:00" "2021-12-31 00:00:00"] | dfr into-df | dfr as-datetime "%Y-%m-%d %H:%M:%S""#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "datetime".to_string(),
-                        vec![
-                            Value::date(
-                                DateTime::parse_from_str(
-                                    "2021-12-30 00:00:00 +0000",
-                                    "%Y-%m-%d %H:%M:%S %z",
-                                )
-                                .expect("date calculation should not fail in test"),
-                                Span::test_data(),
-                            ),
-                            Value::date(
-                                DateTime::parse_from_str(
-                                    "2021-12-31 00:00:00 +0000",
-                                    "%Y-%m-%d %H:%M:%S %z",
-                                )
-                                .expect("date calculation should not fail in test"),
-                                Span::test_data(),
-                            ),
-                        ],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "datetime".to_string(),
+                            vec![
+                                Value::date(
+                                    DateTime::parse_from_str(
+                                        "2021-12-30 00:00:00 +0000",
+                                        "%Y-%m-%d %H:%M:%S %z",
+                                    )
+                                    .expect("date calculation should not fail in test"),
+                                    Span::test_data(),
+                                ),
+                                Value::date(
+                                    DateTime::parse_from_str(
+                                        "2021-12-31 00:00:00 +0000",
+                                        "%Y-%m-%d %H:%M:%S %z",
+                                    )
+                                    .expect("date calculation should not fail in test"),
+                                    Span::test_data(),
+                                ),
+                            ],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -82,27 +85,30 @@ impl Command for AsDateTime {
                 description: "Converts string to datetime with high resolutions",
                 example: r#"["2021-12-30 00:00:00.123456789" "2021-12-31 00:00:00.123456789"] | dfr into-df | dfr as-datetime "%Y-%m-%d %H:%M:%S.%9f""#,
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "datetime".to_string(),
-                        vec![
-                            Value::date(
-                                DateTime::parse_from_str(
-                                    "2021-12-30 00:00:00.123456789 +0000",
-                                    "%Y-%m-%d %H:%M:%S.%9f %z",
-                                )
-                                .expect("date calculation should not fail in test"),
-                                Span::test_data(),
-                            ),
-                            Value::date(
-                                DateTime::parse_from_str(
-                                    "2021-12-31 00:00:00.123456789 +0000",
-                                    "%Y-%m-%d %H:%M:%S.%9f %z",
-                                )
-                                .expect("date calculation should not fail in test"),
-                                Span::test_data(),
-                            ),
-                        ],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "datetime".to_string(),
+                            vec![
+                                Value::date(
+                                    DateTime::parse_from_str(
+                                        "2021-12-30 00:00:00.123456789 +0000",
+                                        "%Y-%m-%d %H:%M:%S.%9f %z",
+                                    )
+                                    .expect("date calculation should not fail in test"),
+                                    Span::test_data(),
+                                ),
+                                Value::date(
+                                    DateTime::parse_from_str(
+                                        "2021-12-31 00:00:00.123456789 +0000",
+                                        "%Y-%m-%d %H:%M:%S.%9f %z",
+                                    )
+                                    .expect("date calculation should not fail in test"),
+                                    Span::test_data(),
+                                ),
+                            ],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -128,11 +134,11 @@ fn command(
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
     let format: String = call.req(engine_state, stack, 0)?;
-    let not_exact = call.has_flag("not-exact");
+    let not_exact = call.has_flag(engine_state, stack, "not-exact")?;
 
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
     let series = df.as_series(call.head)?;
-    let casted = series.utf8().map_err(|e| ShellError::GenericError {
+    let casted = series.str().map_err(|e| ShellError::GenericError {
         error: "Error casting to string".into(),
         msg: e.to_string(),
         span: Some(call.head),

crates/nu-cmd-dataframe/src/dataframe/series/date/get_day.rs

@@ -35,10 +35,13 @@ impl Command for GetDay {
     let df = ([$dt $dt] | dfr into-df);
     $df | dfr get-day"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![Value::test_int(4), Value::test_int(4)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![Value::test_int(4), Value::test_int(4)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/date/get_hour.rs

@@ -35,10 +35,13 @@ impl Command for GetHour {
     let df = ([$dt $dt] | dfr into-df);
     $df | dfr get-hour"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![Value::test_int(16), Value::test_int(16)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![Value::test_int(16), Value::test_int(16)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/date/get_minute.rs

@@ -35,10 +35,13 @@ impl Command for GetMinute {
     let df = ([$dt $dt] | dfr into-df);
     $df | dfr get-minute"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![Value::test_int(39), Value::test_int(39)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![Value::test_int(39), Value::test_int(39)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/date/get_month.rs

@@ -35,10 +35,13 @@ impl Command for GetMonth {
     let df = ([$dt $dt] | dfr into-df);
     $df | dfr get-month"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![Value::test_int(8), Value::test_int(8)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![Value::test_int(8), Value::test_int(8)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/date/get_nanosecond.rs

@@ -35,10 +35,13 @@ impl Command for GetNanosecond {
     let df = ([$dt $dt] | dfr into-df);
     $df | dfr get-nanosecond"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![Value::test_int(0), Value::test_int(0)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![Value::test_int(0), Value::test_int(0)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/date/get_ordinal.rs

@@ -35,10 +35,13 @@ impl Command for GetOrdinal {
     let df = ([$dt $dt] | dfr into-df);
     $df | dfr get-ordinal"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![Value::test_int(217), Value::test_int(217)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![Value::test_int(217), Value::test_int(217)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/date/get_second.rs

@@ -35,10 +35,13 @@ impl Command for GetSecond {
     let df = ([$dt $dt] | dfr into-df);
     $df | dfr get-second"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![Value::test_int(18), Value::test_int(18)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![Value::test_int(18), Value::test_int(18)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/date/get_week.rs

@@ -35,10 +35,13 @@ impl Command for GetWeek {
     let df = ([$dt $dt] | dfr into-df);
     $df | dfr get-week"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![Value::test_int(32), Value::test_int(32)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![Value::test_int(32), Value::test_int(32)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/date/get_weekday.rs

@@ -35,10 +35,13 @@ impl Command for GetWeekDay {
     let df = ([$dt $dt] | dfr into-df);
     $df | dfr get-weekday"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![Value::test_int(2), Value::test_int(2)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![Value::test_int(2), Value::test_int(2)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/date/get_year.rs

@@ -35,10 +35,13 @@ impl Command for GetYear {
     let df = ([$dt $dt] | dfr into-df);
     $df | dfr get-year"#,
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "0".to_string(),
-                    vec![Value::test_int(2020), Value::test_int(2020)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "0".to_string(),
+                        vec![Value::test_int(2020), Value::test_int(2020)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs

@@ -1,5 +1,6 @@
 use super::super::super::values::{Column, NuDataFrame};
 
+use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
@@ -45,16 +46,19 @@ impl Command for ArgSort {
                 description: "Returns indexes for a sorted series",
                 example: "[1 2 2 3 3] | dfr into-df | dfr arg-sort",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "arg_sort".to_string(),
-                        vec![
-                            Value::test_int(0),
-                            Value::test_int(1),
-                            Value::test_int(2),
-                            Value::test_int(3),
-                            Value::test_int(4),
-                        ],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "arg_sort".to_string(),
+                            vec![
+                                Value::test_int(0),
+                                Value::test_int(1),
+                                Value::test_int(2),
+                                Value::test_int(3),
+                                Value::test_int(4),
+                            ],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -63,16 +67,19 @@ impl Command for ArgSort {
                 description: "Returns indexes for a sorted series",
                 example: "[1 2 2 3 3] | dfr into-df | dfr arg-sort --reverse",
                 result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "arg_sort".to_string(),
-                        vec![
-                            Value::test_int(3),
-                            Value::test_int(4),
-                            Value::test_int(1),
-                            Value::test_int(2),
-                            Value::test_int(0),
-                        ],
-                    )])
+                    NuDataFrame::try_from_columns(
+                        vec![Column::new(
+                            "arg_sort".to_string(),
+                            vec![
+                                Value::test_int(3),
+                                Value::test_int(4),
+                                Value::test_int(1),
+                                Value::test_int(2),
+                                Value::test_int(0),
+                            ],
+                        )],
+                        None,
+                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -92,18 +99,18 @@ impl Command for ArgSort {
 }
 
 fn command(
-    _engine_state: &EngineState,
-    _stack: &mut Stack,
+    engine_state: &EngineState,
+    stack: &mut Stack,
     call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     let sort_options = SortOptions {
-        descending: call.has_flag("reverse"),
-        nulls_last: call.has_flag("nulls-last"),
+        descending: call.has_flag(engine_state, stack, "reverse")?,
+        nulls_last: call.has_flag(engine_state, stack, "nulls-last")?,
         multithreaded: true,
-        maintain_order: call.has_flag("maintain-order"),
+        maintain_order: call.has_flag(engine_state, stack, "maintain-order")?,
     };
 
     let mut res = df

crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs

@@ -37,10 +37,13 @@ impl Command for ArgTrue {
             description: "Returns indexes where values are true",
             example: "[false true false] | dfr into-df | dfr arg-true",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "arg_true".to_string(),
-                    vec![Value::test_int(1)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "arg_true".to_string(),
+                        vec![Value::test_int(1)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_unique.rs

@@ -37,10 +37,13 @@ impl Command for ArgUnique {
             description: "Returns indexes for unique values",
             example: "[1 2 2 3 3] | dfr into-df | dfr arg-unique",
             result: Some(
-                NuDataFrame::try_from_columns(vec![Column::new(
-                    "arg_unique".to_string(),
-                    vec![Value::test_int(0), Value::test_int(1), Value::test_int(3)],
-                )])
+                NuDataFrame::try_from_columns(
+                    vec![Column::new(
+                        "arg_unique".to_string(),
+                        vec![Value::test_int(0), Value::test_int(1), Value::test_int(3)],
+                    )],
+                    None,
+                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),