Revert "Add format meta command (#11334)"

This reverts commit fd77114d82.

.github/.typos.toml

@@ -12,4 +12,3 @@ IIF = "IIF"
 numer = "numer"
 ratatui = "ratatui"
 doas = "doas"
-wheres = "wheres"

.github/workflows/ci.yml

@@ -41,7 +41,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
         with:
           rustflags: ""
 
@@ -63,10 +63,6 @@ jobs:
         platform: [windows-latest, macos-latest, ubuntu-20.04]
         feature: [default, dataframe, extra]
         include:
-          # linux CI cannot handle clipboard feature
-          - default-flags: ""
-          - platform:  ubuntu-20.04
-            default-flags: "--no-default-features --features=default-no-clipboard"
           - feature: default
             flags: ""
           - feature: dataframe
@@ -89,23 +85,12 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
         with:
           rustflags: ""
 
       - name: Tests
-        run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} ${{ matrix.flags }}
+        run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.flags }}
-
-      - name: Check for clean repo
-        shell: bash
-        run: |
-          if [ -n "$(git status --porcelain)" ]; then
-            echo "there are changes";
-            git status --porcelain
-            exit 1
-          else
-            echo "no changes in working directory";
-          fi
 
   std-lib-and-python-virtualenv:
     strategy:
@@ -121,7 +106,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
         with:
           rustflags: ""
 
@@ -129,7 +114,7 @@ jobs:
         run: cargo install --path . --locked --no-default-features
 
       - name: Standard library tests
-        run: nu -c 'use crates/nu-std/testing.nu; testing run-tests --path crates/nu-std'
+        run: nu -c 'use std testing; testing run-tests --path crates/nu-std'
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -144,17 +129,6 @@ jobs:
         run: nu scripts/test_virtualenv.nu
         shell: bash
 
-      - name: Check for clean repo
-        shell: bash
-        run: |
-          if [ -n "$(git status --porcelain)" ]; then
-            echo "there are changes";
-            git status --porcelain
-            exit 1
-          else
-            echo "no changes in working directory";
-          fi
-
   plugins:
     strategy:
       fail-fast: true
@@ -167,7 +141,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain and cache
-        uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+        uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
         with:
           rustflags: ""
 
@@ -176,14 +150,3 @@ jobs:
 
       - name: Tests
         run: cargo test --profile ci --package nu_plugin_*
-
-      - name: Check for clean repo
-        shell: bash
-        run: |
-          if [ -n "$(git status --porcelain)" ]; then
-            echo "there are changes";
-            git status --porcelain
-            exit 1
-          else
-            echo "no changes in working directory";
-          fi

.github/workflows/nightly-build.yml

@@ -36,10 +36,12 @@ jobs:
           token: ${{ secrets.WORKFLOW_TOKEN }}
 
       - name: Setup Nushell
-        uses: hustcer/setup-nu@v3.9
+        uses: hustcer/setup-nu@v3.8
         if: github.repository == 'nushell/nightly'
         with:
-          version: 0.90.1
+          version: 0.86.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       # Synchronize the main branch of nightly repo with the main branch of Nushell official repo
       - name: Prepare for Nightly Release
@@ -117,7 +119,7 @@ jobs:
           os: ubuntu-20.04
           target_rustflags: ''
         - target: riscv64gc-unknown-linux-gnu
-          os: ubuntu-latest
+          os: ubuntu-20.04
           target_rustflags: ''
 
     runs-on: ${{matrix.os}}
@@ -133,15 +135,17 @@ jobs:
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
       # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
         rustflags: ''
 
     - name: Setup Nushell
-      uses: hustcer/setup-nu@v3.9
+      uses: hustcer/setup-nu@v3.8
       with:
-        version: 0.90.1
+        version: 0.86.0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Release Nu Binary
       id: nu
@@ -156,7 +160,7 @@ jobs:
 
     - name: Create an Issue for Release Failure
       if: ${{ failure() }}
-      uses: JasonEtco/create-an-issue@v2.9.2
+      uses: JasonEtco/create-an-issue@v2.9.1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -245,15 +249,17 @@ jobs:
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
       # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
         rustflags: ''
 
     - name: Setup Nushell
-      uses: hustcer/setup-nu@v3.9
+      uses: hustcer/setup-nu@v3.8
       with:
-        version: 0.90.1
+        version: 0.86.0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Release Nu Binary
       id: nu
@@ -268,7 +274,7 @@ jobs:
 
     - name: Create an Issue for Release Failure
       if: ${{ failure() }}
-      uses: JasonEtco/create-an-issue@v2.9.2
+      uses: JasonEtco/create-an-issue@v2.9.1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
@@ -315,9 +321,11 @@ jobs:
           ref: main
 
       - name: Setup Nushell
-        uses: hustcer/setup-nu@v3.9
+        uses: hustcer/setup-nu@v3.8
         with:
-          version: 0.90.1
+          version: 0.86.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       # Keep the last a few releases
       - name: Delete Older Releases

.github/workflows/release-pkg.nu

@@ -82,8 +82,8 @@ print $'Start building ($bin)...'; hr-line
 # ----------------------------------------------------------------------------
 # Build for Ubuntu and macOS
 # ----------------------------------------------------------------------------
-if $os in [$USE_UBUNTU, 'macos-latest', 'ubuntu-latest'] {
+if $os in [$USE_UBUNTU, 'macos-latest'] {
-    if $os starts-with ubuntu {
+    if $os == $USE_UBUNTU {
         sudo apt update
         sudo apt-get install libxcb-composite0-dev -y
     }
@@ -106,7 +106,7 @@ if $os in [$USE_UBUNTU, 'macos-latest', 'ubuntu-latest'] {
         _ => {
             # musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?'
             # Actually just for x86_64-unknown-linux-musl target
-            if $os starts-with ubuntu { sudo apt install musl-tools -y }
+            if $os == $USE_UBUNTU { sudo apt install musl-tools -y }
             cargo-build-nu $flags
         }
     }
@@ -153,7 +153,7 @@ if ($ver | str trim | is-empty) {
 # Create a release archive and send it to output for the following steps
 # ----------------------------------------------------------------------------
 cd $dist; print $'(char nl)Creating release archive...'; hr-line
-if $os in [$USE_UBUNTU, 'macos-latest', 'ubuntu-latest'] {
+if $os in [$USE_UBUNTU, 'macos-latest'] {
 
     let files = (ls | get name)
     let dest = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }

.github/workflows/release.yml

@@ -66,7 +66,7 @@ jobs:
           os: ubuntu-20.04
           target_rustflags: ''
         - target: riscv64gc-unknown-linux-gnu
-          os: ubuntu-latest
+          os: ubuntu-20.04
           target_rustflags: ''
 
     runs-on: ${{matrix.os}}
@@ -79,15 +79,17 @@ jobs:
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
       # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
         rustflags: ''
 
     - name: Setup Nushell
-      uses: hustcer/setup-nu@v3.9
+      uses: hustcer/setup-nu@v3.8
       with:
-        version: 0.90.1
+        version: 0.86.0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Release Nu Binary
       id: nu
@@ -168,15 +170,17 @@ jobs:
         echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
 
     - name: Setup Rust toolchain and cache
-      uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
+      uses: actions-rust-lang/setup-rust-toolchain@v1.6.0
       # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec:  Error 0xc0000135`
       with:
         rustflags: ''
 
     - name: Setup Nushell
-      uses: hustcer/setup-nu@v3.9
+      uses: hustcer/setup-nu@v3.8
       with:
-        version: 0.90.1
+        version: 0.86.0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Release Nu Binary
       id: nu

.github/workflows/typos.yml

@@ -10,4 +10,6 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Check spelling
-        uses: crate-ci/typos@v1.19.0
+        uses: crate-ci/typos@v1.16.24
+        with:
+          config: ./.github/.typos.toml

Cargo.toml

@@ -10,8 +10,8 @@ homepage = "https://www.nushell.sh"
 license = "MIT"
 name = "nu"
 repository = "https://github.com/nushell/nushell"
-rust-version = "1.74.1"
+rust-version = "1.72.1"
-version = "0.91.0"
+version = "0.88.2"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -33,52 +33,48 @@ members = [
 	"crates/nu-cmd-lang",
 	"crates/nu-cmd-dataframe",
 	"crates/nu-command",
-	"crates/nu-color-config",
-	"crates/nu-explore",
-	"crates/nu-json",
 	"crates/nu-lsp",
-	"crates/nu-pretty-hex",
 	"crates/nu-protocol",
 	"crates/nu-plugin",
 	"crates/nu_plugin_inc",
 	"crates/nu_plugin_gstat",
 	"crates/nu_plugin_example",
-	"crates/nu_plugin_stream_example",
 	"crates/nu_plugin_query",
 	"crates/nu_plugin_custom_values",
 	"crates/nu_plugin_formats",
 	"crates/nu-std",
-	"crates/nu-table",
-	"crates/nu-term-grid",
-	"crates/nu-test-support",
 	"crates/nu-utils",
 ]
 
 [dependencies]
-nu-cli = { path = "./crates/nu-cli", version = "0.91.0" }
+nu-cli = { path = "./crates/nu-cli", version = "0.88.2" }
-nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.91.0" }
+nu-color-config = { path = "./crates/nu-color-config", version = "0.88.2" }
-nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.91.0" }
+nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.88.2" }
-nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.91.0", features = [
+nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.88.2" }
-	"dataframe",
+nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.88.2", features = ["dataframe"], optional = true }
-], optional = true }
+nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.88.2", optional = true }
-nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.91.0", optional = true }
+nu-command = { path = "./crates/nu-command", version = "0.88.2" }
-nu-command = { path = "./crates/nu-command", version = "0.91.0" }
+nu-engine = { path = "./crates/nu-engine", version = "0.88.2" }
-nu-engine = { path = "./crates/nu-engine", version = "0.91.0" }
+nu-explore = { path = "./crates/nu-explore", version = "0.88.2" }
-nu-explore = { path = "./crates/nu-explore", version = "0.91.0" }
+nu-json = { path = "./crates/nu-json", version = "0.88.2" }
-nu-lsp = { path = "./crates/nu-lsp/", version = "0.91.0" }
+nu-lsp = { path = "./crates/nu-lsp/", version = "0.88.2" }
-nu-parser = { path = "./crates/nu-parser", version = "0.91.0" }
+nu-parser = { path = "./crates/nu-parser", version = "0.88.2" }
-nu-path = { path = "./crates/nu-path", version = "0.91.0" }
+nu-path = { path = "./crates/nu-path", version = "0.88.2" }
-nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.91.0" }
+nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.88.2" }
-nu-protocol = { path = "./crates/nu-protocol", version = "0.91.0" }
+nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.88.2" }
-nu-std = { path = "./crates/nu-std", version = "0.91.0" }
+nu-protocol = { path = "./crates/nu-protocol", version = "0.88.2" }
-nu-utils = { path = "./crates/nu-utils", version = "0.91.0" }
+nu-system = { path = "./crates/nu-system", version = "0.88.2" }
-
+nu-table = { path = "./crates/nu-table", version = "0.88.2" }
-reedline = { version = "0.30.0", features = ["bashisms", "sqlite"] }
+nu-term-grid = { path = "./crates/nu-term-grid", version = "0.88.2" }
+nu-std = { path = "./crates/nu-std", version = "0.88.2" }
+nu-utils = { path = "./crates/nu-utils", version = "0.88.2" }
+nu-ansi-term = "0.49.0"
+reedline = { version = "0.27.0", features = ["bashisms", "sqlite"] }
 
 crossterm = "0.27"
 ctrlc = "3.4"
 log = "0.4"
-miette = { version = "7.1", features = ["fancy-no-backtrace", "fancy"] }
+miette = { version = "5.10", features = ["fancy-no-backtrace"] }
 mimalloc = { version = "0.1.37", default-features = false, optional = true }
 serde_json = "1.0"
 simplelog = "0.12"
@@ -87,6 +83,7 @@ time = "0.3"
 [target.'cfg(not(target_os = "windows"))'.dependencies]
 # Our dependencies don't use OpenSSL on Windows
 openssl = { version = "0.10", features = ["vendored"], optional = true }
+signal-hook = { version = "0.3", default-features = false }
 
 [target.'cfg(windows)'.build-dependencies]
 winresource = "0.1"
@@ -100,13 +97,13 @@ nix = { version = "0.27", default-features = false, features = [
 ] }
 
 [dev-dependencies]
-nu-test-support = { path = "./crates/nu-test-support", version = "0.91.0" }
+nu-test-support = { path = "./crates/nu-test-support", version = "0.88.2" }
 assert_cmd = "2.0"
-divan = "0.1.14"
+criterion = "0.5"
 pretty_assertions = "1.4"
 rstest = { version = "0.18", default-features = false }
-serial_test = "3.0"
+serial_test = "2.0"
-tempfile = "3.10"
+tempfile = "3.8"
 
 [features]
 plugin = [
@@ -117,16 +114,7 @@ plugin = [
 	"nu-protocol/plugin",
 	"nu-engine/plugin",
 ]
-default = ["default-no-clipboard", "system-clipboard"]
+default = ["plugin", "which-support", "trash-support", "sqlite", "mimalloc"]
-# Enables convenient omitting of the system-clipboard feature, as it leads to problems in ci on linux
-# See https://github.com/nushell/nushell/pull/11535
-default-no-clipboard = [
-	"plugin",
-	"which-support",
-	"trash-support",
-	"sqlite",
-	"mimalloc",
-]
 stable = ["default"]
 wasi = ["nu-cmd-lang/wasi"]
 # NOTE: individual features are also passed to `nu-cmd-lang` that uses them to generate the feature matrix in the `version` command
@@ -136,7 +124,6 @@ wasi = ["nu-cmd-lang/wasi"]
 static-link-openssl = ["dep:openssl", "nu-cmd-lang/static-link-openssl"]
 
 mimalloc = ["nu-cmd-lang/mimalloc", "dep:mimalloc"]
-system-clipboard = ["reedline/system_clipboard"]
 
 # Stable (Default)
 which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"]
@@ -179,9 +166,11 @@ bench = false
 # To use a development version of a dependency please use a global override here
 # changing versions in each sub-crate of the workspace is tedious
 [patch.crates-io]
-# reedline = { git = "https://github.com/nushell/reedline", branch = "main" }
+# reedline = { git = "https://github.com/nushell/reedline.git", branch = "main" }
 # nu-ansi-term = {git = "https://github.com/nushell/nu-ansi-term.git", branch = "main"}
+# uu_cp = { git = "https://github.com/uutils/coreutils.git", branch = "main" }
 
+# Criterion benchmarking setup
 # Run all benchmarks with `cargo bench`
 # Run individual benchmarks like `cargo bench -- <regex>` e.g. `cargo bench -- parse`
 [[bench]]

benches/README.md

@@ -1,6 +1,6 @@
-# Divan benchmarks
+# Criterion benchmarks
 
-These are benchmarks using [Divan](https://github.com/nvzqz/divan), a microbenchmarking tool for Rust.
+These are benchmarks using [Criterion](https://github.com/bheisler/criterion.rs), a microbenchmarking tool for Rust.
 
 Run all benchmarks with `cargo bench`
 

benches/benchmarks.rs

@@ -1,17 +1,11 @@
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
 use nu_cli::eval_source;
 use nu_parser::parse;
-use nu_plugin::{Encoder, EncodingType, PluginCallResponse, PluginOutput};
+use nu_plugin::{EncodingType, PluginResponse};
-use nu_protocol::{
+use nu_protocol::{engine::EngineState, PipelineData, Span, Value};
-    engine::EngineState, eval_const::create_nu_constant, PipelineData, Span, Value, NU_VARIABLE_ID,
-};
 use nu_utils::{get_default_config, get_default_env};
 use std::path::{Path, PathBuf};
 
-fn main() {
-    // Run registered benchmarks.
-    divan::main();
-}
-
 fn load_bench_commands() -> EngineState {
     nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
 }
@@ -30,12 +24,21 @@ fn canonicalize_path(engine_state: &EngineState, path: &Path) -> PathBuf {
 }
 
 fn get_home_path(engine_state: &EngineState) -> PathBuf {
-    nu_path::home_dir()
+    let home_path = if let Some(path) = nu_path::home_dir() {
-        .map(|path| canonicalize_path(engine_state, &path))
+        let canon_home_path = canonicalize_path(engine_state, &path);
-        .unwrap_or_default()
+        canon_home_path
+    } else {
+        std::path::PathBuf::new()
+    };
+    home_path
 }
 
-fn setup_engine() -> EngineState {
+// FIXME: All benchmarks live in this 1 file to speed up build times when benchmarking.
+// When the *_benchmarks functions were in different files, `cargo bench` would build
+// an executable for every single one - incredibly slowly. Would be nice to figure out
+// a way to split things up again.
+
+fn parser_benchmarks(c: &mut Criterion) {
     let mut engine_state = load_bench_commands();
     let home_path = get_home_path(&engine_state);
 
@@ -45,82 +48,90 @@ fn setup_engine() -> EngineState {
         Value::string(home_path.to_string_lossy(), Span::test_data()),
     );
 
-    let nu_const = create_nu_constant(&engine_state, Span::unknown())
+    let default_env = get_default_env().as_bytes();
-        .expect("Failed to create nushell constant.");
+    c.bench_function("parse_default_env_file", |b| {
-    engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const);
+        b.iter_batched(
+            || nu_protocol::engine::StateWorkingSet::new(&engine_state),
+            |mut working_set| parse(&mut working_set, None, default_env, false),
+            BatchSize::SmallInput,
+        )
+    });
 
-    engine_state
+    let default_config = get_default_config().as_bytes();
+    c.bench_function("parse_default_config_file", |b| {
+        b.iter_batched(
+            || nu_protocol::engine::StateWorkingSet::new(&engine_state),
+            |mut working_set| parse(&mut working_set, None, default_config, false),
+            BatchSize::SmallInput,
+        )
+    });
+
+    c.bench_function("eval default_env.nu", |b| {
+        b.iter(|| {
+            let mut stack = nu_protocol::engine::Stack::new();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_env().as_bytes(),
+                "default_env.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
+
+    c.bench_function("eval default_config.nu", |b| {
+        b.iter(|| {
+            let mut stack = nu_protocol::engine::Stack::new();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_config().as_bytes(),
+                "default_config.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
 }
 
-// FIXME: All benchmarks live in this 1 file to speed up build times when benchmarking.
+fn eval_benchmarks(c: &mut Criterion) {
-// When the *_benchmarks functions were in different files, `cargo bench` would build
+    let mut engine_state = load_bench_commands();
-// an executable for every single one - incredibly slowly. Would be nice to figure out
+    let home_path = get_home_path(&engine_state);
-// a way to split things up again.
 
-#[divan::bench_group()]
+    // parsing config.nu breaks without PWD set, so set a valid path
-mod parser_benchmarks {
+    engine_state.add_env_var(
-    use super::*;
+        "PWD".into(),
+        Value::string(home_path.to_string_lossy(), Span::test_data()),
+    );
 
-    #[divan::bench()]
+    c.bench_function("eval default_env.nu", |b| {
-    fn parse_default_config_file(bencher: divan::Bencher) {
+        b.iter(|| {
-        let engine_state = setup_engine();
+            let mut stack = nu_protocol::engine::Stack::new();
-        let default_env = get_default_config().as_bytes();
+            eval_source(
+                &mut engine_state,
+                &mut stack,
+                get_default_env().as_bytes(),
+                "default_env.nu",
+                PipelineData::empty(),
+                false,
+            )
+        })
+    });
 
-        bencher
+    c.bench_function("eval default_config.nu", |b| {
-            .with_inputs(|| nu_protocol::engine::StateWorkingSet::new(&engine_state))
+        b.iter(|| {
-            .bench_refs(|mut working_set| parse(&mut working_set, None, default_env, false))
+            let mut stack = nu_protocol::engine::Stack::new();
-    }
+            eval_source(
-
+                &mut engine_state,
-    #[divan::bench()]
+                &mut stack,
-    fn parse_default_env_file(bencher: divan::Bencher) {
+                get_default_config().as_bytes(),
-        let engine_state = setup_engine();
+                "default_config.nu",
-        let default_env = get_default_env().as_bytes();
+                PipelineData::empty(),
-
+                false,
-        bencher
+            )
-            .with_inputs(|| nu_protocol::engine::StateWorkingSet::new(&engine_state))
+        })
-            .bench_refs(|mut working_set| parse(&mut working_set, None, default_env, false))
+    });
-    }
-}
-
-#[divan::bench_group()]
-mod eval_benchmarks {
-    use super::*;
-
-    #[divan::bench()]
-    fn eval_default_env(bencher: divan::Bencher) {
-        let default_env = get_default_env().as_bytes();
-        let fname = "default_env.nu";
-        bencher
-            .with_inputs(|| (setup_engine(), nu_protocol::engine::Stack::new()))
-            .bench_values(|(mut engine_state, mut stack)| {
-                eval_source(
-                    &mut engine_state,
-                    &mut stack,
-                    default_env,
-                    fname,
-                    PipelineData::empty(),
-                    false,
-                )
-            })
-    }
-
-    #[divan::bench()]
-    fn eval_default_config(bencher: divan::Bencher) {
-        let default_env = get_default_config().as_bytes();
-        let fname = "default_config.nu";
-        bencher
-            .with_inputs(|| (setup_engine(), nu_protocol::engine::Stack::new()))
-            .bench_values(|(mut engine_state, mut stack)| {
-                eval_source(
-                    &mut engine_state,
-                    &mut stack,
-                    default_env,
-                    fname,
-                    PipelineData::empty(),
-                    false,
-                )
-            })
-    }
 }
 
 // generate a new table data with `row_cnt` rows, `col_cnt` columns.
@@ -134,76 +145,50 @@ fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
     Value::list(vec![record; row_cnt], Span::test_data())
 }
 
-#[divan::bench_group()]
+fn encoding_benchmarks(c: &mut Criterion) {
-mod encoding_benchmarks {
+    let mut group = c.benchmark_group("Encoding");
-    use super::*;
+    let test_cnt_pairs = [(100, 5), (100, 15), (10000, 5), (10000, 15)];
-
+    for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() {
-    #[divan::bench(args = [(100, 5), (10000, 15)])]
+        for fmt in ["json", "msgpack"] {
-    fn json_encode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
+            group.bench_function(&format!("{fmt} encode {row_cnt} * {col_cnt}"), |b| {
-        let test_data = PluginOutput::CallResponse(
+                let mut res = vec![];
-            0,
+                let test_data =
-            PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
+                    PluginResponse::Value(Box::new(encoding_test_data(row_cnt, col_cnt)));
-        );
+                let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap();
-        let encoder = EncodingType::try_from_bytes(b"json").unwrap();
+                b.iter(|| encoder.encode_response(&test_data, &mut res))
-        bencher
+            });
-            .with_inputs(|| (vec![]))
+        }
-            .bench_values(|mut res| encoder.encode(&test_data, &mut res))
-    }
-
-    #[divan::bench(args = [(100, 5), (10000, 15)])]
-    fn msgpack_encode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
-        let test_data = PluginOutput::CallResponse(
-            0,
-            PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
-        );
-        let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
-        bencher
-            .with_inputs(|| (vec![]))
-            .bench_values(|mut res| encoder.encode(&test_data, &mut res))
     }
+    group.finish();
 }
 
-#[divan::bench_group()]
+fn decoding_benchmarks(c: &mut Criterion) {
-mod decoding_benchmarks {
+    let mut group = c.benchmark_group("Decoding");
-    use super::*;
+    let test_cnt_pairs = [(100, 5), (100, 15), (10000, 5), (10000, 15)];
-
+    for (row_cnt, col_cnt) in test_cnt_pairs.into_iter() {
-    #[divan::bench(args = [(100, 5), (10000, 15)])]
+        for fmt in ["json", "msgpack"] {
-    fn json_decode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
+            group.bench_function(&format!("{fmt} decode for {row_cnt} * {col_cnt}"), |b| {
-        let test_data = PluginOutput::CallResponse(
+                let mut res = vec![];
-            0,
+                let test_data =
-            PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
+                    PluginResponse::Value(Box::new(encoding_test_data(row_cnt, col_cnt)));
-        );
+                let encoder = EncodingType::try_from_bytes(fmt.as_bytes()).unwrap();
-        let encoder = EncodingType::try_from_bytes(b"json").unwrap();
+                encoder.encode_response(&test_data, &mut res).unwrap();
-        let mut res = vec![];
+                let mut binary_data = std::io::Cursor::new(res);
-        encoder.encode(&test_data, &mut res).unwrap();
+                b.iter(|| {
-        bencher
+                    binary_data.set_position(0);
-            .with_inputs(|| {
+                    encoder.decode_response(&mut binary_data)
-                let mut binary_data = std::io::Cursor::new(res.clone());
+                })
-                binary_data.set_position(0);
+            });
-                binary_data
+        }
-            })
-            .bench_values(|mut binary_data| -> Result<Option<PluginOutput>, _> {
-                encoder.decode(&mut binary_data)
-            })
-    }
-
-    #[divan::bench(args = [(100, 5), (10000, 15)])]
-    fn msgpack_decode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
-        let test_data = PluginOutput::CallResponse(
-            0,
-            PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
-        );
-        let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
-        let mut res = vec![];
-        encoder.encode(&test_data, &mut res).unwrap();
-        bencher
-            .with_inputs(|| {
-                let mut binary_data = std::io::Cursor::new(res.clone());
-                binary_data.set_position(0);
-                binary_data
-            })
-            .bench_values(|mut binary_data| -> Result<Option<PluginOutput>, _> {
-                encoder.decode(&mut binary_data)
-            })
     }
+    group.finish();
 }
+
+criterion_group!(
+    benches,
+    parser_benchmarks,
+    eval_benchmarks,
+    encoding_benchmarks,
+    decoding_benchmarks
+);
+criterion_main!(benches);

crates/nu-cli/Cargo.toml

@@ -5,43 +5,42 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli"
 edition = "2021"
 license = "MIT"
 name = "nu-cli"
-version = "0.91.0"
+version = "0.88.2"
 
 [lib]
 bench = false
 
 [dev-dependencies]
-nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.91.0" }
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.88.2" }
-nu-command = { path = "../nu-command", version = "0.91.0" }
+nu-command = { path = "../nu-command", version = "0.88.2" }
-nu-test-support = { path = "../nu-test-support", version = "0.91.0" }
+nu-test-support = { path = "../nu-test-support", version = "0.88.2" }
 rstest = { version = "0.18.1", default-features = false }
 
 [dependencies]
-nu-cmd-base = { path = "../nu-cmd-base", version = "0.91.0" }
+nu-cmd-base = { path = "../nu-cmd-base", version = "0.88.2" }
-nu-engine = { path = "../nu-engine", version = "0.91.0" }
+nu-engine = { path = "../nu-engine", version = "0.88.2" }
-nu-path = { path = "../nu-path", version = "0.91.0" }
+nu-path = { path = "../nu-path", version = "0.88.2" }
-nu-parser = { path = "../nu-parser", version = "0.91.0" }
+nu-parser = { path = "../nu-parser", version = "0.88.2" }
-nu-protocol = { path = "../nu-protocol", version = "0.91.0" }
+nu-protocol = { path = "../nu-protocol", version = "0.88.2" }
-nu-utils = { path = "../nu-utils", version = "0.91.0" }
+nu-utils = { path = "../nu-utils", version = "0.88.2" }
-nu-color-config = { path = "../nu-color-config", version = "0.91.0" }
+nu-color-config = { path = "../nu-color-config", version = "0.88.2" }
-nu-ansi-term = "0.50.0"
+nu-ansi-term = "0.49.0"
-reedline = { version = "0.30.0", features = ["bashisms", "sqlite"] }
+reedline = { version = "0.27.0", features = ["bashisms", "sqlite"] }
 
 chrono = { default-features = false, features = ["std"], version = "0.4" }
 crossterm = "0.27"
-fancy-regex = "0.13"
+fancy-regex = "0.11"
 fuzzy-matcher = "0.3"
 is_executable = "1.0"
 log = "0.4"
-miette = { version = "7.1", features = ["fancy-no-backtrace"] }
+miette = { version = "5.10", features = ["fancy-no-backtrace"] }
-lscolors = { version = "0.17", default-features = false, features = ["nu-ansi-term"] }
 once_cell = "1.18"
 percent-encoding = "2"
 pathdiff = "0.2"
-sysinfo = "0.30"
+sysinfo = "0.29"
-unicode-segmentation = "1.11"
+unicode-segmentation = "1.10"
 uuid = { version = "1.6.0", features = ["v4"] }
-which = "6.0.0"
+which = "5.0.0"
 
 [features]
 plugin = []

crates/nu-cli/src/commands/commandline.rs

@@ -0,0 +1,129 @@
+use nu_engine::CallExt;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Type, Value,
+};
+use unicode_segmentation::UnicodeSegmentation;
+
+#[derive(Clone)]
+pub struct Commandline;
+
+impl Command for Commandline {
+    fn name(&self) -> &str {
+        "commandline"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("commandline")
+            .input_output_types(vec![
+                (Type::Nothing, Type::Nothing),
+                (Type::String, Type::String),
+            ])
+            .switch(
+                "cursor",
+                "Set or get the current cursor position",
+                Some('c'),
+            )
+            .switch(
+                "cursor-end",
+                "Set the current cursor position to the end of the buffer",
+                Some('e'),
+            )
+            .switch(
+                "append",
+                "appends the string to the end of the buffer",
+                Some('a'),
+            )
+            .switch(
+                "insert",
+                "inserts the string into the buffer at the cursor position",
+                Some('i'),
+            )
+            .switch(
+                "replace",
+                "replaces the current contents of the buffer (default)",
+                Some('r'),
+            )
+            .optional(
+                "cmd",
+                SyntaxShape::String,
+                "the string to perform the operation with",
+            )
+            .category(Category::Core)
+    }
+
+    fn usage(&self) -> &str {
+        "View or modify the current command line input buffer."
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["repl", "interactive"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        if let Some(cmd) = call.opt::<Value>(engine_state, stack, 0)? {
+            let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
+
+            if call.has_flag("cursor") {
+                let cmd_str = cmd.as_string()?;
+                match cmd_str.parse::<i64>() {
+                    Ok(n) => {
+                        repl.cursor_pos = if n <= 0 {
+                            0usize
+                        } else {
+                            repl.buffer
+                                .grapheme_indices(true)
+                                .map(|(i, _c)| i)
+                                .nth(n as usize)
+                                .unwrap_or(repl.buffer.len())
+                        }
+                    }
+                    Err(_) => {
+                        return Err(ShellError::CantConvert {
+                            to_type: "int".to_string(),
+                            from_type: "string".to_string(),
+                            span: cmd.span(),
+                            help: Some(format!(
+                                r#"string "{cmd_str}" does not represent a valid int"#
+                            )),
+                        })
+                    }
+                }
+            } else if call.has_flag("append") {
+                repl.buffer.push_str(&cmd.as_string()?);
+            } else if call.has_flag("insert") {
+                let cmd_str = cmd.as_string()?;
+                let cursor_pos = repl.cursor_pos;
+                repl.buffer.insert_str(cursor_pos, &cmd_str);
+                repl.cursor_pos += cmd_str.len();
+            } else {
+                repl.buffer = cmd.as_string()?;
+                repl.cursor_pos = repl.buffer.len();
+            }
+            Ok(Value::nothing(call.head).into_pipeline_data())
+        } else {
+            let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
+            if call.has_flag("cursor-end") {
+                repl.cursor_pos = repl.buffer.graphemes(true).count();
+                Ok(Value::nothing(call.head).into_pipeline_data())
+            } else if call.has_flag("cursor") {
+                let char_pos = repl
+                    .buffer
+                    .grapheme_indices(true)
+                    .chain(std::iter::once((repl.buffer.len(), "")))
+                    .position(|(i, _c)| i == repl.cursor_pos)
+                    .expect("Cursor position isn't on a grapheme boundary");
+                Ok(Value::string(char_pos.to_string(), call.head).into_pipeline_data())
+            } else {
+                Ok(Value::string(repl.buffer.to_string(), call.head).into_pipeline_data())
+            }
+        }
+    }
+}

crates/nu-cli/src/commands/commandline/commandline_.rs

@@ -1,189 +0,0 @@
-use nu_engine::CallExt;
-use nu_protocol::{
-    ast::Call,
-    engine::{Command, EngineState, Stack},
-    Category, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Type, Value,
-};
-use unicode_segmentation::UnicodeSegmentation;
-
-#[derive(Clone)]
-pub struct Commandline;
-
-impl Command for Commandline {
-    fn name(&self) -> &str {
-        "commandline"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build("commandline")
-            .input_output_types(vec![
-                (Type::Nothing, Type::Nothing),
-                (Type::String, Type::String),
-            ])
-            .switch(
-                "cursor",
-                "Set or get the current cursor position",
-                Some('c'),
-            )
-            .switch(
-                "cursor-end",
-                "Set the current cursor position to the end of the buffer",
-                Some('e'),
-            )
-            .switch(
-                "append",
-                "appends the string to the end of the buffer",
-                Some('a'),
-            )
-            .switch(
-                "insert",
-                "inserts the string into the buffer at the cursor position",
-                Some('i'),
-            )
-            .switch(
-                "replace",
-                "replaces the current contents of the buffer (default)",
-                Some('r'),
-            )
-            .optional(
-                "cmd",
-                SyntaxShape::String,
-                "the string to perform the operation with",
-            )
-            .category(Category::Core)
-    }
-
-    fn usage(&self) -> &str {
-        "View or modify the current command line input buffer."
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["repl", "interactive"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        if let Some(cmd) = call.opt::<Value>(engine_state, stack, 0)? {
-            let span = cmd.span();
-            let cmd = cmd.coerce_into_string()?;
-            let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
-
-            if call.has_flag(engine_state, stack, "cursor")? {
-                nu_protocol::report_error_new(
-                    engine_state,
-                    &ShellError::GenericError {
-                        error: "`--cursor (-c)` is deprecated".into(),
-                        msg: "Setting the current cursor position by `--cursor (-c)` is deprecated"
-                            .into(),
-                        span: Some(call.arguments_span()),
-                        help: Some("Use `commandline set-cursor`".into()),
-                        inner: vec![],
-                    },
-                );
-                match cmd.parse::<i64>() {
-                    Ok(n) => {
-                        repl.cursor_pos = if n <= 0 {
-                            0usize
-                        } else {
-                            repl.buffer
-                                .grapheme_indices(true)
-                                .map(|(i, _c)| i)
-                                .nth(n as usize)
-                                .unwrap_or(repl.buffer.len())
-                        }
-                    }
-                    Err(_) => {
-                        return Err(ShellError::CantConvert {
-                            to_type: "int".to_string(),
-                            from_type: "string".to_string(),
-                            span,
-                            help: Some(format!(r#"string "{cmd}" does not represent a valid int"#)),
-                        })
-                    }
-                }
-            } else if call.has_flag(engine_state, stack, "append")? {
-                nu_protocol::report_error_new(
-                    engine_state,
-                    &ShellError::GenericError {
-                        error: "`--append (-a)` is deprecated".into(),
-                        msg: "Appending the string to the end of the buffer by `--append (-a)` is deprecated".into(),
-                        span: Some(call.arguments_span()),
-                        help: Some("Use `commandline edit --append (-a)`".into()),
-                        inner: vec![],
-                    },
-                );
-                repl.buffer.push_str(&cmd);
-            } else if call.has_flag(engine_state, stack, "insert")? {
-                nu_protocol::report_error_new(
-                    engine_state,
-                    &ShellError::GenericError {
-                        error: "`--insert (-i)` is deprecated".into(),
-                        msg: "Inserts the string into the buffer at the cursor position by `--insert (-i)` is deprecated".into(),
-                        span: Some(call.arguments_span()),
-                        help: Some("Use `commandline edit --insert (-i)`".into()),
-                        inner: vec![],
-                    },
-                );
-                let cursor_pos = repl.cursor_pos;
-                repl.buffer.insert_str(cursor_pos, &cmd);
-                repl.cursor_pos += cmd.len();
-            } else {
-                nu_protocol::report_error_new(
-                    engine_state,
-                    &ShellError::GenericError {
-                        error: "`--replace (-r)` is deprecated".into(),
-                        msg: "Replaceing the current contents of the buffer by `--replace (-p)` or positional argument is deprecated".into(),
-                        span: Some(call.arguments_span()),
-                        help: Some("Use `commandline edit --replace (-r)`".into()),
-                        inner: vec![],
-                    },
-                );
-                repl.buffer = cmd;
-                repl.cursor_pos = repl.buffer.len();
-            }
-            Ok(Value::nothing(call.head).into_pipeline_data())
-        } else {
-            let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
-            if call.has_flag(engine_state, stack, "cursor-end")? {
-                nu_protocol::report_error_new(
-                    engine_state,
-                    &ShellError::GenericError {
-                        error: "`--cursor-end (-e)` is deprecated".into(),
-                        msg: "Setting the current cursor position to the end of the buffer by `--cursor-end (-e)` is deprecated".into(),
-                        span: Some(call.arguments_span()),
-                        help: Some("Use `commandline set-cursor --end (-e)`".into()),
-                        inner: vec![],
-                    },
-                );
-                repl.cursor_pos = repl.buffer.len();
-                Ok(Value::nothing(call.head).into_pipeline_data())
-            } else if call.has_flag(engine_state, stack, "cursor")? {
-                nu_protocol::report_error_new(
-                    engine_state,
-                    &ShellError::GenericError {
-                        error: "`--cursor (-c)` is deprecated".into(),
-                        msg: "Getting the current cursor position by `--cursor (-c)` is deprecated"
-                            .into(),
-                        span: Some(call.arguments_span()),
-                        help: Some("Use `commandline get-cursor`".into()),
-                        inner: vec![],
-                    },
-                );
-                let char_pos = repl
-                    .buffer
-                    .grapheme_indices(true)
-                    .chain(std::iter::once((repl.buffer.len(), "")))
-                    .position(|(i, _c)| i == repl.cursor_pos)
-                    .expect("Cursor position isn't on a grapheme boundary");
-                Ok(Value::string(char_pos.to_string(), call.head).into_pipeline_data())
-            } else {
-                Ok(Value::string(repl.buffer.to_string(), call.head).into_pipeline_data())
-            }
-        }
-    }
-}

crates/nu-cli/src/commands/commandline/edit.rs

@@ -1,71 +0,0 @@
-use nu_engine::CallExt;
-use nu_protocol::{
-    ast::Call,
-    engine::{Command, EngineState, Stack},
-    Category, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Type, Value,
-};
-
-#[derive(Clone)]
-pub struct SubCommand;
-
-impl Command for SubCommand {
-    fn name(&self) -> &str {
-        "commandline edit"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
-            .switch(
-                "append",
-                "appends the string to the end of the buffer",
-                Some('a'),
-            )
-            .switch(
-                "insert",
-                "inserts the string into the buffer at the cursor position",
-                Some('i'),
-            )
-            .switch(
-                "replace",
-                "replaces the current contents of the buffer (default)",
-                Some('r'),
-            )
-            .required(
-                "str",
-                SyntaxShape::String,
-                "the string to perform the operation with",
-            )
-            .category(Category::Core)
-    }
-
-    fn usage(&self) -> &str {
-        "Modify the current command line input buffer."
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["repl", "interactive"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let str: String = call.req(engine_state, stack, 0)?;
-        let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
-        if call.has_flag(engine_state, stack, "append")? {
-            repl.buffer.push_str(&str);
-        } else if call.has_flag(engine_state, stack, "insert")? {
-            let cursor_pos = repl.cursor_pos;
-            repl.buffer.insert_str(cursor_pos, &str);
-            repl.cursor_pos += str.len();
-        } else {
-            repl.buffer = str;
-            repl.cursor_pos = repl.buffer.len();
-        }
-        Ok(Value::nothing(call.head).into_pipeline_data())
-    }
-}

crates/nu-cli/src/commands/commandline/get_cursor.rs

@@ -1,56 +0,0 @@
-use nu_protocol::{
-    ast::Call,
-    engine::{Command, EngineState, Stack},
-    Category, IntoPipelineData, PipelineData, ShellError, Signature, Type, Value,
-};
-use unicode_segmentation::UnicodeSegmentation;
-
-#[derive(Clone)]
-pub struct SubCommand;
-
-impl Command for SubCommand {
-    fn name(&self) -> &str {
-        "commandline get-cursor"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .input_output_types(vec![(Type::Nothing, Type::Int)])
-            .allow_variants_without_examples(true)
-            .category(Category::Core)
-    }
-
-    fn usage(&self) -> &str {
-        "Get the current cursor position."
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["repl", "interactive"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        _stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let repl = engine_state.repl_state.lock().expect("repl state mutex");
-        let char_pos = repl
-            .buffer
-            .grapheme_indices(true)
-            .chain(std::iter::once((repl.buffer.len(), "")))
-            .position(|(i, _c)| i == repl.cursor_pos)
-            .expect("Cursor position isn't on a grapheme boundary");
-        match i64::try_from(char_pos) {
-            Ok(pos) => Ok(Value::int(pos, call.head).into_pipeline_data()),
-            Err(e) => Err(ShellError::GenericError {
-                error: "Failed to convert cursor position to int".to_string(),
-                msg: e.to_string(),
-                span: None,
-                help: None,
-                inner: vec![],
-            }),
-        }
-    }
-}

crates/nu-cli/src/commands/commandline/mod.rs

@@ -1,9 +0,0 @@
-mod commandline_;
-mod edit;
-mod get_cursor;
-mod set_cursor;
-
-pub use commandline_::Commandline;
-pub use edit::SubCommand as CommandlineEdit;
-pub use get_cursor::SubCommand as CommandlineGetCursor;
-pub use set_cursor::SubCommand as CommandlineSetCursor;

crates/nu-cli/src/commands/commandline/set_cursor.rs

@@ -1,69 +0,0 @@
-use nu_engine::CallExt;
-use nu_protocol::{
-    ast::Call,
-    engine::{Command, EngineState, Stack},
-    Category, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Type, Value,
-};
-use unicode_segmentation::UnicodeSegmentation;
-
-#[derive(Clone)]
-pub struct SubCommand;
-
-impl Command for SubCommand {
-    fn name(&self) -> &str {
-        "commandline set-cursor"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .input_output_types(vec![(Type::Nothing, Type::Nothing)])
-            .switch(
-                "end",
-                "set the current cursor position to the end of the buffer",
-                Some('e'),
-            )
-            .optional("pos", SyntaxShape::Int, "Cursor position to be set")
-            .category(Category::Core)
-    }
-
-    fn usage(&self) -> &str {
-        "Set the current cursor position."
-    }
-
-    fn search_terms(&self) -> Vec<&str> {
-        vec!["repl", "interactive"]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        _input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let mut repl = engine_state.repl_state.lock().expect("repl state mutex");
-        if let Some(pos) = call.opt::<i64>(engine_state, stack, 0)? {
-            repl.cursor_pos = if pos <= 0 {
-                0usize
-            } else {
-                repl.buffer
-                    .grapheme_indices(true)
-                    .map(|(i, _c)| i)
-                    .nth(pos as usize)
-                    .unwrap_or(repl.buffer.len())
-            };
-            Ok(Value::nothing(call.head).into_pipeline_data())
-        } else if call.has_flag(engine_state, stack, "end")? {
-            repl.cursor_pos = repl.buffer.len();
-            Ok(Value::nothing(call.head).into_pipeline_data())
-        } else {
-            Err(ShellError::GenericError {
-                error: "Required a positional argument or a flag".to_string(),
-                msg: "".to_string(),
-                span: None,
-                help: None,
-                inner: vec![],
-            })
-        }
-    }
-}

crates/nu-cli/src/commands/default_context.rs

@@ -14,9 +14,6 @@ pub fn add_cli_context(mut engine_state: EngineState) -> EngineState {
 
         bind_command! {
             Commandline,
-            CommandlineEdit,
-            CommandlineGetCursor,
-            CommandlineSetCursor,
             History,
             HistorySession,
             Keybindings,

crates/nu-cli/src/commands/history/history_.rs

@@ -1,4 +1,3 @@
-use nu_engine::CallExt;
 use nu_protocol::ast::Call;
 use nu_protocol::engine::{Command, EngineState, Stack};
 use nu_protocol::{
@@ -24,7 +23,10 @@ impl Command for History {
 
     fn signature(&self) -> nu_protocol::Signature {
         Signature::build("history")
-            .input_output_types(vec![(Type::Nothing, Type::Any)])
+            .input_output_types(vec![
+                (Type::Nothing, Type::Table(vec![])),
+                (Type::Nothing, Type::Nothing),
+            ])
             .allow_variants_without_examples(true)
             .switch("clear", "Clears out the history entries", Some('c'))
             .switch(
@@ -38,25 +40,21 @@ impl Command for History {
     fn run(
         &self,
         engine_state: &EngineState,
-        stack: &mut Stack,
+        _stack: &mut Stack,
         call: &Call,
         _input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let head = call.head;
 
-        let Some(history) = engine_state.history_config() else {
-            return Ok(PipelineData::empty());
-        };
-
         // todo for sqlite history this command should be an alias to `open ~/.config/nushell/history.sqlite3 | get history`
         if let Some(config_path) = nu_path::config_dir() {
-            let clear = call.has_flag(engine_state, stack, "clear")?;
+            let clear = call.has_flag("clear");
-            let long = call.has_flag(engine_state, stack, "long")?;
+            let long = call.has_flag("long");
             let ctrlc = engine_state.ctrlc.clone();
 
             let mut history_path = config_path;
             history_path.push("nushell");
-            match history.file_format {
+            match engine_state.config.history_file_format {
                 HistoryFileFormat::Sqlite => {
                     history_path.push("history.sqlite3");
                 }
@@ -70,28 +68,29 @@ impl Command for History {
                 // TODO: FIXME also clear the auxiliary files when using sqlite
                 Ok(PipelineData::empty())
             } else {
-                let history_reader: Option<Box<dyn ReedlineHistory>> = match history.file_format {
+                let history_reader: Option<Box<dyn ReedlineHistory>> =
-                    HistoryFileFormat::Sqlite => {
+                    match engine_state.config.history_file_format {
-                        SqliteBackedHistory::with_file(history_path.clone(), None, None)
+                        HistoryFileFormat::Sqlite => {
-                            .map(|inner| {
+                            SqliteBackedHistory::with_file(history_path, None, None)
-                                let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
+                                .map(|inner| {
-                                boxed
+                                    let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
-                            })
+                                    boxed
-                            .ok()
+                                })
-                    }
+                                .ok()
+                        }
 
-                    HistoryFileFormat::PlainText => FileBackedHistory::with_file(
+                        HistoryFileFormat::PlainText => FileBackedHistory::with_file(
-                        history.max_size as usize,
+                            engine_state.config.max_history_size as usize,
-                        history_path.clone(),
+                            history_path,
-                    )
+                        )
-                    .map(|inner| {
+                        .map(|inner| {
-                        let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
+                            let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
-                        boxed
+                            boxed
-                    })
+                        })
-                    .ok(),
+                        .ok(),
-                };
+                    };
 
-                match history.file_format {
+                match engine_state.config.history_file_format {
                     HistoryFileFormat::PlainText => Ok(history_reader
                         .and_then(|h| {
                             h.search(SearchQuery::everything(SearchDirection::Forward, None))
@@ -108,10 +107,7 @@ impl Command for History {
                                 )
                             })
                         })
-                        .ok_or(ShellError::FileNotFound {
+                        .ok_or(ShellError::FileNotFound { span: head })?
-                            file: history_path.display().to_string(),
-                            span: head,
-                        })?
                         .into_pipeline_data(ctrlc)),
                     HistoryFileFormat::Sqlite => Ok(history_reader
                         .and_then(|h| {
@@ -123,15 +119,12 @@ impl Command for History {
                                 create_history_record(idx, entry, long, head)
                             })
                         })
-                        .ok_or(ShellError::FileNotFound {
+                        .ok_or(ShellError::FileNotFound { span: head })?
-                            file: history_path.display().to_string(),
-                            span: head,
-                        })?
                         .into_pipeline_data(ctrlc)),
                 }
             }
         } else {
-            Err(ShellError::ConfigDirNotFound { span: Some(head) })
+            Err(ShellError::FileNotFound { span: head })
         }
     }
 

crates/nu-cli/src/commands/keybindings_listen.rs

@@ -112,7 +112,7 @@ pub fn print_events(engine_state: &EngineState) -> Result<Value, ShellError> {
         let o = match v {
             Value::Record { val, .. } => val
                 .iter()
-                .map(|(x, y)| format!("{}: {}", x, y.to_expanded_string("", config)))
+                .map(|(x, y)| format!("{}: {}", x, y.into_string("", config)))
                 .collect::<Vec<String>>()
                 .join(", "),
 

crates/nu-cli/src/commands/mod.rs

@@ -6,7 +6,7 @@ mod keybindings_default;
 mod keybindings_list;
 mod keybindings_listen;
 
-pub use commandline::{Commandline, CommandlineEdit, CommandlineGetCursor, CommandlineSetCursor};
+pub use commandline::Commandline;
 pub use history::{History, HistorySession};
 pub use keybindings::Keybindings;
 pub use keybindings_default::KeybindingsDefault;

crates/nu-cli/src/completions/command_completions.rs

@@ -43,9 +43,9 @@ impl CommandCompletion {
         if let Some(paths) = paths {
             if let Ok(paths) = paths.as_list() {
                 for path in paths {
-                    let path = path.coerce_str().unwrap_or_default();
+                    let path = path.as_string().unwrap_or_default();
 
-                    if let Ok(mut contents) = std::fs::read_dir(path.as_ref()) {
+                    if let Ok(mut contents) = std::fs::read_dir(path) {
                         while let Some(Ok(item)) = contents.next() {
                             if self.engine_state.config.max_external_completion_results
                                 > executables.len() as i64
@@ -94,7 +94,6 @@ impl CommandCompletion {
             .map(move |x| Suggestion {
                 value: String::from_utf8_lossy(&x.0).to_string(),
                 description: x.1,
-                style: None,
                 extra: None,
                 span: reedline::Span::new(span.start - offset, span.end - offset),
                 append_whitespace: true,
@@ -111,7 +110,6 @@ impl CommandCompletion {
                 .map(move |x| Suggestion {
                     value: x,
                     description: None,
-                    style: None,
                     extra: None,
                     span: reedline::Span::new(span.start - offset, span.end - offset),
                     append_whitespace: true,
@@ -125,7 +123,6 @@ impl CommandCompletion {
                     results.push(Suggestion {
                         value: format!("^{}", external.value),
                         description: None,
-                        style: None,
                         extra: None,
                         span: external.span,
                         append_whitespace: true,
@@ -253,7 +250,7 @@ mod command_completions_tests {
 
     #[test]
     fn test_find_non_whitespace_index() {
-        let commands = [
+        let commands = vec![
             ("    hello", 4),
             ("sudo ", 0),
             (" 	sudo ", 2),
@@ -273,7 +270,7 @@ mod command_completions_tests {
 
     #[test]
     fn test_is_last_command_passthrough() {
-        let commands = [
+        let commands = vec![
             ("    hello", false),
             ("    sudo ", true),
             ("sudo ", true),

crates/nu-cli/src/completions/completer.rs

@@ -2,7 +2,6 @@ use crate::completions::{
     CommandCompletion, Completer, CompletionOptions, CustomCompletion, DirectoryCompletion,
     DotNuCompletion, FileCompletion, FlagCompletion, VariableCompletion,
 };
-use nu_color_config::{color_record_to_nustyle, lookup_ansi_color_style};
 use nu_engine::eval_block;
 use nu_parser::{flatten_expression, parse, FlatShape};
 use nu_protocol::{
@@ -111,16 +110,10 @@ impl NuCompleter {
     fn completion_helper(&mut self, line: &str, pos: usize) -> Vec<Suggestion> {
         let mut working_set = StateWorkingSet::new(&self.engine_state);
         let offset = working_set.next_span_start();
-        // TODO: Callers should be trimming the line themselves
-        let line = if line.len() > pos { &line[..pos] } else { line };
-        // Adjust offset so that the spans of the suggestions will start at the right
-        // place even with `only_buffer_difference: true`
-        let fake_offset = offset + line.len() - pos;
-        let pos = offset + line.len();
         let initial_line = line.to_string();
         let mut line = line.to_string();
-        line.push('a');
+        line.insert(pos, 'a');
-
+        let pos = offset + pos;
         let config = self.engine_state.get_config();
 
         let output = parse(&mut working_set, Some("completer"), line.as_bytes(), false);
@@ -129,8 +122,6 @@ impl NuCompleter {
             for pipeline_element in pipeline.elements {
                 match pipeline_element {
                     PipelineElement::Expression(_, expr)
-                    | PipelineElement::ErrPipedExpression(_, expr)
-                    | PipelineElement::OutErrPipedExpression(_, expr)
                     | PipelineElement::Redirection(_, _, expr, _)
                     | PipelineElement::And(_, expr)
                     | PipelineElement::Or(_, expr)
@@ -195,7 +186,7 @@ impl NuCompleter {
                                         &working_set,
                                         prefix,
                                         new_span,
-                                        fake_offset,
+                                        offset,
                                         pos,
                                     );
                                 }
@@ -209,7 +200,7 @@ impl NuCompleter {
                                         &working_set,
                                         prefix.clone(),
                                         new_span,
-                                        fake_offset,
+                                        offset,
                                         pos,
                                     );
 
@@ -220,12 +211,9 @@ impl NuCompleter {
                                     // We got no results for internal completion
                                     // now we can check if external completer is set and use it
                                     if let Some(block_id) = config.external_completer {
-                                        if let Some(external_result) = self.external_completion(
+                                        if let Some(external_result) = self
-                                            block_id,
+                                            .external_completion(block_id, &spans, offset, new_span)
-                                            &spans,
+                                        {
-                                            fake_offset,
-                                            new_span,
-                                        ) {
                                             return external_result;
                                         }
                                     }
@@ -249,7 +237,7 @@ impl NuCompleter {
                                         &working_set,
                                         prefix,
                                         new_span,
-                                        fake_offset,
+                                        offset,
                                         pos,
                                     );
                                 }
@@ -262,35 +250,29 @@ impl NuCompleter {
                                             working_set.get_span_contents(previous_expr.0).to_vec();
 
                                         // Completion for .nu files
-                                        if prev_expr_str == b"use"
+                                        if prev_expr_str == b"use" || prev_expr_str == b"source-env"
-                                            || prev_expr_str == b"overlay use"
-                                            || prev_expr_str == b"source-env"
                                         {
-                                            let mut completer = DotNuCompletion::new(
+                                            let mut completer =
-                                                self.engine_state.clone(),
+                                                DotNuCompletion::new(self.engine_state.clone());
-                                                self.stack.clone(),
-                                            );
 
                                             return self.process_completion(
                                                 &mut completer,
                                                 &working_set,
                                                 prefix,
                                                 new_span,
-                                                fake_offset,
+                                                offset,
                                                 pos,
                                             );
                                         } else if prev_expr_str == b"ls" {
-                                            let mut completer = FileCompletion::new(
+                                            let mut completer =
-                                                self.engine_state.clone(),
+                                                FileCompletion::new(self.engine_state.clone());
-                                                self.stack.clone(),
-                                            );
 
                                             return self.process_completion(
                                                 &mut completer,
                                                 &working_set,
                                                 prefix,
                                                 new_span,
-                                                fake_offset,
+                                                offset,
                                                 pos,
                                             );
                                         }
@@ -312,37 +294,33 @@ impl NuCompleter {
                                             &working_set,
                                             prefix,
                                             new_span,
-                                            fake_offset,
+                                            offset,
                                             pos,
                                         );
                                     }
                                     FlatShape::Directory => {
-                                        let mut completer = DirectoryCompletion::new(
+                                        let mut completer =
-                                            self.engine_state.clone(),
+                                            DirectoryCompletion::new(self.engine_state.clone());
-                                            self.stack.clone(),
-                                        );
 
                                         return self.process_completion(
                                             &mut completer,
                                             &working_set,
                                             prefix,
                                             new_span,
-                                            fake_offset,
+                                            offset,
                                             pos,
                                         );
                                     }
                                     FlatShape::Filepath | FlatShape::GlobPattern => {
-                                        let mut completer = FileCompletion::new(
+                                        let mut completer =
-                                            self.engine_state.clone(),
+                                            FileCompletion::new(self.engine_state.clone());
-                                            self.stack.clone(),
-                                        );
 
                                         return self.process_completion(
                                             &mut completer,
                                             &working_set,
                                             prefix,
                                             new_span,
-                                            fake_offset,
+                                            offset,
                                             pos,
                                         );
                                     }
@@ -361,7 +339,7 @@ impl NuCompleter {
                                             &working_set,
                                             prefix.clone(),
                                             new_span,
-                                            fake_offset,
+                                            offset,
                                             pos,
                                         );
 
@@ -372,26 +350,23 @@ impl NuCompleter {
                                         // Try to complete using an external completer (if set)
                                         if let Some(block_id) = config.external_completer {
                                             if let Some(external_result) = self.external_completion(
-                                                block_id,
+                                                block_id, &spans, offset, new_span,
-                                                &spans,
-                                                fake_offset,
-                                                new_span,
                                             ) {
-                                                return external_result;
+                                                if !external_result.is_empty() {
+                                                    return external_result;
+                                                }
                                             }
                                         }
 
                                         // Check for file completion
-                                        let mut completer = FileCompletion::new(
+                                        let mut completer =
-                                            self.engine_state.clone(),
+                                            FileCompletion::new(self.engine_state.clone());
-                                            self.stack.clone(),
-                                        );
                                         out = self.process_completion(
                                             &mut completer,
                                             &working_set,
                                             prefix,
                                             new_span,
-                                            fake_offset,
+                                            offset,
                                             pos,
                                         );
 
@@ -474,11 +449,10 @@ pub fn map_value_completions<'a>(
 ) -> Vec<Suggestion> {
     list.filter_map(move |x| {
         // Match for string values
-        if let Ok(s) = x.coerce_string() {
+        if let Ok(s) = x.as_string() {
             return Some(Suggestion {
                 value: s,
                 description: None,
-                style: None,
                 extra: None,
                 span: reedline::Span {
                     start: span.start - offset,
@@ -493,7 +467,6 @@ pub fn map_value_completions<'a>(
             let mut suggestion = Suggestion {
                 value: String::from(""), // Initialize with empty string
                 description: None,
-                style: None,
                 extra: None,
                 span: reedline::Span {
                     start: span.start - offset,
@@ -507,7 +480,7 @@ pub fn map_value_completions<'a>(
                 // Match `value` column
                 if it.0 == "value" {
                     // Convert the value to string
-                    if let Ok(val_str) = it.1.coerce_string() {
+                    if let Ok(val_str) = it.1.as_string() {
                         // Update the suggestion value
                         suggestion.value = val_str;
                     }
@@ -516,21 +489,11 @@ pub fn map_value_completions<'a>(
                 // Match `description` column
                 if it.0 == "description" {
                     // Convert the value to string
-                    if let Ok(desc_str) = it.1.coerce_string() {
+                    if let Ok(desc_str) = it.1.as_string() {
                         // Update the suggestion value
                         suggestion.description = Some(desc_str);
                     }
                 }
-
-                // Match `style` column
-                if it.0 == "style" {
-                    // Convert the value to string
-                    suggestion.style = match it.1 {
-                        Value::String { val, .. } => Some(lookup_ansi_color_style(val)),
-                        Value::Record { .. } => Some(color_record_to_nustyle(it.1)),
-                        _ => None,
-                    };
-                }
             });
 
             return Some(suggestion);
@@ -564,7 +527,7 @@ mod completer_tests {
         );
 
         let mut completer = NuCompleter::new(engine_state.into(), Stack::new());
-        let dataset = [
+        let dataset = vec![
             ("sudo", false, "", Vec::new()),
             ("sudo l", true, "l", vec!["ls", "let", "lines", "loop"]),
             (" sudo", false, "", Vec::new()),

crates/nu-cli/src/completions/completion_common.rs

@@ -1,11 +1,6 @@
 use crate::completions::{matches, CompletionOptions};
-use nu_ansi_term::Style;
-use nu_engine::env_to_string;
 use nu_path::home_dir;
-use nu_protocol::engine::{EngineState, Stack};
 use nu_protocol::{engine::StateWorkingSet, Span};
-use nu_utils::get_ls_colors;
-use std::ffi::OsStr;
 use std::path::{is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP};
 
 fn complete_rec(
@@ -27,10 +22,7 @@ fn complete_rec(
                     Some(base) if matches(base, &entry_name, options) => {
                         let partial = &partial[1..];
                         if !partial.is_empty() || isdir {
-                            completions.extend(complete_rec(partial, &path, options, dir, isdir));
+                            completions.extend(complete_rec(partial, &path, options, dir, isdir))
-                            if entry_name.eq(base) {
-                                break;
-                            }
                         } else {
                             completions.push(path)
                         }
@@ -97,31 +89,12 @@ pub fn complete_item(
     partial: &str,
     cwd: &str,
     options: &CompletionOptions,
-    engine_state: &EngineState,
+) -> Vec<(nu_protocol::Span, String)> {
-    stack: &Stack,
-) -> Vec<(nu_protocol::Span, String, Option<Style>)> {
     let partial = surround_remove(partial);
     let isdir = partial.ends_with(is_separator);
     let cwd_pathbuf = Path::new(cwd).to_path_buf();
-    let ls_colors = (engine_state.config.use_ls_colors_completions
-        && engine_state.config.use_ansi_coloring)
-        .then(|| {
-            let ls_colors_env_str = match stack.get_env_var(engine_state, "LS_COLORS") {
-                Some(v) => env_to_string("LS_COLORS", &v, engine_state, stack).ok(),
-                None => None,
-            };
-            get_ls_colors(ls_colors_env_str)
-        });
     let mut original_cwd = OriginalCwd::None;
-    let mut components_vec: Vec<Component> = Path::new(&partial).components().collect();
+    let mut components = Path::new(&partial).components().peekable();
-
-    // Path components that end with a single "." get normalized away,
-    // so if the partial path ends in a literal "." we must add it back in manually
-    if partial.ends_with('.') && partial.len() > 1 {
-        components_vec.push(Component::Normal(OsStr::new(".")));
-    };
-    let mut components = components_vec.into_iter().peekable();
-
     let mut cwd = match components.peek().cloned() {
         Some(c @ Component::Prefix(..)) => {
             // windows only by definition
@@ -172,35 +145,12 @@ pub fn complete_item(
 
     complete_rec(partial.as_slice(), &cwd, options, want_directory, isdir)
         .into_iter()
-        .map(|p| {
+        .map(|p| (span, escape_path(original_cwd.apply(&p), want_directory)))
-            let path = original_cwd.apply(&p);
-            let style = ls_colors.as_ref().map(|lsc| {
-                lsc.style_for_path_with_metadata(
-                    &path,
-                    std::fs::symlink_metadata(&path).ok().as_ref(),
-                )
-                .map(lscolors::Style::to_nu_ansi_term_style)
-                .unwrap_or_default()
-            });
-            (span, escape_path(path, want_directory), style)
-        })
         .collect()
 }
 
 // Fix files or folders with quotes or hashes
 pub fn escape_path(path: String, dir: bool) -> String {
-    // make glob pattern have the highest priority.
-    let glob_contaminated = path.contains(['[', '*', ']', '?']);
-    if glob_contaminated {
-        return if path.contains('\'') {
-            // decide to use double quote, also need to escape `"` in path
-            // or else users can't do anything with completed path either.
-            format!("\"{}\"", path.replace('"', r#"\""#))
-        } else {
-            format!("'{path}'")
-        };
-    }
-
     let filename_contaminated = !dir && path.contains(['\'', '"', ' ', '#', '(', ')']);
     let dirname_contaminated = dir && path.contains(['\'', '"', ' ', '#']);
     let maybe_flag = path.starts_with('-');

crates/nu-cli/src/completions/custom_completions.rs

@@ -117,7 +117,7 @@ impl Completer for CustomCompletion {
                                 },
                                 match_algorithm: match options.get("completion_algorithm") {
                                     Some(option) => option
-                                        .coerce_string()
+                                        .as_string()
                                         .ok()
                                         .and_then(|option| option.try_into().ok())
                                         .unwrap_or(MatchAlgorithm::Prefix),

crates/nu-cli/src/completions/directory_completions.rs

@@ -2,9 +2,8 @@ use crate::completions::{
     completion_common::{adjust_if_intermediate, complete_item, AdjustView},
     Completer, CompletionOptions, SortBy,
 };
-use nu_ansi_term::Style;
 use nu_protocol::{
-    engine::{EngineState, Stack, StateWorkingSet},
+    engine::{EngineState, StateWorkingSet},
     levenshtein_distance, Span,
 };
 use reedline::Suggestion;
@@ -14,15 +13,11 @@ use std::sync::Arc;
 #[derive(Clone)]
 pub struct DirectoryCompletion {
     engine_state: Arc<EngineState>,
-    stack: Stack,
 }
 
 impl DirectoryCompletion {
-    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
+    pub fn new(engine_state: Arc<EngineState>) -> Self {
-        Self {
+        Self { engine_state }
-            engine_state,
-            stack,
-        }
     }
 }
 
@@ -44,14 +39,11 @@ impl Completer for DirectoryCompletion {
             &prefix,
             &self.engine_state.current_work_dir(),
             options,
-            self.engine_state.as_ref(),
-            &self.stack,
         )
         .into_iter()
         .map(move |x| Suggestion {
             value: x.1,
             description: None,
-            style: x.2,
             extra: None,
             span: reedline::Span {
                 start: x.0.start - offset,
@@ -120,8 +112,6 @@ pub fn directory_completion(
     partial: &str,
     cwd: &str,
     options: &CompletionOptions,
-    engine_state: &EngineState,
+) -> Vec<(nu_protocol::Span, String)> {
-    stack: &Stack,
+    complete_item(true, span, partial, cwd, options)
-) -> Vec<(nu_protocol::Span, String, Option<Style>)> {
-    complete_item(true, span, partial, cwd, options, engine_state, stack)
 }

crates/nu-cli/src/completions/dotnu_completions.rs

@@ -1,26 +1,22 @@
 use crate::completions::{file_path_completion, Completer, CompletionOptions, SortBy};
 use nu_protocol::{
-    engine::{EngineState, Stack, StateWorkingSet},
+    engine::{EngineState, StateWorkingSet},
     Span,
 };
 use reedline::Suggestion;
 use std::{
-    path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR},
+    path::{is_separator, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR},
     sync::Arc,
 };
 
 #[derive(Clone)]
 pub struct DotNuCompletion {
     engine_state: Arc<EngineState>,
-    stack: Stack,
 }
 
 impl DotNuCompletion {
-    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
+    pub fn new(engine_state: Arc<EngineState>) -> Self {
-        Self {
+        Self { engine_state }
-            engine_state,
-            stack,
-        }
     }
 }
 
@@ -54,7 +50,7 @@ impl Completer for DotNuCompletion {
                     .into_iter()
                     .flat_map(|it| {
                         it.iter().map(|x| {
-                            x.to_path()
+                            x.as_path()
                                 .expect("internal error: failed to convert lib path")
                         })
                     })
@@ -95,34 +91,21 @@ impl Completer for DotNuCompletion {
         // and transform them into suggestions
         let output: Vec<Suggestion> = search_dirs
             .into_iter()
-            .flat_map(|search_dir| {
+            .flat_map(|it| {
-                let completions = file_path_completion(
+                file_path_completion(span, &partial, &it, options)
-                    span,
-                    &partial,
-                    &search_dir,
-                    options,
-                    self.engine_state.as_ref(),
-                    &self.stack,
-                );
-                completions
                     .into_iter()
-                    .filter(move |it| {
+                    .filter(|it| {
                         // Different base dir, so we list the .nu files or folders
                         if !is_current_folder {
                             it.1.ends_with(".nu") || it.1.ends_with(SEP)
                         } else {
-                            // Lib dirs, so we filter only the .nu files or directory modules
+                            // Lib dirs, so we filter only the .nu files
-                            if it.1.ends_with(SEP) {
+                            it.1.ends_with(".nu")
-                                Path::new(&search_dir).join(&it.1).join("mod.nu").exists()
-                            } else {
-                                it.1.ends_with(".nu")
-                            }
                         }
                     })
                     .map(move |x| Suggestion {
                         value: x.1,
                         description: None,
-                        style: x.2,
                         extra: None,
                         span: reedline::Span {
                             start: x.0.start - offset,

crates/nu-cli/src/completions/file_completions.rs

@@ -2,9 +2,8 @@ use crate::completions::{
     completion_common::{adjust_if_intermediate, complete_item, AdjustView},
     Completer, CompletionOptions, SortBy,
 };
-use nu_ansi_term::Style;
 use nu_protocol::{
-    engine::{EngineState, Stack, StateWorkingSet},
+    engine::{EngineState, StateWorkingSet},
     levenshtein_distance, Span,
 };
 use nu_utils::IgnoreCaseExt;
@@ -15,15 +14,11 @@ use std::sync::Arc;
 #[derive(Clone)]
 pub struct FileCompletion {
     engine_state: Arc<EngineState>,
-    stack: Stack,
 }
 
 impl FileCompletion {
-    pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
+    pub fn new(engine_state: Arc<EngineState>) -> Self {
-        Self {
+        Self { engine_state }
-            engine_state,
-            stack,
-        }
     }
 }
 
@@ -49,14 +44,11 @@ impl Completer for FileCompletion {
             &prefix,
             &self.engine_state.current_work_dir(),
             options,
-            self.engine_state.as_ref(),
-            &self.stack,
         )
         .into_iter()
         .map(move |x| Suggestion {
             value: x.1,
             description: None,
-            style: x.2,
             extra: None,
             span: reedline::Span {
                 start: x.0.start - offset,
@@ -125,10 +117,8 @@ pub fn file_path_completion(
     partial: &str,
     cwd: &str,
     options: &CompletionOptions,
-    engine_state: &EngineState,
+) -> Vec<(nu_protocol::Span, String)> {
-    stack: &Stack,
+    complete_item(false, span, partial, cwd, options)
-) -> Vec<(nu_protocol::Span, String, Option<Style>)> {
-    complete_item(false, span, partial, cwd, options, engine_state, stack)
 }
 
 pub fn matches(partial: &str, from: &str, options: &CompletionOptions) -> bool {

crates/nu-cli/src/completions/flag_completions.rs

@@ -46,7 +46,6 @@ impl Completer for FlagCompletion {
                         output.push(Suggestion {
                             value: String::from_utf8_lossy(&named).to_string(),
                             description: Some(flag_desc.to_string()),
-                            style: None,
                             extra: None,
                             span: reedline::Span {
                                 start: span.start - offset,
@@ -69,7 +68,6 @@ impl Completer for FlagCompletion {
                     output.push(Suggestion {
                         value: String::from_utf8_lossy(&named).to_string(),
                         description: Some(flag_desc.to_string()),
-                        style: None,
                         extra: None,
                         span: reedline::Span {
                             start: span.start - offset,

crates/nu-cli/src/completions/variable_completions.rs

@@ -95,7 +95,6 @@ impl Completer for VariableCompletion {
                             output.push(Suggestion {
                                 value: env_var.0,
                                 description: None,
-                                style: None,
                                 extra: None,
                                 span: current_span,
                                 append_whitespace: false,
@@ -166,7 +165,6 @@ impl Completer for VariableCompletion {
                 output.push(Suggestion {
                     value: builtin.to_string(),
                     description: None,
-                    style: None,
                     extra: None,
                     span: current_span,
                     append_whitespace: false,
@@ -189,7 +187,6 @@ impl Completer for VariableCompletion {
                         output.push(Suggestion {
                             value: String::from_utf8_lossy(v.0).to_string(),
                             description: None,
-                            style: None,
                             extra: None,
                             span: current_span,
                             append_whitespace: false,
@@ -211,7 +208,6 @@ impl Completer for VariableCompletion {
                     output.push(Suggestion {
                         value: String::from_utf8_lossy(v.0).to_string(),
                         description: None,
-                        style: None,
                         extra: None,
                         span: current_span,
                         append_whitespace: false,
@@ -243,7 +239,6 @@ fn nested_suggestions(
                 output.push(Suggestion {
                     value: col,
                     description: None,
-                    style: None,
                     extra: None,
                     span: current_span,
                     append_whitespace: false,
@@ -258,7 +253,6 @@ fn nested_suggestions(
                 output.push(Suggestion {
                     value: column_name.to_string(),
                     description: None,
-                    style: None,
                     extra: None,
                     span: current_span,
                     append_whitespace: false,
@@ -272,7 +266,6 @@ fn nested_suggestions(
                 output.push(Suggestion {
                     value: column_name,
                     description: None,
-                    style: None,
                     extra: None,
                     span: current_span,
                     append_whitespace: false,

crates/nu-cli/src/config_files.rs

@@ -61,18 +61,17 @@ pub fn add_plugin_file(
     plugin_file: Option<Spanned<String>>,
     storage_path: &str,
 ) {
-    let working_set = StateWorkingSet::new(engine_state);
-    let cwd = working_set.get_cwd();
-
     if let Some(plugin_file) = plugin_file {
+        let working_set = StateWorkingSet::new(engine_state);
+        let cwd = working_set.get_cwd();
+
         if let Ok(path) = canonicalize_with(&plugin_file.item, cwd) {
             engine_state.plugin_signatures = Some(path)
         } else {
             let e = ParseError::FileNotFound(plugin_file.item, plugin_file.span);
             report_error(&working_set, &e);
         }
-    } else if let Some(plugin_path) = nu_path::config_dir() {
+    } else if let Some(mut plugin_path) = nu_path::config_dir() {
-        let mut plugin_path = canonicalize_with(&plugin_path, cwd).unwrap_or(plugin_path);
         // Path to store plugins signatures
         plugin_path.push(storage_path);
         plugin_path.push(PLUGIN_FILE);

crates/nu-cli/src/eval_cmds.rs

@@ -28,17 +28,13 @@ pub fn evaluate_commands(
     let (block, delta) = {
         if let Some(ref t_mode) = table_mode {
             let mut config = engine_state.get_config().clone();
-            config.table_mode = t_mode.coerce_str()?.parse().unwrap_or_default();
+            config.table_mode = t_mode.as_string()?.parse().unwrap_or_default();
             engine_state.set_config(config);
         }
 
         let mut working_set = StateWorkingSet::new(engine_state);
 
         let output = parse(&mut working_set, None, commands.item.as_bytes(), false);
-        if let Some(warning) = working_set.parse_warnings.first() {
-            report_error(&working_set, warning);
-        }
-
         if let Some(err) = working_set.parse_errors.first() {
             report_error(&working_set, err);
 
@@ -59,7 +55,7 @@ pub fn evaluate_commands(
         Ok(pipeline_data) => {
             let mut config = engine_state.get_config().clone();
             if let Some(t_mode) = table_mode {
-                config.table_mode = t_mode.coerce_str()?.parse().unwrap_or_default();
+                config.table_mode = t_mode.as_string()?.parse().unwrap_or_default();
             }
             crate::eval_file::print_table_or_error(engine_state, stack, pipeline_data, &mut config)
         }

crates/nu-cli/src/eval_file.rs

@@ -256,7 +256,7 @@ fn print_or_exit(pipeline_data: PipelineData, engine_state: &mut EngineState, co
             std::process::exit(1);
         }
 
-        let out = item.to_expanded_string("\n", config) + "\n";
+        let out = item.into_string("\n", config) + "\n";
         let _ = stdout_write_all_and_flush(out).map_err(|err| eprintln!("{err}"));
     }
 }

crates/nu-cli/src/lib.rs

@@ -19,7 +19,7 @@ pub use completions::{FileCompletion, NuCompleter};
 pub use config_files::eval_config_contents;
 pub use eval_cmds::evaluate_commands;
 pub use eval_file::evaluate_file;
-pub use menus::NuHelpCompleter;
+pub use menus::{DescriptionMenu, NuHelpCompleter};
 pub use nu_cmd_base::util::get_init_cwd;
 pub use nu_highlight::NuHighlight;
 pub use print::Print;

crates/nu-cli/src/menus/description_menu.rs

@@ -0,0 +1,730 @@
+use {
+    nu_ansi_term::{ansi::RESET, Style},
+    reedline::{
+        menu_functions::string_difference, Completer, Editor, Menu, MenuEvent, MenuTextStyle,
+        Painter, Suggestion, UndoBehavior,
+    },
+};
+
+/// Default values used as reference for the menu. These values are set during
+/// the initial declaration of the menu and are always kept as reference for the
+/// changeable [`WorkingDetails`]
+struct DefaultMenuDetails {
+    /// Number of columns that the menu will have
+    pub columns: u16,
+    /// Column width
+    pub col_width: Option<usize>,
+    /// Column padding
+    pub col_padding: usize,
+    /// Number of rows for commands
+    pub selection_rows: u16,
+    /// Number of rows allowed to display the description
+    pub description_rows: usize,
+}
+
+impl Default for DefaultMenuDetails {
+    fn default() -> Self {
+        Self {
+            columns: 4,
+            col_width: None,
+            col_padding: 2,
+            selection_rows: 4,
+            description_rows: 10,
+        }
+    }
+}
+
+/// Represents the actual column conditions of the menu. These conditions change
+/// since they need to accommodate possible different line sizes for the column values
+#[derive(Default)]
+struct WorkingDetails {
+    /// Number of columns that the menu will have
+    pub columns: u16,
+    /// Column width
+    pub col_width: usize,
+    /// Number of rows for description
+    pub description_rows: usize,
+}
+
+/// Completion menu definition
+pub struct DescriptionMenu {
+    /// Menu name
+    name: String,
+    /// Menu status
+    active: bool,
+    /// Menu coloring
+    color: MenuTextStyle,
+    /// Default column details that are set when creating the menu
+    /// These values are the reference for the working details
+    default_details: DefaultMenuDetails,
+    /// Number of minimum rows that are displayed when
+    /// the required lines is larger than the available lines
+    min_rows: u16,
+    /// Working column details keep changing based on the collected values
+    working_details: WorkingDetails,
+    /// Menu cached values
+    values: Vec<Suggestion>,
+    /// column position of the cursor. Starts from 0
+    col_pos: u16,
+    /// row position in the menu. Starts from 0
+    row_pos: u16,
+    /// Menu marker when active
+    marker: String,
+    /// Event sent to the menu
+    event: Option<MenuEvent>,
+    /// String collected after the menu is activated
+    input: Option<String>,
+    /// Examples to select
+    examples: Vec<String>,
+    /// Example index
+    example_index: Option<usize>,
+    /// Examples may not be shown if there is not enough space in the screen
+    show_examples: bool,
+    /// Skipped description rows
+    skipped_rows: usize,
+    /// Calls the completer using only the line buffer difference difference
+    /// after the menu was activated
+    only_buffer_difference: bool,
+}
+
+impl Default for DescriptionMenu {
+    fn default() -> Self {
+        Self {
+            name: "description_menu".to_string(),
+            active: false,
+            color: MenuTextStyle::default(),
+            default_details: DefaultMenuDetails::default(),
+            min_rows: 3,
+            working_details: WorkingDetails::default(),
+            values: Vec::new(),
+            col_pos: 0,
+            row_pos: 0,
+            marker: "? ".to_string(),
+            event: None,
+            input: None,
+            examples: Vec::new(),
+            example_index: None,
+            show_examples: true,
+            skipped_rows: 0,
+            only_buffer_difference: true,
+        }
+    }
+}
+
+// Menu configuration
+impl DescriptionMenu {
+    /// Menu builder with new name
+    pub fn with_name(mut self, name: &str) -> Self {
+        self.name = name.into();
+        self
+    }
+
+    /// Menu builder with new value for text style
+    pub fn with_text_style(mut self, text_style: Style) -> Self {
+        self.color.text_style = text_style;
+        self
+    }
+
+    /// Menu builder with new value for text style
+    pub fn with_selected_text_style(mut self, selected_text_style: Style) -> Self {
+        self.color.selected_text_style = selected_text_style;
+        self
+    }
+
+    /// Menu builder with new value for text style
+    pub fn with_description_text_style(mut self, description_text_style: Style) -> Self {
+        self.color.description_style = description_text_style;
+        self
+    }
+
+    /// Menu builder with new columns value
+    pub fn with_columns(mut self, columns: u16) -> Self {
+        self.default_details.columns = columns;
+        self
+    }
+
+    /// Menu builder with new column width value
+    pub fn with_column_width(mut self, col_width: Option<usize>) -> Self {
+        self.default_details.col_width = col_width;
+        self
+    }
+
+    /// Menu builder with new column width value
+    pub fn with_column_padding(mut self, col_padding: usize) -> Self {
+        self.default_details.col_padding = col_padding;
+        self
+    }
+
+    /// Menu builder with new selection rows value
+    pub fn with_selection_rows(mut self, selection_rows: u16) -> Self {
+        self.default_details.selection_rows = selection_rows;
+        self
+    }
+
+    /// Menu builder with new description rows value
+    pub fn with_description_rows(mut self, description_rows: usize) -> Self {
+        self.default_details.description_rows = description_rows;
+        self
+    }
+
+    /// Menu builder with marker
+    pub fn with_marker(mut self, marker: String) -> Self {
+        self.marker = marker;
+        self
+    }
+
+    /// Menu builder with new only buffer difference
+    pub fn with_only_buffer_difference(mut self, only_buffer_difference: bool) -> Self {
+        self.only_buffer_difference = only_buffer_difference;
+        self
+    }
+}
+
+// Menu functionality
+impl DescriptionMenu {
+    /// Move menu cursor to the next element
+    fn move_next(&mut self) {
+        let mut new_col = self.col_pos + 1;
+        let mut new_row = self.row_pos;
+
+        if new_col >= self.get_cols() {
+            new_row += 1;
+            new_col = 0;
+        }
+
+        if new_row >= self.get_rows() {
+            new_row = 0;
+            new_col = 0;
+        }
+
+        let position = new_row * self.get_cols() + new_col;
+        if position >= self.get_values().len() as u16 {
+            self.reset_position();
+        } else {
+            self.col_pos = new_col;
+            self.row_pos = new_row;
+        }
+    }
+
+    /// Move menu cursor to the previous element
+    fn move_previous(&mut self) {
+        let new_col = self.col_pos.checked_sub(1);
+
+        let (new_col, new_row) = match new_col {
+            Some(col) => (col, self.row_pos),
+            None => match self.row_pos.checked_sub(1) {
+                Some(row) => (self.get_cols().saturating_sub(1), row),
+                None => (
+                    self.get_cols().saturating_sub(1),
+                    self.get_rows().saturating_sub(1),
+                ),
+            },
+        };
+
+        let position = new_row * self.get_cols() + new_col;
+        if position >= self.get_values().len() as u16 {
+            self.col_pos = (self.get_values().len() as u16 % self.get_cols()).saturating_sub(1);
+            self.row_pos = self.get_rows().saturating_sub(1);
+        } else {
+            self.col_pos = new_col;
+            self.row_pos = new_row;
+        }
+    }
+
+    /// Menu index based on column and row position
+    fn index(&self) -> usize {
+        let index = self.row_pos * self.get_cols() + self.col_pos;
+        index as usize
+    }
+
+    /// Get selected value from the menu
+    fn get_value(&self) -> Option<Suggestion> {
+        self.get_values().get(self.index()).cloned()
+    }
+
+    /// Calculates how many rows the Menu will use
+    fn get_rows(&self) -> u16 {
+        let values = self.get_values().len() as u16;
+
+        if values == 0 {
+            // When the values are empty the no_records_msg is shown, taking 1 line
+            return 1;
+        }
+
+        let rows = values / self.get_cols();
+        if values % self.get_cols() != 0 {
+            rows + 1
+        } else {
+            rows
+        }
+    }
+
+    /// Returns working details col width
+    fn get_width(&self) -> usize {
+        self.working_details.col_width
+    }
+
+    /// Reset menu position
+    fn reset_position(&mut self) {
+        self.col_pos = 0;
+        self.row_pos = 0;
+        self.skipped_rows = 0;
+    }
+
+    fn no_records_msg(&self, use_ansi_coloring: bool) -> String {
+        let msg = "TYPE TO START SEARCH";
+        if use_ansi_coloring {
+            format!(
+                "{}{}{}",
+                self.color.selected_text_style.prefix(),
+                msg,
+                RESET
+            )
+        } else {
+            msg.to_string()
+        }
+    }
+
+    /// Returns working details columns
+    fn get_cols(&self) -> u16 {
+        self.working_details.columns.max(1)
+    }
+
+    /// End of line for menu
+    fn end_of_line(&self, column: u16, index: usize) -> &str {
+        let is_last = index == self.values.len().saturating_sub(1);
+        if column == self.get_cols().saturating_sub(1) || is_last {
+            "\r\n"
+        } else {
+            ""
+        }
+    }
+
+    /// Update list of examples from the actual value
+    fn update_examples(&mut self) {
+        self.examples = self
+            .get_value()
+            .and_then(|suggestion| suggestion.extra)
+            .unwrap_or_default();
+
+        self.example_index = None;
+    }
+
+    /// Creates default string that represents one suggestion from the menu
+    fn create_entry_string(
+        &self,
+        suggestion: &Suggestion,
+        index: usize,
+        column: u16,
+        empty_space: usize,
+        use_ansi_coloring: bool,
+    ) -> String {
+        if use_ansi_coloring {
+            if index == self.index() {
+                format!(
+                    "{}{}{}{:>empty$}{}",
+                    self.color.selected_text_style.prefix(),
+                    &suggestion.value,
+                    RESET,
+                    "",
+                    self.end_of_line(column, index),
+                    empty = empty_space,
+                )
+            } else {
+                format!(
+                    "{}{}{}{:>empty$}{}",
+                    self.color.text_style.prefix(),
+                    &suggestion.value,
+                    RESET,
+                    "",
+                    self.end_of_line(column, index),
+                    empty = empty_space,
+                )
+            }
+        } else {
+            // If no ansi coloring is found, then the selection word is
+            // the line in uppercase
+            let (marker, empty_space) = if index == self.index() {
+                (">", empty_space.saturating_sub(1))
+            } else {
+                ("", empty_space)
+            };
+
+            let line = format!(
+                "{}{}{:>empty$}{}",
+                marker,
+                &suggestion.value,
+                "",
+                self.end_of_line(column, index),
+                empty = empty_space,
+            );
+
+            if index == self.index() {
+                line.to_uppercase()
+            } else {
+                line
+            }
+        }
+    }
+
+    /// Description string with color
+    fn create_description_string(&self, use_ansi_coloring: bool) -> String {
+        let description = self
+            .get_value()
+            .and_then(|suggestion| suggestion.description)
+            .unwrap_or_default()
+            .lines()
+            .skip(self.skipped_rows)
+            .take(self.working_details.description_rows)
+            .collect::<Vec<&str>>()
+            .join("\r\n");
+
+        if use_ansi_coloring && !description.is_empty() {
+            format!(
+                "{}{}{}",
+                self.color.description_style.prefix(),
+                description,
+                RESET,
+            )
+        } else {
+            description
+        }
+    }
+
+    /// Selectable list of examples from the actual value
+    fn create_example_string(&self, use_ansi_coloring: bool) -> String {
+        if !self.show_examples {
+            return "".into();
+        }
+
+        let examples: String = self
+            .examples
+            .iter()
+            .enumerate()
+            .map(|(index, example)| {
+                if let Some(example_index) = self.example_index {
+                    if index == example_index {
+                        format!(
+                            "  {}{}{}\r\n",
+                            self.color.selected_text_style.prefix(),
+                            example,
+                            RESET
+                        )
+                    } else {
+                        format!("  {example}\r\n")
+                    }
+                } else {
+                    format!("  {example}\r\n")
+                }
+            })
+            .collect();
+
+        if examples.is_empty() {
+            "".into()
+        } else if use_ansi_coloring {
+            format!(
+                "{}\r\n\r\nExamples:\r\n{}{}",
+                self.color.description_style.prefix(),
+                RESET,
+                examples,
+            )
+        } else {
+            format!("\r\n\r\nExamples:\r\n{examples}",)
+        }
+    }
+}
+
+impl Menu for DescriptionMenu {
+    /// Menu name
+    fn name(&self) -> &str {
+        self.name.as_str()
+    }
+
+    /// Menu indicator
+    fn indicator(&self) -> &str {
+        self.marker.as_str()
+    }
+
+    /// Deactivates context menu
+    fn is_active(&self) -> bool {
+        self.active
+    }
+
+    /// The menu stays active even with one record
+    fn can_quick_complete(&self) -> bool {
+        false
+    }
+
+    /// The menu does not need to partially complete
+    fn can_partially_complete(
+        &mut self,
+        _values_updated: bool,
+        _editor: &mut Editor,
+        _completer: &mut dyn Completer,
+    ) -> bool {
+        false
+    }
+
+    /// Selects what type of event happened with the menu
+    fn menu_event(&mut self, event: MenuEvent) {
+        match &event {
+            MenuEvent::Activate(_) => self.active = true,
+            MenuEvent::Deactivate => {
+                self.active = false;
+                self.input = None;
+                self.values = Vec::new();
+            }
+            _ => {}
+        };
+
+        self.event = Some(event);
+    }
+
+    /// Updates menu values
+    fn update_values(&mut self, editor: &mut Editor, completer: &mut dyn Completer) {
+        if self.only_buffer_difference {
+            if let Some(old_string) = &self.input {
+                let (start, input) = string_difference(editor.get_buffer(), old_string);
+                if !input.is_empty() {
+                    self.reset_position();
+                    self.values = completer.complete(input, start);
+                }
+            }
+        } else {
+            let trimmed_buffer = editor.get_buffer().replace('\n', " ");
+            self.values = completer.complete(
+                trimmed_buffer.as_str(),
+                editor.line_buffer().insertion_point(),
+            );
+            self.reset_position();
+        }
+    }
+
+    /// The working details for the menu changes based on the size of the lines
+    /// collected from the completer
+    fn update_working_details(
+        &mut self,
+        editor: &mut Editor,
+        completer: &mut dyn Completer,
+        painter: &Painter,
+    ) {
+        if let Some(event) = self.event.take() {
+            // Updating all working parameters from the menu before executing any of the
+            // possible event
+            let max_width = self.get_values().iter().fold(0, |acc, suggestion| {
+                let str_len = suggestion.value.len() + self.default_details.col_padding;
+                if str_len > acc {
+                    str_len
+                } else {
+                    acc
+                }
+            });
+
+            // If no default width is found, then the total screen width is used to estimate
+            // the column width based on the default number of columns
+            let default_width = if let Some(col_width) = self.default_details.col_width {
+                col_width
+            } else {
+                let col_width = painter.screen_width() / self.default_details.columns;
+                col_width as usize
+            };
+
+            // Adjusting the working width of the column based the max line width found
+            // in the menu values
+            if max_width > default_width {
+                self.working_details.col_width = max_width;
+            } else {
+                self.working_details.col_width = default_width;
+            };
+
+            // The working columns is adjusted based on possible number of columns
+            // that could be fitted in the screen with the calculated column width
+            let possible_cols = painter.screen_width() / self.working_details.col_width as u16;
+            if possible_cols > self.default_details.columns {
+                self.working_details.columns = self.default_details.columns.max(1);
+            } else {
+                self.working_details.columns = possible_cols;
+            }
+
+            // Updating the working rows to display the description
+            if self.menu_required_lines(painter.screen_width()) <= painter.remaining_lines() {
+                self.working_details.description_rows = self.default_details.description_rows;
+                self.show_examples = true;
+            } else {
+                self.working_details.description_rows = painter
+                    .remaining_lines()
+                    .saturating_sub(self.default_details.selection_rows + 1)
+                    as usize;
+
+                self.show_examples = false;
+            }
+
+            match event {
+                MenuEvent::Activate(_) => {
+                    self.reset_position();
+                    self.input = Some(editor.get_buffer().to_string());
+                    self.update_values(editor, completer);
+                }
+                MenuEvent::Deactivate => self.active = false,
+                MenuEvent::Edit(_) => {
+                    self.reset_position();
+                    self.update_values(editor, completer);
+                    self.update_examples()
+                }
+                MenuEvent::NextElement => {
+                    self.skipped_rows = 0;
+                    self.move_next();
+                    self.update_examples();
+                }
+                MenuEvent::PreviousElement => {
+                    self.skipped_rows = 0;
+                    self.move_previous();
+                    self.update_examples();
+                }
+                MenuEvent::MoveUp => {
+                    if let Some(example_index) = self.example_index {
+                        if let Some(index) = example_index.checked_sub(1) {
+                            self.example_index = Some(index);
+                        } else {
+                            self.example_index = Some(self.examples.len().saturating_sub(1));
+                        }
+                    } else if !self.examples.is_empty() {
+                        self.example_index = Some(0);
+                    }
+                }
+                MenuEvent::MoveDown => {
+                    if let Some(example_index) = self.example_index {
+                        let index = example_index + 1;
+                        if index < self.examples.len() {
+                            self.example_index = Some(index);
+                        } else {
+                            self.example_index = Some(0);
+                        }
+                    } else if !self.examples.is_empty() {
+                        self.example_index = Some(0);
+                    }
+                }
+                MenuEvent::MoveLeft => self.skipped_rows = self.skipped_rows.saturating_sub(1),
+                MenuEvent::MoveRight => {
+                    let skipped = self.skipped_rows + 1;
+                    let description_rows = self
+                        .get_value()
+                        .and_then(|suggestion| suggestion.description)
+                        .unwrap_or_default()
+                        .lines()
+                        .count();
+
+                    let allowed_skips =
+                        description_rows.saturating_sub(self.working_details.description_rows);
+
+                    if skipped < allowed_skips {
+                        self.skipped_rows = skipped;
+                    } else {
+                        self.skipped_rows = allowed_skips;
+                    }
+                }
+                MenuEvent::PreviousPage | MenuEvent::NextPage => {}
+            }
+        }
+    }
+
+    /// The buffer gets replaced in the Span location
+    fn replace_in_buffer(&self, editor: &mut Editor) {
+        if let Some(Suggestion { value, span, .. }) = self.get_value() {
+            let start = span.start.min(editor.line_buffer().len());
+            let end = span.end.min(editor.line_buffer().len());
+
+            let replacement = if let Some(example_index) = self.example_index {
+                self.examples
+                    .get(example_index)
+                    .expect("the example index is always checked")
+            } else {
+                &value
+            };
+
+            editor.edit_buffer(
+                |lb| {
+                    lb.replace_range(start..end, replacement);
+                    let mut offset = lb.insertion_point();
+                    offset += lb
+                        .len()
+                        .saturating_sub(end.saturating_sub(start))
+                        .saturating_sub(start);
+                    lb.set_insertion_point(offset);
+                },
+                UndoBehavior::CreateUndoPoint,
+            );
+        }
+    }
+
+    /// Minimum rows that should be displayed by the menu
+    fn min_rows(&self) -> u16 {
+        self.get_rows().min(self.min_rows)
+    }
+
+    /// Gets values from filler that will be displayed in the menu
+    fn get_values(&self) -> &[Suggestion] {
+        &self.values
+    }
+
+    fn menu_required_lines(&self, _terminal_columns: u16) -> u16 {
+        let example_lines = self
+            .examples
+            .iter()
+            .fold(0, |acc, example| example.lines().count() + acc);
+
+        self.default_details.selection_rows
+            + self.default_details.description_rows as u16
+            + example_lines as u16
+            + 3
+    }
+
+    fn menu_string(&self, _available_lines: u16, use_ansi_coloring: bool) -> String {
+        if self.get_values().is_empty() {
+            self.no_records_msg(use_ansi_coloring)
+        } else {
+            // The skip values represent the number of lines that should be skipped
+            // while printing the menu
+            let available_lines = self.default_details.selection_rows;
+            let skip_values = if self.row_pos >= available_lines {
+                let skip_lines = self.row_pos.saturating_sub(available_lines) + 1;
+                (skip_lines * self.get_cols()) as usize
+            } else {
+                0
+            };
+
+            // It seems that crossterm prefers to have a complete string ready to be printed
+            // rather than looping through the values and printing multiple things
+            // This reduces the flickering when printing the menu
+            let available_values = (available_lines * self.get_cols()) as usize;
+            let selection_values: String = self
+                .get_values()
+                .iter()
+                .skip(skip_values)
+                .take(available_values)
+                .enumerate()
+                .map(|(index, suggestion)| {
+                    // Correcting the enumerate index based on the number of skipped values
+                    let index = index + skip_values;
+                    let column = index as u16 % self.get_cols();
+                    let empty_space = self.get_width().saturating_sub(suggestion.value.len());
+
+                    self.create_entry_string(
+                        suggestion,
+                        index,
+                        column,
+                        empty_space,
+                        use_ansi_coloring,
+                    )
+                })
+                .collect();
+
+            format!(
+                "{}{}{}",
+                selection_values,
+                self.create_description_string(use_ansi_coloring),
+                self.create_example_string(use_ansi_coloring)
+            )
+        }
+    }
+}

crates/nu-cli/src/menus/help_completions.rs

@@ -57,7 +57,7 @@ impl NuHelpCompleter {
 
                 if !sig.named.is_empty() {
                     long_desc.push_str(&get_flags_section(Some(&*self.0.clone()), sig, |v| {
-                        v.to_parsable_string(", ", &self.0.config)
+                        v.into_string_parsable(", ", &self.0.config)
                     }))
                 }
 
@@ -73,7 +73,7 @@ impl NuHelpCompleter {
                         let opt_suffix = if let Some(value) = &positional.default_value {
                             format!(
                                 " (optional, default: {})",
-                                &value.to_parsable_string(", ", &self.0.config),
+                                &value.into_string_parsable(", ", &self.0.config),
                             )
                         } else {
                             (" (optional)").to_string()
@@ -102,11 +102,10 @@ impl NuHelpCompleter {
                 Suggestion {
                     value: sig.name.clone(),
                     description: Some(long_desc),
-                    style: None,
                     extra: Some(extra),
                     span: reedline::Span {
-                        start: pos - line.len(),
+                        start: pos,
-                        end: pos,
+                        end: pos + line.len(),
                     },
                     append_whitespace: false,
                 }
@@ -120,42 +119,3 @@ impl Completer for NuHelpCompleter {
         self.completion_helper(line, pos)
     }
 }
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use rstest::rstest;
-
-    #[rstest]
-    #[case("who", 5, 8, &["whoami"])]
-    #[case("hash", 1, 5, &["hash", "hash md5", "hash sha256"])]
-    #[case("into f", 0, 6, &["into float", "into filesize"])]
-    #[case("into nonexistent", 0, 16, &[])]
-    fn test_help_completer(
-        #[case] line: &str,
-        #[case] start: usize,
-        #[case] end: usize,
-        #[case] expected: &[&str],
-    ) {
-        let engine_state =
-            nu_command::add_shell_command_context(nu_cmd_lang::create_default_context());
-        let mut completer = NuHelpCompleter::new(engine_state.into());
-        let suggestions = completer.complete(line, end);
-
-        assert_eq!(
-            expected.len(),
-            suggestions.len(),
-            "expected {:?}, got {:?}",
-            expected,
-            suggestions
-                .iter()
-                .map(|s| s.value.clone())
-                .collect::<Vec<_>>()
-        );
-
-        for (exp, actual) in expected.iter().zip(suggestions) {
-            assert_eq!(exp, &actual.value);
-            assert_eq!(reedline::Span::new(start, end), actual.span);
-        }
-    }
-}

crates/nu-cli/src/menus/menu_completions.rs

@@ -83,12 +83,10 @@ fn convert_to_suggestions(
         Value::Record { val, .. } => {
             let text = val
                 .get("value")
-                .and_then(|val| val.coerce_string().ok())
+                .and_then(|val| val.as_string().ok())
                 .unwrap_or_else(|| "No value key".to_string());
 
-            let description = val
+            let description = val.get("description").and_then(|val| val.as_string().ok());
-                .get("description")
-                .and_then(|val| val.coerce_string().ok());
 
             let span = match val.get("span") {
                 Some(Value::Record { val: span, .. }) => {
@@ -103,13 +101,9 @@ fn convert_to_suggestions(
                             }
                         }
                         _ => reedline::Span {
-                            start: if only_buffer_difference {
+                            start: if only_buffer_difference { pos } else { 0 },
-                                pos - line.len()
-                            } else {
-                                0
-                            },
                             end: if only_buffer_difference {
-                                pos
+                                pos + line.len()
                             } else {
                                 line.len()
                             },
@@ -117,13 +111,9 @@ fn convert_to_suggestions(
                     }
                 }
                 _ => reedline::Span {
-                    start: if only_buffer_difference {
+                    start: if only_buffer_difference { pos } else { 0 },
-                        pos - line.len()
-                    } else {
-                        0
-                    },
                     end: if only_buffer_difference {
-                        pos
+                        pos + line.len()
                     } else {
                         line.len()
                     },
@@ -148,7 +138,6 @@ fn convert_to_suggestions(
             vec![Suggestion {
                 value: text,
                 description,
-                style: None,
                 extra,
                 span,
                 append_whitespace: false,
@@ -161,19 +150,10 @@ fn convert_to_suggestions(
         _ => vec![Suggestion {
             value: format!("Not a record: {value:?}"),
             description: None,
-            style: None,
             extra: None,
             span: reedline::Span {
-                start: if only_buffer_difference {
+                start: 0,
-                    pos - line.len()
+                end: line.len(),
-                } else {
-                    0
-                },
-                end: if only_buffer_difference {
-                    pos
-                } else {
-                    line.len()
-                },
             },
             append_whitespace: false,
         }],

crates/nu-cli/src/menus/mod.rs

@@ -1,5 +1,7 @@
+mod description_menu;
 mod help_completions;
 mod menu_completions;
 
+pub use description_menu::DescriptionMenu;
 pub use help_completions::NuHelpCompleter;
 pub use menu_completions::NuMenuCompleter;

crates/nu-cli/src/nu_highlight.rs

@@ -28,7 +28,7 @@ impl Command for NuHighlight {
     fn run(
         &self,
         engine_state: &EngineState,
-        stack: &mut Stack,
+        _stack: &mut Stack,
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
@@ -40,14 +40,14 @@ impl Command for NuHighlight {
 
         let highlighter = crate::NuHighlighter {
             engine_state,
-            stack: std::sync::Arc::new(stack.clone()),
             config,
         };
 
         input.map(
-            move |x| match x.coerce_into_string() {
+            move |x| match x.as_string() {
                 Ok(line) => {
                     let highlights = highlighter.highlight(&line, line.len());
+
                     Value::string(highlights.render_simple(), head)
                 }
                 Err(err) => Value::error(err, head),

crates/nu-cli/src/print.rs

@@ -54,8 +54,8 @@ Since this command has no output, there is no point in piping it with other comm
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let args: Vec<Value> = call.rest(engine_state, stack, 0)?;
-        let no_newline = call.has_flag(engine_state, stack, "no-newline")?;
+        let no_newline = call.has_flag("no-newline");
-        let to_stderr = call.has_flag(engine_state, stack, "stderr")?;
+        let to_stderr = call.has_flag("stderr");
 
         // This will allow for easy printing of pipelines as well
         if !args.is_empty() {

crates/nu-cli/src/prompt.rs

@@ -1,4 +1,3 @@
-use crate::prompt_update::{POST_PROMPT_MARKER, PRE_PROMPT_MARKER};
 #[cfg(windows)]
 use nu_utils::enable_vt_processing;
 use reedline::DefaultPrompt;
@@ -12,7 +11,6 @@ use {
 /// Nushell prompt definition
 #[derive(Clone)]
 pub struct NushellPrompt {
-    shell_integration: bool,
     left_prompt_string: Option<String>,
     right_prompt_string: Option<String>,
     default_prompt_indicator: Option<String>,
@@ -22,10 +20,15 @@ pub struct NushellPrompt {
     render_right_prompt_on_last_line: bool,
 }
 
+impl Default for NushellPrompt {
+    fn default() -> Self {
+        NushellPrompt::new()
+    }
+}
+
 impl NushellPrompt {
-    pub fn new(shell_integration: bool) -> NushellPrompt {
+    pub fn new() -> NushellPrompt {
         NushellPrompt {
-            shell_integration,
             left_prompt_string: None,
             right_prompt_string: None,
             default_prompt_indicator: None,
@@ -108,11 +111,7 @@ impl Prompt for NushellPrompt {
                 .to_string()
                 .replace('\n', "\r\n");
 
-            if self.shell_integration {
+            prompt.into()
-                format!("{PRE_PROMPT_MARKER}{prompt}{POST_PROMPT_MARKER}").into()
-            } else {
-                prompt.into()
-            }
         }
     }
 

crates/nu-cli/src/prompt_update.rs

@@ -7,6 +7,8 @@ use nu_protocol::{
     Config, PipelineData, Value,
 };
 use reedline::Prompt;
+use std::borrow::Cow;
+use std::sync::Arc;
 
 // Name of environment variable where the prompt could be stored
 pub(crate) const PROMPT_COMMAND: &str = "PROMPT_COMMAND";
@@ -26,8 +28,8 @@ pub(crate) const TRANSIENT_PROMPT_MULTILINE_INDICATOR: &str =
     "TRANSIENT_PROMPT_MULTILINE_INDICATOR";
 // According to Daniel Imms @Tyriar, we need to do these this way:
 // <133 A><prompt><133 B><command><133 C><command output>
-pub(crate) const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
+const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
-pub(crate) const POST_PROMPT_MARKER: &str = "\x1b]133;B\x1b\\";
+const POST_PROMPT_MARKER: &str = "\x1b]133;B\x1b\\";
 
 fn get_prompt_string(
     prompt: &str,
@@ -96,12 +98,12 @@ fn get_prompt_string(
         })
 }
 
-pub(crate) fn update_prompt(
+pub(crate) fn update_prompt<'prompt>(
     config: &Config,
     engine_state: &EngineState,
     stack: &Stack,
-    nu_prompt: &mut NushellPrompt,
+    nu_prompt: &'prompt mut NushellPrompt,
-) {
+) -> &'prompt dyn Prompt {
     let mut stack = stack.clone();
 
     let left_prompt_string = get_prompt_string(PROMPT_COMMAND, config, engine_state, &mut stack);
@@ -144,55 +146,125 @@ pub(crate) fn update_prompt(
         (prompt_vi_insert_string, prompt_vi_normal_string),
         config.render_right_prompt_on_last_line,
     );
+
+    let ret_val = nu_prompt as &dyn Prompt;
     trace!("update_prompt {}:{}:{}", file!(), line!(), column!());
+
+    ret_val
 }
 
-/// Construct the transient prompt based on the normal nu_prompt
+struct TransientPrompt {
-pub(crate) fn make_transient_prompt(
+    engine_state: Arc<EngineState>,
+    stack: Stack,
+}
+
+/// Try getting `$env.TRANSIENT_PROMPT_<X>`, and get `$env.PROMPT_<X>` if that fails
+fn get_transient_prompt_string(
+    transient_prompt: &str,
+    prompt: &str,
     config: &Config,
     engine_state: &EngineState,
     stack: &mut Stack,
-    nu_prompt: &NushellPrompt,
+) -> Option<String> {
-) -> Box<dyn Prompt> {
+    get_prompt_string(transient_prompt, config, engine_state, stack)
-    let mut nu_prompt = nu_prompt.clone();
+        .or_else(|| get_prompt_string(prompt, config, engine_state, stack))
-
+}
-    if let Some(s) = get_prompt_string(TRANSIENT_PROMPT_COMMAND, config, engine_state, stack) {
+
-        nu_prompt.update_prompt_left(Some(s))
+impl Prompt for TransientPrompt {
-    }
+    fn render_prompt_left(&self) -> Cow<str> {
-
+        let mut nu_prompt = NushellPrompt::new();
-    if let Some(s) = get_prompt_string(TRANSIENT_PROMPT_COMMAND_RIGHT, config, engine_state, stack)
+        let config = &self.engine_state.get_config().clone();
-    {
+        let mut stack = self.stack.clone();
-        nu_prompt.update_prompt_right(Some(s), config.render_right_prompt_on_last_line)
+        nu_prompt.update_prompt_left(get_transient_prompt_string(
-    }
+            TRANSIENT_PROMPT_COMMAND,
-
+            PROMPT_COMMAND,
-    if let Some(s) = get_prompt_string(TRANSIENT_PROMPT_INDICATOR, config, engine_state, stack) {
+            config,
-        nu_prompt.update_prompt_indicator(Some(s))
+            &self.engine_state,
-    }
+            &mut stack,
-    if let Some(s) = get_prompt_string(
+        ));
-        TRANSIENT_PROMPT_INDICATOR_VI_INSERT,
+        nu_prompt.render_prompt_left().to_string().into()
-        config,
+    }
-        engine_state,
+
-        stack,
+    fn render_prompt_right(&self) -> Cow<str> {
-    ) {
+        let mut nu_prompt = NushellPrompt::new();
-        nu_prompt.update_prompt_vi_insert(Some(s))
+        let config = &self.engine_state.get_config().clone();
-    }
+        let mut stack = self.stack.clone();
-    if let Some(s) = get_prompt_string(
+        nu_prompt.update_prompt_right(
-        TRANSIENT_PROMPT_INDICATOR_VI_NORMAL,
+            get_transient_prompt_string(
-        config,
+                TRANSIENT_PROMPT_COMMAND_RIGHT,
-        engine_state,
+                PROMPT_COMMAND_RIGHT,
-        stack,
+                config,
-    ) {
+                &self.engine_state,
-        nu_prompt.update_prompt_vi_normal(Some(s))
+                &mut stack,
-    }
+            ),
-
+            config.render_right_prompt_on_last_line,
-    if let Some(s) = get_prompt_string(
+        );
-        TRANSIENT_PROMPT_MULTILINE_INDICATOR,
+        nu_prompt.render_prompt_right().to_string().into()
-        config,
+    }
-        engine_state,
+
-        stack,
+    fn render_prompt_indicator(&self, prompt_mode: reedline::PromptEditMode) -> Cow<str> {
-    ) {
+        let mut nu_prompt = NushellPrompt::new();
-        nu_prompt.update_prompt_multiline(Some(s))
+        let config = &self.engine_state.get_config().clone();
-    }
+        let mut stack = self.stack.clone();
-
+        nu_prompt.update_prompt_indicator(get_transient_prompt_string(
-    Box::new(nu_prompt)
+            TRANSIENT_PROMPT_INDICATOR,
+            PROMPT_INDICATOR,
+            config,
+            &self.engine_state,
+            &mut stack,
+        ));
+        nu_prompt.update_prompt_vi_insert(get_transient_prompt_string(
+            TRANSIENT_PROMPT_INDICATOR_VI_INSERT,
+            PROMPT_INDICATOR_VI_INSERT,
+            config,
+            &self.engine_state,
+            &mut stack,
+        ));
+        nu_prompt.update_prompt_vi_normal(get_transient_prompt_string(
+            TRANSIENT_PROMPT_INDICATOR_VI_NORMAL,
+            PROMPT_INDICATOR_VI_NORMAL,
+            config,
+            &self.engine_state,
+            &mut stack,
+        ));
+        nu_prompt
+            .render_prompt_indicator(prompt_mode)
+            .to_string()
+            .into()
+    }
+
+    fn render_prompt_multiline_indicator(&self) -> Cow<str> {
+        let mut nu_prompt = NushellPrompt::new();
+        let config = &self.engine_state.get_config().clone();
+        let mut stack = self.stack.clone();
+        nu_prompt.update_prompt_multiline(get_transient_prompt_string(
+            TRANSIENT_PROMPT_MULTILINE_INDICATOR,
+            PROMPT_MULTILINE_INDICATOR,
+            config,
+            &self.engine_state,
+            &mut stack,
+        ));
+        nu_prompt
+            .render_prompt_multiline_indicator()
+            .to_string()
+            .into()
+    }
+
+    fn render_prompt_history_search_indicator(
+        &self,
+        history_search: reedline::PromptHistorySearch,
+    ) -> Cow<str> {
+        NushellPrompt::new()
+            .render_prompt_history_search_indicator(history_search)
+            .to_string()
+            .into()
+    }
+}
+
+/// Construct the transient prompt
+pub(crate) fn transient_prompt(engine_state: Arc<EngineState>, stack: &Stack) -> Box<dyn Prompt> {
+    Box::new(TransientPrompt {
+        engine_state,
+        stack: stack.clone(),
+    })
 }

crates/nu-cli/src/reedline_config.rs

@@ -1,3 +1,4 @@
+use super::DescriptionMenu;
 use crate::{menus::NuMenuCompleter, NuHelpCompleter};
 use crossterm::event::{KeyCode, KeyModifiers};
 use nu_color_config::{color_record_to_nustyle, lookup_ansi_color_style};
@@ -11,8 +12,7 @@ use nu_protocol::{
 };
 use reedline::{
     default_emacs_keybindings, default_vi_insert_keybindings, default_vi_normal_keybindings,
-    ColumnarMenu, DescriptionMenu, DescriptionMode, EditCommand, IdeMenu, Keybindings, ListMenu,
+    ColumnarMenu, EditCommand, Keybindings, ListMenu, Reedline, ReedlineEvent, ReedlineMenu,
-    MenuBuilder, Reedline, ReedlineEvent, ReedlineMenu,
 };
 use std::sync::Arc;
 
@@ -84,7 +84,7 @@ pub(crate) fn add_menus(
     }
 
     // Checking if the default menus have been added from the config file
-    let default_menus = [
+    let default_menus = vec![
         ("completion_menu", DEFAULT_COMPLETION_MENU),
         ("history_menu", DEFAULT_HISTORY_MENU),
         ("help_menu", DEFAULT_HELP_MENU),
@@ -94,7 +94,7 @@ pub(crate) fn add_menus(
         if !config
             .menus
             .iter()
-            .any(|menu| menu.name.to_expanded_string("", config) == name)
+            .any(|menu| menu.name.into_string("", config) == name)
         {
             let (block, _) = {
                 let mut working_set = StateWorkingSet::new(&engine_state);
@@ -133,23 +133,22 @@ fn add_menu(
 ) -> Result<Reedline, ShellError> {
     let span = menu.menu_type.span();
     if let Value::Record { val, .. } = &menu.menu_type {
-        let layout = extract_value("layout", val, span)?.to_expanded_string("", config);
+        let layout = extract_value("layout", val, span)?.into_string("", config);
 
         match layout.as_str() {
             "columnar" => add_columnar_menu(line_editor, menu, engine_state, stack, config),
             "list" => add_list_menu(line_editor, menu, engine_state, stack, config),
-            "ide" => add_ide_menu(line_editor, menu, engine_state, stack, config),
             "description" => add_description_menu(line_editor, menu, engine_state, stack, config),
             _ => Err(ShellError::UnsupportedConfigValue {
-                expected: "columnar, list, ide or description".to_string(),
+                expected: "columnar, list or description".to_string(),
-                value: menu.menu_type.to_abbreviated_string(config),
+                value: menu.menu_type.into_abbreviated_string(config),
                 span: menu.menu_type.span(),
             }),
         }
     } else {
         Err(ShellError::UnsupportedConfigValue {
             expected: "only record type".to_string(),
-            value: menu.menu_type.to_abbreviated_string(config),
+            value: menu.menu_type.into_abbreviated_string(config),
             span: menu.menu_type.span(),
         })
     }
@@ -181,7 +180,7 @@ pub(crate) fn add_columnar_menu(
     config: &Config,
 ) -> Result<Reedline, ShellError> {
     let span = menu.menu_type.span();
-    let name = menu.name.to_expanded_string("", config);
+    let name = menu.name.into_string("", config);
     let mut columnar_menu = ColumnarMenu::default().with_name(&name);
 
     if let Value::Record { val, .. } = &menu.menu_type {
@@ -236,26 +235,10 @@ pub(crate) fn add_columnar_menu(
             columnar_menu,
             ColumnarMenu::with_description_text_style
         );
-        add_style!(
-            "match_text",
-            val,
-            span,
-            config,
-            columnar_menu,
-            ColumnarMenu::with_match_text_style
-        );
-        add_style!(
-            "selected_match_text",
-            val,
-            span,
-            config,
-            columnar_menu,
-            ColumnarMenu::with_selected_match_text_style
-        );
     }
 
-    let marker = menu.marker.to_expanded_string("", config);
+    let marker = menu.marker.into_string("", config);
-    columnar_menu = columnar_menu.with_marker(&marker);
+    columnar_menu = columnar_menu.with_marker(marker);
 
     let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
     columnar_menu = columnar_menu.with_only_buffer_difference(only_buffer_difference);
@@ -280,7 +263,7 @@ pub(crate) fn add_columnar_menu(
         }
         _ => Err(ShellError::UnsupportedConfigValue {
             expected: "block or omitted value".to_string(),
-            value: menu.source.to_abbreviated_string(config),
+            value: menu.source.into_abbreviated_string(config),
             span,
         }),
     }
@@ -294,7 +277,7 @@ pub(crate) fn add_list_menu(
     stack: &Stack,
     config: &Config,
 ) -> Result<Reedline, ShellError> {
-    let name = menu.name.to_expanded_string("", config);
+    let name = menu.name.into_string("", config);
     let mut list_menu = ListMenu::default().with_name(&name);
 
     let span = menu.menu_type.span();
@@ -336,8 +319,8 @@ pub(crate) fn add_list_menu(
         );
     }
 
-    let marker = menu.marker.to_expanded_string("", config);
+    let marker = menu.marker.into_string("", config);
-    list_menu = list_menu.with_marker(&marker);
+    list_menu = list_menu.with_marker(marker);
 
     let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
     list_menu = list_menu.with_only_buffer_difference(only_buffer_difference);
@@ -362,235 +345,12 @@ pub(crate) fn add_list_menu(
         }
         _ => Err(ShellError::UnsupportedConfigValue {
             expected: "block or omitted value".to_string(),
-            value: menu.source.to_abbreviated_string(config),
+            value: menu.source.into_abbreviated_string(config),
             span: menu.source.span(),
         }),
     }
 }
 
-// Adds an IDE menu to the line editor
-pub(crate) fn add_ide_menu(
-    line_editor: Reedline,
-    menu: &ParsedMenu,
-    engine_state: Arc<EngineState>,
-    stack: &Stack,
-    config: &Config,
-) -> Result<Reedline, ShellError> {
-    let span = menu.menu_type.span();
-    let name = menu.name.to_expanded_string("", config);
-    let mut ide_menu = IdeMenu::default().with_name(&name);
-
-    if let Value::Record { val, .. } = &menu.menu_type {
-        ide_menu = match extract_value("min_completion_width", val, span) {
-            Ok(min_completion_width) => {
-                let min_completion_width = min_completion_width.as_int()?;
-                ide_menu.with_min_completion_width(min_completion_width as u16)
-            }
-            Err(_) => ide_menu,
-        };
-
-        ide_menu = match extract_value("max_completion_width", val, span) {
-            Ok(max_completion_width) => {
-                let max_completion_width = max_completion_width.as_int()?;
-                ide_menu.with_max_completion_width(max_completion_width as u16)
-            }
-            Err(_) => ide_menu,
-        };
-
-        ide_menu = match extract_value("max_completion_height", val, span) {
-            Ok(max_completion_height) => {
-                let max_completion_height = max_completion_height.as_int()?;
-                ide_menu.with_max_completion_height(max_completion_height as u16)
-            }
-            Err(_) => ide_menu.with_max_completion_height(10u16),
-        };
-
-        ide_menu = match extract_value("padding", val, span) {
-            Ok(padding) => {
-                let padding = padding.as_int()?;
-                ide_menu.with_padding(padding as u16)
-            }
-            Err(_) => ide_menu,
-        };
-
-        ide_menu = match extract_value("border", val, span) {
-            Ok(border) => {
-                if let Ok(border) = border.as_bool() {
-                    if border {
-                        ide_menu.with_default_border()
-                    } else {
-                        ide_menu
-                    }
-                } else if let Ok(border_chars) = border.as_record() {
-                    let top_right = extract_value("top_right", border_chars, span)?.as_char()?;
-                    let top_left = extract_value("top_left", border_chars, span)?.as_char()?;
-                    let bottom_right =
-                        extract_value("bottom_right", border_chars, span)?.as_char()?;
-                    let bottom_left =
-                        extract_value("bottom_left", border_chars, span)?.as_char()?;
-                    let horizontal = extract_value("horizontal", border_chars, span)?.as_char()?;
-                    let vertical = extract_value("vertical", border_chars, span)?.as_char()?;
-
-                    ide_menu.with_border(
-                        top_right,
-                        top_left,
-                        bottom_right,
-                        bottom_left,
-                        horizontal,
-                        vertical,
-                    )
-                } else {
-                    return Err(ShellError::UnsupportedConfigValue {
-                        expected: "bool or record".to_string(),
-                        value: border.to_abbreviated_string(config),
-                        span: border.span(),
-                    });
-                }
-            }
-            Err(_) => ide_menu.with_default_border(),
-        };
-
-        ide_menu = match extract_value("cursor_offset", val, span) {
-            Ok(cursor_offset) => {
-                let cursor_offset = cursor_offset.as_int()?;
-                ide_menu.with_cursor_offset(cursor_offset as i16)
-            }
-            Err(_) => ide_menu,
-        };
-
-        ide_menu = match extract_value("description_mode", val, span) {
-            Ok(description_mode) => match description_mode.coerce_str()?.as_ref() {
-                "left" => ide_menu.with_description_mode(DescriptionMode::Left),
-                "right" => ide_menu.with_description_mode(DescriptionMode::Right),
-                "prefer_right" => ide_menu.with_description_mode(DescriptionMode::PreferRight),
-                _ => {
-                    return Err(ShellError::UnsupportedConfigValue {
-                        expected: "\"left\", \"right\" or \"prefer_right\"".to_string(),
-                        value: description_mode.to_abbreviated_string(config),
-                        span: description_mode.span(),
-                    });
-                }
-            },
-            Err(_) => ide_menu,
-        };
-
-        ide_menu = match extract_value("min_description_width", val, span) {
-            Ok(min_description_width) => {
-                let min_description_width = min_description_width.as_int()?;
-                ide_menu.with_min_description_width(min_description_width as u16)
-            }
-            Err(_) => ide_menu,
-        };
-
-        ide_menu = match extract_value("max_description_width", val, span) {
-            Ok(max_description_width) => {
-                let max_description_width = max_description_width.as_int()?;
-                ide_menu.with_max_description_width(max_description_width as u16)
-            }
-            Err(_) => ide_menu,
-        };
-
-        ide_menu = match extract_value("max_description_height", val, span) {
-            Ok(max_description_height) => {
-                let max_description_height = max_description_height.as_int()?;
-                ide_menu.with_max_description_height(max_description_height as u16)
-            }
-            Err(_) => ide_menu,
-        };
-
-        ide_menu = match extract_value("description_offset", val, span) {
-            Ok(description_padding) => {
-                let description_padding = description_padding.as_int()?;
-                ide_menu.with_description_offset(description_padding as u16)
-            }
-            Err(_) => ide_menu,
-        };
-
-        ide_menu = match extract_value("correct_cursor_pos", val, span) {
-            Ok(correct_cursor_pos) => {
-                let correct_cursor_pos = correct_cursor_pos.as_bool()?;
-                ide_menu.with_correct_cursor_pos(correct_cursor_pos)
-            }
-            Err(_) => ide_menu,
-        };
-    }
-
-    let span = menu.style.span();
-    if let Value::Record { val, .. } = &menu.style {
-        add_style!(
-            "text",
-            val,
-            span,
-            config,
-            ide_menu,
-            IdeMenu::with_text_style
-        );
-        add_style!(
-            "selected_text",
-            val,
-            span,
-            config,
-            ide_menu,
-            IdeMenu::with_selected_text_style
-        );
-        add_style!(
-            "description_text",
-            val,
-            span,
-            config,
-            ide_menu,
-            IdeMenu::with_description_text_style
-        );
-        add_style!(
-            "match_text",
-            val,
-            span,
-            config,
-            ide_menu,
-            IdeMenu::with_match_text_style
-        );
-        add_style!(
-            "selected_match_text",
-            val,
-            span,
-            config,
-            ide_menu,
-            IdeMenu::with_selected_match_text_style
-        );
-    }
-
-    let marker = menu.marker.to_expanded_string("", config);
-    ide_menu = ide_menu.with_marker(&marker);
-
-    let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
-    ide_menu = ide_menu.with_only_buffer_difference(only_buffer_difference);
-
-    let span = menu.source.span();
-    match &menu.source {
-        Value::Nothing { .. } => {
-            Ok(line_editor.with_menu(ReedlineMenu::EngineCompleter(Box::new(ide_menu))))
-        }
-        Value::Closure { val, .. } => {
-            let menu_completer = NuMenuCompleter::new(
-                val.block_id,
-                span,
-                stack.captures_to_stack(val.captures.clone()),
-                engine_state,
-                only_buffer_difference,
-            );
-            Ok(line_editor.with_menu(ReedlineMenu::WithCompleter {
-                menu: Box::new(ide_menu),
-                completer: Box::new(menu_completer),
-            }))
-        }
-        _ => Err(ShellError::UnsupportedConfigValue {
-            expected: "block or omitted value".to_string(),
-            value: menu.source.to_abbreviated_string(config),
-            span,
-        }),
-    }
-}
-
 // Adds a description menu to the line editor
 pub(crate) fn add_description_menu(
     line_editor: Reedline,
@@ -599,7 +359,7 @@ pub(crate) fn add_description_menu(
     stack: &Stack,
     config: &Config,
 ) -> Result<Reedline, ShellError> {
-    let name = menu.name.to_expanded_string("", config);
+    let name = menu.name.into_string("", config);
     let mut description_menu = DescriptionMenu::default().with_name(&name);
 
     let span = menu.menu_type.span();
@@ -673,8 +433,8 @@ pub(crate) fn add_description_menu(
         );
     }
 
-    let marker = menu.marker.to_expanded_string("", config);
+    let marker = menu.marker.into_string("", config);
-    description_menu = description_menu.with_marker(&marker);
+    description_menu = description_menu.with_marker(marker);
 
     let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
     description_menu = description_menu.with_only_buffer_difference(only_buffer_difference);
@@ -703,7 +463,7 @@ pub(crate) fn add_description_menu(
         }
         _ => Err(ShellError::UnsupportedConfigValue {
             expected: "closure or omitted value".to_string(),
-            value: menu.source.to_abbreviated_string(config),
+            value: menu.source.into_abbreviated_string(config),
             span: menu.source.span(),
         }),
     }
@@ -842,7 +602,7 @@ fn add_keybinding(
         }
         v => Err(ShellError::UnsupportedConfigValue {
             expected: "string or list of strings".to_string(),
-            value: v.to_abbreviated_string(config),
+            value: v.into_abbreviated_string(config),
             span: v.span(),
         }),
     }
@@ -855,7 +615,7 @@ fn add_parsed_keybinding(
 ) -> Result<(), ShellError> {
     let modifier = match keybinding
         .modifier
-        .to_expanded_string("", config)
+        .into_string("", config)
         .to_ascii_lowercase()
         .as_str()
     {
@@ -872,7 +632,7 @@ fn add_parsed_keybinding(
         _ => {
             return Err(ShellError::UnsupportedConfigValue {
                 expected: "CONTROL, SHIFT, ALT or NONE".to_string(),
-                value: keybinding.modifier.to_abbreviated_string(config),
+                value: keybinding.modifier.into_abbreviated_string(config),
                 span: keybinding.modifier.span(),
             })
         }
@@ -880,7 +640,7 @@ fn add_parsed_keybinding(
 
     let keycode = match keybinding
         .keycode
-        .to_expanded_string("", config)
+        .into_string("", config)
         .to_ascii_lowercase()
         .as_str()
     {
@@ -933,7 +693,7 @@ fn add_parsed_keybinding(
         _ => {
             return Err(ShellError::UnsupportedConfigValue {
                 expected: "crossterm KeyCode".to_string(),
-                value: keybinding.keycode.to_abbreviated_string(config),
+                value: keybinding.keycode.into_abbreviated_string(config),
                 span: keybinding.keycode.span(),
             })
         }
@@ -971,10 +731,7 @@ fn parse_event(value: &Value, config: &Config) -> Result<Option<ReedlineEvent>,
     match value {
         Value::Record { val: record, .. } => match EventType::try_from_record(record, span)? {
             EventType::Send(value) => event_from_record(
-                value
+                value.into_string("", config).to_ascii_lowercase().as_str(),
-                    .to_expanded_string("", config)
-                    .to_ascii_lowercase()
-                    .as_str(),
                 record,
                 config,
                 span,
@@ -982,10 +739,7 @@ fn parse_event(value: &Value, config: &Config) -> Result<Option<ReedlineEvent>,
             .map(Some),
             EventType::Edit(value) => {
                 let edit = edit_from_record(
-                    value
+                    value.into_string("", config).to_ascii_lowercase().as_str(),
-                        .to_expanded_string("", config)
-                        .to_ascii_lowercase()
-                        .as_str(),
                     record,
                     config,
                     span,
@@ -1013,7 +767,7 @@ fn parse_event(value: &Value, config: &Config) -> Result<Option<ReedlineEvent>,
                 }
                 v => Err(ShellError::UnsupportedConfigValue {
                     expected: "list of events".to_string(),
-                    value: v.to_abbreviated_string(config),
+                    value: v.into_abbreviated_string(config),
                     span: v.span(),
                 }),
             },
@@ -1039,7 +793,7 @@ fn parse_event(value: &Value, config: &Config) -> Result<Option<ReedlineEvent>,
         Value::Nothing { .. } => Ok(None),
         v => Err(ShellError::UnsupportedConfigValue {
             expected: "record or list of records, null to unbind key".to_string(),
-            value: v.to_abbreviated_string(config),
+            value: v.into_abbreviated_string(config),
             span: v.span(),
         }),
     }
@@ -1082,11 +836,11 @@ fn event_from_record(
         "openeditor" => ReedlineEvent::OpenEditor,
         "menu" => {
             let menu = extract_value("name", record, span)?;
-            ReedlineEvent::Menu(menu.to_expanded_string("", config))
+            ReedlineEvent::Menu(menu.into_string("", config))
         }
         "executehostcommand" => {
             let cmd = extract_value("cmd", record, span)?;
-            ReedlineEvent::ExecuteHostCommand(cmd.to_expanded_string("", config))
+            ReedlineEvent::ExecuteHostCommand(cmd.into_string("", config))
         }
         v => {
             return Err(ShellError::UnsupportedConfigValue {
@@ -1107,82 +861,22 @@ fn edit_from_record(
     span: Span,
 ) -> Result<EditCommand, ShellError> {
     let edit = match name {
-        "movetostart" => EditCommand::MoveToStart {
+        "movetostart" => EditCommand::MoveToStart,
-            select: extract_value("select", record, span)
+        "movetolinestart" => EditCommand::MoveToLineStart,
-                .and_then(|value| value.as_bool())
+        "movetoend" => EditCommand::MoveToEnd,
-                .unwrap_or(false),
+        "movetolineend" => EditCommand::MoveToLineEnd,
-        },
+        "moveleft" => EditCommand::MoveLeft,
-        "movetolinestart" => EditCommand::MoveToLineStart {
+        "moveright" => EditCommand::MoveRight,
-            select: extract_value("select", record, span)
+        "movewordleft" => EditCommand::MoveWordLeft,
-                .and_then(|value| value.as_bool())
+        "movebigwordleft" => EditCommand::MoveBigWordLeft,
-                .unwrap_or(false),
+        "movewordright" => EditCommand::MoveWordRight,
-        },
+        "movewordrightend" => EditCommand::MoveWordRightEnd,
-
+        "movebigwordrightend" => EditCommand::MoveBigWordRightEnd,
-        "movetoend" => EditCommand::MoveToEnd {
+        "movewordrightstart" => EditCommand::MoveWordRightStart,
-            select: extract_value("select", record, span)
+        "movebigwordrightstart" => EditCommand::MoveBigWordRightStart,
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
-        "movetolineend" => EditCommand::MoveToLineEnd {
-            select: extract_value("select", record, span)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
-        "moveleft" => EditCommand::MoveLeft {
-            select: extract_value("select", record, span)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
-        "moveright" => EditCommand::MoveRight {
-            select: extract_value("select", record, span)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
-        "movewordleft" => EditCommand::MoveWordLeft {
-            select: extract_value("select", record, span)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
-        "movebigwordleft" => EditCommand::MoveBigWordLeft {
-            select: extract_value("select", record, span)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
-        "movewordright" => EditCommand::MoveWordRight {
-            select: extract_value("select", record, span)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
-        "movewordrightend" => EditCommand::MoveWordRightEnd {
-            select: extract_value("select", record, span)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
-        "movebigwordrightend" => EditCommand::MoveBigWordRightEnd {
-            select: extract_value("select", record, span)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
-        "movewordrightstart" => EditCommand::MoveWordRightStart {
-            select: extract_value("select", record, span)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
-        "movebigwordrightstart" => EditCommand::MoveBigWordRightStart {
-            select: extract_value("select", record, span)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false),
-        },
         "movetoposition" => {
             let value = extract_value("value", record, span)?;
-            let select = extract_value("select", record, span)
+            EditCommand::MoveToPosition(value.as_int()? as usize)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false);
-
-            EditCommand::MoveToPosition {
-                position: value.as_int()? as usize,
-                select,
-            }
         }
         "insertchar" => {
             let value = extract_value("value", record, span)?;
@@ -1191,7 +885,7 @@ fn edit_from_record(
         }
         "insertstring" => {
             let value = extract_value("value", record, span)?;
-            EditCommand::InsertString(value.to_expanded_string("", config))
+            EditCommand::InsertString(value.into_string("", config))
         }
         "insertnewline" => EditCommand::InsertNewline,
         "backspace" => EditCommand::Backspace,
@@ -1234,18 +928,12 @@ fn edit_from_record(
         "moverightuntil" => {
             let value = extract_value("value", record, span)?;
             let char = extract_char(value, config)?;
-            let select = extract_value("select", record, span)
+            EditCommand::MoveRightUntil(char)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false);
-            EditCommand::MoveRightUntil { c: char, select }
         }
         "moverightbefore" => {
             let value = extract_value("value", record, span)?;
             let char = extract_char(value, config)?;
-            let select = extract_value("select", record, span)
+            EditCommand::MoveRightBefore(char)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false);
-            EditCommand::MoveRightBefore { c: char, select }
         }
         "cutleftuntil" => {
             let value = extract_value("value", record, span)?;
@@ -1260,23 +948,14 @@ fn edit_from_record(
         "moveleftuntil" => {
             let value = extract_value("value", record, span)?;
             let char = extract_char(value, config)?;
-            let select = extract_value("select", record, span)
+            EditCommand::MoveLeftUntil(char)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false);
-            EditCommand::MoveLeftUntil { c: char, select }
         }
         "moveleftbefore" => {
             let value = extract_value("value", record, span)?;
             let char = extract_char(value, config)?;
-            let select = extract_value("select", record, span)
+            EditCommand::MoveLeftBefore(char)
-                .and_then(|value| value.as_bool())
-                .unwrap_or(false);
-            EditCommand::MoveLeftBefore { c: char, select }
         }
         "complete" => EditCommand::Complete,
-        "cutselection" => EditCommand::CutSelection,
-        "copyselection" => EditCommand::CopySelection,
-        "selectall" => EditCommand::SelectAll,
         e => {
             return Err(ShellError::UnsupportedConfigValue {
                 expected: "reedline EditCommand".to_string(),
@@ -1292,7 +971,7 @@ fn edit_from_record(
 fn extract_char(value: &Value, config: &Config) -> Result<char, ShellError> {
     let span = value.span();
     value
-        .to_expanded_string("", config)
+        .into_string("", config)
         .chars()
         .next()
         .ok_or_else(|| ShellError::MissingConfigValue {
@@ -1430,57 +1109,4 @@ mod test {
         let b = EventType::try_from_record(&event, span);
         assert!(matches!(b, Err(ShellError::MissingConfigValue { .. })));
     }
-
-    #[test]
-    fn test_move_without_optional_select() {
-        let event = record! {
-            "edit" => Value::test_string("moveleft")
-        };
-        let event = Value::test_record(event);
-        let config = Config::default();
-
-        let parsed_event = parse_event(&event, &config).unwrap();
-        assert_eq!(
-            parsed_event,
-            Some(ReedlineEvent::Edit(vec![EditCommand::MoveLeft {
-                select: false
-            }]))
-        );
-    }
-
-    #[test]
-    fn test_move_with_select_false() {
-        let event = record! {
-            "edit" => Value::test_string("moveleft"),
-            "select" => Value::test_bool(false)
-        };
-        let event = Value::test_record(event);
-        let config = Config::default();
-
-        let parsed_event = parse_event(&event, &config).unwrap();
-        assert_eq!(
-            parsed_event,
-            Some(ReedlineEvent::Edit(vec![EditCommand::MoveLeft {
-                select: false
-            }]))
-        );
-    }
-
-    #[test]
-    fn test_move_with_select_true() {
-        let event = record! {
-            "edit" => Value::test_string("moveleft"),
-            "select" => Value::test_bool(true)
-        };
-        let event = Value::test_record(event);
-        let config = Config::default();
-
-        let parsed_event = parse_event(&event, &config).unwrap();
-        assert_eq!(
-            parsed_event,
-            Some(ReedlineEvent::Edit(vec![EditCommand::MoveLeft {
-                select: true
-            }]))
-        );
-    }
 }

crates/nu-cli/src/syntax_highlight.rs

@@ -1,17 +1,15 @@
 use log::trace;
 use nu_ansi_term::Style;
 use nu_color_config::{get_matching_brackets_style, get_shape_color};
-use nu_engine::env;
 use nu_parser::{flatten_block, parse, FlatShape};
 use nu_protocol::ast::{Argument, Block, Expr, Expression, PipelineElement, RecordItem};
-use nu_protocol::engine::{EngineState, Stack, StateWorkingSet};
+use nu_protocol::engine::{EngineState, StateWorkingSet};
 use nu_protocol::{Config, Span};
 use reedline::{Highlighter, StyledText};
 use std::sync::Arc;
 
 pub struct NuHighlighter {
     pub engine_state: Arc<EngineState>,
-    pub stack: Arc<Stack>,
     pub config: Config,
 }
 
@@ -34,17 +32,7 @@ impl Highlighter for NuHighlighter {
                             working_set.get_span_contents(Span::new(span.start, span.end));
 
                         let str_word = String::from_utf8_lossy(str_contents).to_string();
-                        let paths = env::path_str(&self.engine_state, &self.stack, *span).ok();
+                        if which::which(str_word).ok().is_some() {
-                        let res = if let Ok(cwd) =
-                            env::current_dir_str(&self.engine_state, &self.stack)
-                        {
-                            which::which_in(str_word, paths.as_ref(), cwd).ok()
-                        } else {
-                            which::which_in_global(str_word, paths.as_ref())
-                                .ok()
-                                .and_then(|mut i| i.next())
-                        };
-                        if res.is_some() {
                             *shape = FlatShape::ExternalResolved;
                         }
                     }
@@ -264,8 +252,6 @@ fn find_matching_block_end_in_block(
         for e in &p.elements {
             match e {
                 PipelineElement::Expression(_, e)
-                | PipelineElement::ErrPipedExpression(_, e)
-                | PipelineElement::OutErrPipedExpression(_, e)
                 | PipelineElement::Redirection(_, _, e, _)
                 | PipelineElement::And(_, e)
                 | PipelineElement::Or(_, e)
@@ -335,14 +321,15 @@ fn find_matching_block_end_in_expr(
             Expr::Keyword(..) => None,
             Expr::ValueWithUnit(..) => None,
             Expr::DateTime(_) => None,
-            Expr::Filepath(_, _) => None,
+            Expr::Filepath(_) => None,
-            Expr::Directory(_, _) => None,
+            Expr::Directory(_) => None,
-            Expr::GlobPattern(_, _) => None,
+            Expr::GlobPattern(_) => None,
             Expr::String(_) => None,
             Expr::CellPath(_) => None,
             Expr::ImportPattern(_) => None,
             Expr::Overlay(_) => None,
             Expr::Signature(_) => None,
+            Expr::MatchPattern(_) => None,
             Expr::MatchBlock(_) => None,
             Expr::Nothing => None,
             Expr::Garbage => None,
@@ -399,7 +386,6 @@ fn find_matching_block_end_in_expr(
                         Argument::Named((_, _, opt_expr)) => opt_expr.as_ref(),
                         Argument::Positional(inner_expr) => Some(inner_expr),
                         Argument::Unknown(inner_expr) => Some(inner_expr),
-                        Argument::Spread(inner_expr) => Some(inner_expr),
                     };
 
                     if let Some(inner_expr) = opt_expr {

crates/nu-cli/src/util.rs

@@ -220,10 +220,6 @@ pub fn eval_source(
             source,
             false,
         );
-        if let Some(warning) = working_set.parse_warnings.first() {
-            report_error(&working_set, warning);
-        }
-
         if let Some(err) = working_set.parse_errors.first() {
             set_last_exit_code(stack, 1);
             report_error(&working_set, err);

crates/nu-cli/tests/completions.rs

@@ -91,7 +91,7 @@ fn variables_dollar_sign_with_varialblecompletion() {
     let target_dir = "$ ";
     let suggestions = completer.complete(target_dir, target_dir.len());
 
-    assert_eq!(8, suggestions.len());
+    assert_eq!(7, suggestions.len());
 }
 
 #[rstest]
@@ -144,34 +144,15 @@ fn dotnu_completions() {
     let completion_str = "source-env ".to_string();
     let suggestions = completer.complete(&completion_str, completion_str.len());
 
-    assert_eq!(2, suggestions.len());
+    assert_eq!(1, suggestions.len());
     assert_eq!("custom_completion.nu", suggestions.first().unwrap().value);
-    #[cfg(windows)]
-    assert_eq!("directory_completion\\", suggestions.get(1).unwrap().value);
-    #[cfg(not(windows))]
-    assert_eq!("directory_completion/", suggestions.get(1).unwrap().value);
 
     // Test use completion
     let completion_str = "use ".to_string();
     let suggestions = completer.complete(&completion_str, completion_str.len());
 
-    assert_eq!(2, suggestions.len());
+    assert_eq!(1, suggestions.len());
     assert_eq!("custom_completion.nu", suggestions.first().unwrap().value);
-    #[cfg(windows)]
-    assert_eq!("directory_completion\\", suggestions.get(1).unwrap().value);
-    #[cfg(not(windows))]
-    assert_eq!("directory_completion/", suggestions.get(1).unwrap().value);
-
-    // Test overlay use completion
-    let completion_str = "overlay use ".to_string();
-    let suggestions = completer.complete(&completion_str, completion_str.len());
-
-    assert_eq!(2, suggestions.len());
-    assert_eq!("custom_completion.nu", suggestions.first().unwrap().value);
-    #[cfg(windows)]
-    assert_eq!("directory_completion\\", suggestions.get(1).unwrap().value);
-    #[cfg(not(windows))]
-    assert_eq!("directory_completion/", suggestions.get(1).unwrap().value);
 }
 
 #[test]
@@ -227,7 +208,6 @@ fn file_completions() {
     let expected_paths: Vec<String> = vec![
         folder(dir.join("another")),
         file(dir.join("custom_completion.nu")),
-        folder(dir.join("directory_completion")),
         file(dir.join("nushell")),
         folder(dir.join("test_a")),
         folder(dir.join("test_b")),
@@ -247,16 +227,6 @@ fn file_completions() {
 
     // Match the results
     match_suggestions(expected_paths, suggestions);
-
-    // Test completions for hidden files
-    let target_dir = format!("ls {}/.", folder(dir.join(".hidden_folder")));
-    let suggestions = completer.complete(&target_dir, target_dir.len());
-
-    let expected_paths: Vec<String> =
-        vec![file(dir.join(".hidden_folder").join(".hidden_subfile"))];
-
-    // Match the results
-    match_suggestions(expected_paths, suggestions);
 }
 
 #[test]
@@ -353,7 +323,6 @@ fn command_ls_with_filecompletion() {
     let expected_paths: Vec<String> = vec![
         "another\\".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
@@ -364,7 +333,6 @@ fn command_ls_with_filecompletion() {
     let expected_paths: Vec<String> = vec![
         "another/".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
@@ -387,7 +355,6 @@ fn command_open_with_filecompletion() {
     let expected_paths: Vec<String> = vec![
         "another\\".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
@@ -398,7 +365,6 @@ fn command_open_with_filecompletion() {
     let expected_paths: Vec<String> = vec![
         "another/".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
@@ -422,7 +388,6 @@ fn command_rm_with_globcompletion() {
     let expected_paths: Vec<String> = vec![
         "another\\".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
@@ -433,7 +398,6 @@ fn command_rm_with_globcompletion() {
     let expected_paths: Vec<String> = vec![
         "another/".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
@@ -457,7 +421,6 @@ fn command_cp_with_globcompletion() {
     let expected_paths: Vec<String> = vec![
         "another\\".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
@@ -468,7 +431,6 @@ fn command_cp_with_globcompletion() {
     let expected_paths: Vec<String> = vec![
         "another/".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
@@ -492,7 +454,6 @@ fn command_save_with_filecompletion() {
     let expected_paths: Vec<String> = vec![
         "another\\".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
@@ -503,7 +464,6 @@ fn command_save_with_filecompletion() {
     let expected_paths: Vec<String> = vec![
         "another/".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
@@ -527,7 +487,6 @@ fn command_touch_with_filecompletion() {
     let expected_paths: Vec<String> = vec![
         "another\\".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
@@ -538,7 +497,6 @@ fn command_touch_with_filecompletion() {
     let expected_paths: Vec<String> = vec![
         "another/".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
@@ -562,7 +520,6 @@ fn command_watch_with_filecompletion() {
     let expected_paths: Vec<String> = vec![
         "another\\".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
@@ -573,7 +530,6 @@ fn command_watch_with_filecompletion() {
     let expected_paths: Vec<String> = vec![
         "another/".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
@@ -594,7 +550,6 @@ fn file_completion_quoted() {
     let suggestions = completer.complete(target_dir, target_dir.len());
 
     let expected_paths: Vec<String> = vec![
-        "\'[a] bc.txt\'".to_string(),
         "`--help`".to_string(),
         "`-42`".to_string(),
         "`-inf`".to_string(),
@@ -670,7 +625,6 @@ fn folder_with_directorycompletions() {
     // Create the expected values
     let expected_paths: Vec<String> = vec![
         folder(dir.join("another")),
-        folder(dir.join("directory_completion")),
         folder(dir.join("test_a")),
         folder(dir.join("test_b")),
         folder(dir.join(".hidden_folder")),
@@ -695,14 +649,13 @@ fn variables_completions() {
     // Test completions for $nu
     let suggestions = completer.complete("$nu.", 4);
 
-    assert_eq!(15, suggestions.len());
+    assert_eq!(14, suggestions.len());
 
     let expected: Vec<String> = vec![
         "config-path".into(),
         "current-exe".into(),
         "default-config-dir".into(),
         "env-path".into(),
-        "history-enabled".into(),
         "history-path".into(),
         "home-path".into(),
         "is-interactive".into(),
@@ -721,13 +674,9 @@ fn variables_completions() {
     // Test completions for $nu.h (filter)
     let suggestions = completer.complete("$nu.h", 5);
 
-    assert_eq!(3, suggestions.len());
+    assert_eq!(2, suggestions.len());
 
-    let expected: Vec<String> = vec![
+    let expected: Vec<String> = vec!["history-path".into(), "home-path".into()];
-        "history-enabled".into(),
-        "history-path".into(),
-        "home-path".into(),
-    ];
 
     // Match results
     match_suggestions(expected, suggestions);
@@ -890,7 +839,6 @@ fn unknown_command_completion() {
     let expected_paths: Vec<String> = vec![
         "another\\".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
@@ -901,7 +849,6 @@ fn unknown_command_completion() {
     let expected_paths: Vec<String> = vec![
         "another/".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),
@@ -952,7 +899,6 @@ fn filecompletions_triggers_after_cursor() {
     let expected_paths: Vec<String> = vec![
         "another\\".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion\\".to_string(),
         "nushell".to_string(),
         "test_a\\".to_string(),
         "test_b\\".to_string(),
@@ -963,7 +909,6 @@ fn filecompletions_triggers_after_cursor() {
     let expected_paths: Vec<String> = vec![
         "another/".to_string(),
         "custom_completion.nu".to_string(),
-        "directory_completion/".to_string(),
         "nushell".to_string(),
         "test_a/".to_string(),
         "test_b/".to_string(),

crates/nu-cmd-base/Cargo.toml

@@ -5,17 +5,21 @@ edition = "2021"
 license = "MIT"
 name = "nu-cmd-base"
 repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-base"
-version = "0.91.0"
+version = "0.88.2"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nu-engine = { path = "../nu-engine", version = "0.91.0" }
+nu-engine = { path = "../nu-engine", version = "0.88.2" }
-nu-parser = { path = "../nu-parser", version = "0.91.0" }
+nu-glob = { path = "../nu-glob", version = "0.88.2" }
-nu-path = { path = "../nu-path", version = "0.91.0" }
+nu-parser = { path = "../nu-parser", version = "0.88.2" }
-nu-protocol = { path = "../nu-protocol", version = "0.91.0" }
+nu-path = { path = "../nu-path", version = "0.88.2" }
+nu-protocol = { path = "../nu-protocol", version = "0.88.2" }
+nu-utils = { path = "../nu-utils", version = "0.88.2" }
 
-indexmap = "2.2"
+indexmap = "2.1"
-miette = "7.1.0"
+miette = "5.10.0"
 
 [dev-dependencies]
+nu-test-support = { path = "../nu-test-support", version = "0.88.2" }
+rstest = "0.18.2"

crates/nu-cmd-base/src/arg_glob.rs

@@ -0,0 +1,205 @@
+// utilities for expanding globs in command arguments
+
+use nu_glob::{glob_with_parent, MatchOptions, Paths};
+use nu_protocol::{ShellError, Spanned};
+use std::fs;
+use std::path::{Path, PathBuf};
+
+// standard glob options to use for filesystem command arguments
+
+const GLOB_PARAMS: MatchOptions = MatchOptions {
+    case_sensitive: true,
+    require_literal_separator: false,
+    require_literal_leading_dot: false,
+    recursive_match_hidden_dir: true,
+};
+
+// handle an argument that could be a literal path or a glob.
+// if literal path, return just that (whether user can access it or not).
+// if glob, expand into matching paths, using GLOB_PARAMS options.
+pub fn arg_glob(
+    pattern: &Spanned<String>, // alleged path or glob
+    cwd: &Path,                // current working directory
+) -> Result<Paths, ShellError> {
+    arg_glob_opt(pattern, cwd, GLOB_PARAMS)
+}
+
+// variant of [arg_glob] that requires literal dot prefix in pattern to match dot-prefixed path.
+pub fn arg_glob_leading_dot(pattern: &Spanned<String>, cwd: &Path) -> Result<Paths, ShellError> {
+    arg_glob_opt(
+        pattern,
+        cwd,
+        MatchOptions {
+            require_literal_leading_dot: true,
+            ..GLOB_PARAMS
+        },
+    )
+}
+
+fn arg_glob_opt(
+    pattern: &Spanned<String>,
+    cwd: &Path,
+    options: MatchOptions,
+) -> Result<Paths, ShellError> {
+    // remove ansi coloring (?)
+    let pattern = {
+        Spanned {
+            item: nu_utils::strip_ansi_string_unlikely(pattern.item.clone()),
+            span: pattern.span,
+        }
+    };
+
+    // if there's a file with same path as the pattern, just return that.
+    let pp = cwd.join(&pattern.item);
+    let md = fs::metadata(pp);
+    #[allow(clippy::single_match)]
+    match md {
+        Ok(_metadata) => {
+            return Ok(Paths::single(&PathBuf::from(pattern.item), cwd));
+        }
+        // file not found, but also "invalid chars in file" (e.g * on Windows).  Fall through and glob
+        Err(_) => {}
+    }
+
+    // user wasn't referring to a specific thing in filesystem, try to glob it.
+    match glob_with_parent(&pattern.item, options, cwd) {
+        Ok(p) => Ok(p),
+        Err(pat_err) => Err(ShellError::InvalidGlobPattern {
+            msg: pat_err.msg.into(),
+            span: pattern.span,
+        }),
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use nu_glob::GlobResult;
+    use nu_protocol::{Span, Spanned};
+    use nu_test_support::fs::Stub::EmptyFile;
+    use nu_test_support::playground::Playground;
+    use rstest::rstest;
+
+    fn spanned_string(str: &str) -> Spanned<String> {
+        Spanned {
+            item: str.to_string(),
+            span: Span::test_data(),
+        }
+    }
+
+    #[test]
+    fn does_something() {
+        let act = arg_glob(&spanned_string("*"), &PathBuf::from("."));
+        assert!(act.is_ok());
+        for f in act.expect("checked ok") {
+            match f {
+                Ok(p) => {
+                    assert!(!p.to_str().unwrap().is_empty());
+                }
+                Err(e) => panic!("unexpected error {:?}", e),
+            };
+        }
+    }
+
+    #[test]
+    fn glob_format_error() {
+        let act = arg_glob(&spanned_string(r#"ab]c[def"#), &PathBuf::from("."));
+        assert!(act.is_err());
+    }
+
+    #[rstest]
+    #[case("*", 4, "no dirs")]
+    #[case("**/*", 7, "incl dirs")]
+    fn glob_subdirs(#[case] pat: &str, #[case] exp_count: usize, #[case] case: &str) {
+        Playground::setup("glob_subdirs", |dirs, sandbox| {
+            sandbox.with_files(vec![
+                EmptyFile("yehuda.txt"),
+                EmptyFile("jttxt"),
+                EmptyFile("andres.txt"),
+            ]);
+            sandbox.mkdir(".children");
+            sandbox.within(".children").with_files(vec![
+                EmptyFile("timothy.txt"),
+                EmptyFile("tiffany.txt"),
+                EmptyFile("trish.txt"),
+            ]);
+
+            let p: Vec<GlobResult> = arg_glob(&spanned_string(pat), &dirs.test)
+                .expect("no error")
+                .collect();
+            assert_eq!(
+                exp_count,
+                p.iter().filter(|i| i.is_ok()).count(),
+                " case: {case} ",
+            );
+
+            // expected behavior -- that directories are included in results (if name matches pattern)
+            let t = p
+                .iter()
+                .any(|i| i.as_ref().unwrap().to_string_lossy().contains(".children"));
+            assert!(t, "check for dir, case {case}");
+        })
+    }
+
+    #[rstest]
+    #[case("yehuda.txt", true, 1, "matches literal path")]
+    #[case("*", false, 3, "matches glob")]
+    #[case(r#"bad[glob.foo"#, true, 1, "matches literal, would be bad glob pat")]
+    fn exact_vs_glob(
+        #[case] pat: &str,
+        #[case] exp_matches_input: bool,
+        #[case] exp_count: usize,
+        #[case] case: &str,
+    ) {
+        Playground::setup("exact_vs_glob", |dirs, sandbox| {
+            sandbox.with_files(vec![
+                EmptyFile("yehuda.txt"),
+                EmptyFile("jttxt"),
+                EmptyFile("bad[glob.foo"),
+            ]);
+
+            let res = arg_glob(&spanned_string(pat), &dirs.test)
+                .expect("no error")
+                .collect::<Vec<GlobResult>>();
+
+            eprintln!("res: {:?}", res);
+            if exp_matches_input {
+                assert_eq!(
+                    exp_count,
+                    res.len(),
+                    " case {case}: matches input, but count not 1? "
+                );
+                assert_eq!(
+                    &res[0].as_ref().unwrap().to_string_lossy(),
+                    pat, // todo: is it OK for glob to return relative paths (not to current cwd, but to arg cwd of arg_glob)?
+                );
+            } else {
+                assert_eq!(exp_count, res.len(), " case: {}: matched glob", case);
+            }
+        })
+    }
+
+    #[rstest]
+    #[case(r#"realbad[glob.foo"#, true, 1, "error, bad glob")]
+    fn exact_vs_bad_glob(
+        // if path doesn't exist but pattern is not valid glob, should get error.
+        #[case] pat: &str,
+        #[case] _exp_matches_input: bool,
+        #[case] _exp_count: usize,
+        #[case] _tag: &str,
+    ) {
+        Playground::setup("exact_vs_bad_glob", |dirs, sandbox| {
+            sandbox.with_files(vec![
+                EmptyFile("yehuda.txt"),
+                EmptyFile("jttxt"),
+                EmptyFile("bad[glob.foo"),
+            ]);
+
+            let res = arg_glob(&spanned_string(pat), &dirs.test);
+            assert!(res
+                .expect_err("expected error")
+                .to_string()
+                .contains("Invalid glob pattern"));
+        })
+    }
+}

crates/nu-cmd-base/src/hook.rs

@@ -150,7 +150,7 @@ pub fn eval_hook(
             // If it returns true (the default if a condition block is not specified), the hook should be run.
             let do_run_hook = if let Some(condition) = val.get("condition") {
                 let other_span = condition.span();
-                if let Ok(block_id) = condition.coerce_block() {
+                if let Ok(block_id) = condition.as_block() {
                     match run_hook_block(
                         engine_state,
                         stack,

crates/nu-cmd-base/src/lib.rs

@@ -1,4 +1,7 @@
+mod arg_glob;
 pub mod formats;
 pub mod hook;
 pub mod input_handler;
 pub mod util;
+pub use arg_glob::arg_glob;
+pub use arg_glob::arg_glob_leading_dot;

crates/nu-cmd-base/src/util.rs

@@ -66,7 +66,7 @@ fn get_editor_commandline(
     match value {
         Value::String { val, .. } if !val.is_empty() => Ok((val.to_string(), Vec::new())),
         Value::List { vals, .. } if !vals.is_empty() => {
-            let mut editor_cmd = vals.iter().map(|l| l.coerce_string());
+            let mut editor_cmd = vals.iter().map(|l| l.as_string());
             match editor_cmd.next().transpose()? {
                 Some(editor) if !editor.is_empty() => {
                     let params = editor_cmd.collect::<Result<_, ShellError>>()?;

crates/nu-cmd-dataframe/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 license = "MIT"
 name = "nu-cmd-dataframe"
 repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-dataframe"
-version = "0.91.0"
+version = "0.88.2"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -13,23 +13,22 @@ version = "0.91.0"
 bench = false
 
 [dependencies]
-nu-engine = { path = "../nu-engine", version = "0.91.0" }
+nu-engine = { path = "../nu-engine", version = "0.88.2" }
-nu-parser = { path = "../nu-parser", version = "0.91.0" }
+nu-parser = { path = "../nu-parser", version = "0.88.2" }
-nu-protocol = { path = "../nu-protocol", version = "0.91.0" }
+nu-protocol = { path = "../nu-protocol", version = "0.88.2" }
 
 # Potential dependencies for extras
 chrono = { version = "0.4", features = ["std", "unstable-locales"], default-features = false }
 chrono-tz = "0.8"
-fancy-regex = "0.13"
+fancy-regex = "0.12"
-indexmap = { version = "2.2" }
+indexmap = { version = "2.1" }
 num = { version = "0.4", optional = true }
 serde = { version = "1.0", features = ["derive"] }
-sqlparser = { version = "0.43", optional = true }
+sqlparser = { version = "0.39", optional = true }
-polars-io = { version = "0.37", features = ["avro"], optional = true }
+polars-io = { version = "0.35", features = ["avro"], optional = true }
-polars-arrow = { version = "0.37", optional = true }
+polars-arrow = "0.35"
-polars-ops = { version = "0.37", optional = true }
+polars-ops = "0.35"
-polars-plan = { version = "0.37", features = ["regex"], optional = true }
+polars-plan = "0.35"
-polars-utils = { version = "0.37", optional = true }
 
 [dependencies.polars]
 features = [
@@ -63,11 +62,12 @@ features = [
 	"to_dummies",
 ]
 optional = true
-version = "0.37"
+version = "0.35"
 
 [features]
-dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"]
+dataframe = ["num", "polars", "polars-io", "sqlparser"]
 default = []
 
 [dev-dependencies]
-nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.91.0" }
+nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.88.2" }
+nu-test-support = { path = "../nu-test-support", version = "0.88.2" }

crates/nu-cmd-dataframe/src/dataframe/eager/append.rs

@@ -37,27 +37,24 @@ impl Command for AppendDF {
                 example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
     $a | dfr append $a"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
-                                vec![Value::test_int(1), Value::test_int(3)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "a_x".to_string(),
-                                "a_x".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
-                                vec![Value::test_int(1), Value::test_int(3)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_x".to_string(),
-                                "b_x".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -67,29 +64,26 @@ impl Command for AppendDF {
                 example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
     $a | dfr append $a --col"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![
-                                vec![
+                                Value::test_int(1),
-                                    Value::test_int(1),
+                                Value::test_int(3),
-                                    Value::test_int(3),
+                                Value::test_int(1),
-                                    Value::test_int(1),
+                                Value::test_int(3),
-                                    Value::test_int(3),
+                            ],
-                                ],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![
-                                vec![
+                                Value::test_int(2),
-                                    Value::test_int(2),
+                                Value::test_int(4),
-                                    Value::test_int(4),
+                                Value::test_int(2),
-                                    Value::test_int(2),
+                                Value::test_int(4),
-                                    Value::test_int(4),
+                            ],
-                                ],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -116,7 +110,7 @@ fn command(
 ) -> Result<PipelineData, ShellError> {
     let other: Value = call.req(engine_state, stack, 0)?;
 
-    let axis = if call.has_flag(engine_state, stack, "col")? {
+    let axis = if call.has_flag("col") {
         Axis::Column
     } else {
         Axis::Row

crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs

@@ -1,201 +0,0 @@
-use crate::dataframe::values::{str_to_dtype, NuExpression, NuLazyFrame};
-
-use super::super::values::NuDataFrame;
-use nu_engine::CallExt;
-use nu_protocol::{
-    ast::Call,
-    engine::{Command, EngineState, Stack},
-    record, Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
-};
-use polars::prelude::*;
-
-#[derive(Clone)]
-pub struct CastDF;
-
-impl Command for CastDF {
-    fn name(&self) -> &str {
-        "dfr cast"
-    }
-
-    fn usage(&self) -> &str {
-        "Cast a column to a different dtype."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .input_output_types(vec![
-                (
-                    Type::Custom("expression".into()),
-                    Type::Custom("expression".into()),
-                ),
-                (
-                    Type::Custom("dataframe".into()),
-                    Type::Custom("dataframe".into()),
-                ),
-            ])
-            .required(
-                "dtype",
-                SyntaxShape::String,
-                "The dtype to cast the column to",
-            )
-            .optional(
-                "column",
-                SyntaxShape::String,
-                "The column to cast. Required when used with a dataframe.",
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Cast a column in a dataframe to a different dtype",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr cast u8 a | dfr schema",
-                result: Some(Value::record(
-                    record! {
-                        "a" => Value::string("u8", Span::test_data()),
-                        "b" => Value::string("i64", Span::test_data()),
-                    },
-                    Span::test_data(),
-                )),
-            },
-            Example {
-                description: "Cast a column in a lazy dataframe to a different dtype",
-                example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-lazy | dfr cast u8 a | dfr schema",
-                result: Some(Value::record(
-                    record! {
-                        "a" => Value::string("u8", Span::test_data()),
-                        "b" => Value::string("i64", Span::test_data()),
-                    },
-                    Span::test_data(),
-                )),
-            },
-            Example {
-                description: "Cast a column in a expression to a different dtype",
-                example: r#"[[a b]; [1 2] [1 4]] | dfr into-df | dfr group-by a | dfr agg [ (dfr col b | dfr cast u8 | dfr min | dfr as "b_min") ] | dfr schema"#,
-                result: None
-            }
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        let value = input.into_value(call.head);
-        if NuLazyFrame::can_downcast(&value) {
-            let (dtype, column_nm) = df_args(engine_state, stack, call)?;
-            let df = NuLazyFrame::try_from_value(value)?;
-            command_lazy(call, column_nm, dtype, df)
-        } else if NuDataFrame::can_downcast(&value) {
-            let (dtype, column_nm) = df_args(engine_state, stack, call)?;
-            let df = NuDataFrame::try_from_value(value)?;
-            command_eager(call, column_nm, dtype, df)
-        } else {
-            let dtype: String = call.req(engine_state, stack, 0)?;
-            let dtype = str_to_dtype(&dtype, call.head)?;
-
-            let expr = NuExpression::try_from_value(value)?;
-            let expr: NuExpression = expr.into_polars().cast(dtype).into();
-
-            Ok(PipelineData::Value(
-                NuExpression::into_value(expr, call.head),
-                None,
-            ))
-        }
-    }
-}
-
-fn df_args(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<(DataType, String), ShellError> {
-    let dtype = dtype_arg(engine_state, stack, call)?;
-    let column_nm: String =
-        call.opt(engine_state, stack, 1)?
-            .ok_or(ShellError::MissingParameter {
-                param_name: "column_name".into(),
-                span: call.head,
-            })?;
-    Ok((dtype, column_nm))
-}
-
-fn dtype_arg(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-) -> Result<DataType, ShellError> {
-    let dtype: String = call.req(engine_state, stack, 0)?;
-    str_to_dtype(&dtype, call.head)
-}
-
-fn command_lazy(
-    call: &Call,
-    column_nm: String,
-    dtype: DataType,
-    lazy: NuLazyFrame,
-) -> Result<PipelineData, ShellError> {
-    let column = col(&column_nm).cast(dtype);
-    let lazy = lazy.into_polars().with_columns(&[column]);
-    let lazy = NuLazyFrame::new(false, lazy);
-
-    Ok(PipelineData::Value(
-        NuLazyFrame::into_value(lazy, call.head)?,
-        None,
-    ))
-}
-
-fn command_eager(
-    call: &Call,
-    column_nm: String,
-    dtype: DataType,
-    nu_df: NuDataFrame,
-) -> Result<PipelineData, ShellError> {
-    let mut df = nu_df.df;
-    let column = df
-        .column(&column_nm)
-        .map_err(|e| ShellError::GenericError {
-            error: format!("{e}"),
-            msg: "".into(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let casted = column.cast(&dtype).map_err(|e| ShellError::GenericError {
-        error: format!("{e}"),
-        msg: "".into(),
-        span: Some(call.head),
-        help: None,
-        inner: vec![],
-    })?;
-
-    let _ = df
-        .with_column(casted)
-        .map_err(|e| ShellError::GenericError {
-            error: format!("{e}"),
-            msg: "".into(),
-            span: Some(call.head),
-            help: None,
-            inner: vec![],
-        })?;
-
-    let df = NuDataFrame::new(false, df);
-    Ok(PipelineData::Value(df.into_value(call.head), None))
-}
-
-#[cfg(test)]
-mod test {
-
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(CastDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs

@@ -35,13 +35,10 @@ impl Command for DropDF {
             description: "drop column a",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr drop a",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![Column::new(
-                    vec![Column::new(
+                    "b".to_string(),
-                        "b".to_string(),
+                    vec![Value::test_int(2), Value::test_int(4)],
-                        vec![Value::test_int(2), Value::test_int(4)],
+                )])
-                    )],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs

@@ -46,19 +46,16 @@ impl Command for DropDuplicates {
             description: "drop duplicates",
             example: "[[a b]; [1 2] [3 4] [1 2]] | dfr into-df | dfr drop-duplicates",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_int(3), Value::test_int(1)],
-                            vec![Value::test_int(3), Value::test_int(1)],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_int(4), Value::test_int(2)],
-                            vec![Value::test_int(4), Value::test_int(2)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -95,7 +92,7 @@ fn command(
 
     let subset_slice = subset.as_ref().map(|cols| &cols[..]);
 
-    let keep_strategy = if call.has_flag(engine_state, stack, "last")? {
+    let keep_strategy = if call.has_flag("last") {
         UniqueKeepStrategy::Last
     } else {
         UniqueKeepStrategy::First

crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs

@@ -43,23 +43,20 @@ impl Command for DropNulls {
     let a = ($df | dfr with-column $res --name res);
     $a | dfr drop-nulls"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(1)],
-                                vec![Value::test_int(1), Value::test_int(1)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(2)],
-                                vec![Value::test_int(2), Value::test_int(2)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "res".to_string(),
-                                "res".to_string(),
+                            vec![Value::test_int(1), Value::test_int(1)],
-                                vec![Value::test_int(1), Value::test_int(1)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -69,18 +66,15 @@ impl Command for DropNulls {
                 example: r#"let s = ([1 2 0 0 3 4] | dfr into-df);
     ($s / $s) | dfr drop-nulls"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![Column::new(
-                        vec![Column::new(
+                        "div_0_0".to_string(),
-                            "div_0_0".to_string(),
+                        vec![
-                            vec![
+                            Value::test_int(1),
-                                Value::test_int(1),
+                            Value::test_int(1),
-                                Value::test_int(1),
+                            Value::test_int(1),
-                                Value::test_int(1),
+                            Value::test_int(1),
-                                Value::test_int(1),
+                        ],
-                            ],
+                    )])
-                        )],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs

@@ -31,19 +31,16 @@ impl Command for DataTypes {
             description: "Dataframe dtypes",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dtypes",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "column".to_string(),
-                            "column".to_string(),
+                        vec![Value::test_string("a"), Value::test_string("b")],
-                            vec![Value::test_string("a"), Value::test_string("b")],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "dtype".to_string(),
-                            "dtype".to_string(),
+                        vec![Value::test_string("i64"), Value::test_string("i64")],
-                            vec![Value::test_string("i64"), Value::test_string("i64")],
+                    ),
-                        ),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -82,7 +79,6 @@ fn command(
                 .dtype();
 
             let dtype_str = dtype.to_string();
-
             dtypes.push(Value::string(dtype_str, call.head));
 
             Value::string(*v, call.head)
@@ -92,7 +88,7 @@ fn command(
     let names_col = Column::new("column".to_string(), names);
     let dtypes_col = Column::new("dtype".to_string(), dtypes);
 
-    NuDataFrame::try_from_columns(vec![names_col, dtypes_col], None)
+    NuDataFrame::try_from_columns(vec![names_col, dtypes_col])
         .map(|df| PipelineData::Value(df.into_value(call.head), None))
 }
 

crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs

@@ -1,5 +1,4 @@
 use super::super::values::NuDataFrame;
-use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
@@ -79,12 +78,12 @@ impl Command for Dummies {
 }
 
 fn command(
-    engine_state: &EngineState,
+    _engine_state: &EngineState,
-    stack: &mut Stack,
+    _stack: &mut Stack,
     call: &Call,
     input: PipelineData,
 ) -> Result<PipelineData, ShellError> {
-    let drop_first: bool = call.has_flag(engine_state, stack, "drop-first")?;
+    let drop_first: bool = call.has_flag("drop-first");
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     df.as_ref()

crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs

@@ -43,13 +43,10 @@ impl Command for FilterWith {
                 example: r#"let mask = ([true false] | dfr into-df);
     [[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with $mask"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new("a".to_string(), vec![Value::test_int(1)]),
-                            Column::new("a".to_string(), vec![Value::test_int(1)]),
+                        Column::new("b".to_string(), vec![Value::test_int(2)]),
-                            Column::new("b".to_string(), vec![Value::test_int(2)]),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -58,13 +55,10 @@ impl Command for FilterWith {
                 description: "Filter dataframe using an expression",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with ((dfr col a) > 1)",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new("a".to_string(), vec![Value::test_int(3)]),
-                            Column::new("a".to_string(), vec![Value::test_int(3)]),
+                        Column::new("b".to_string(), vec![Value::test_int(4)]),
-                            Column::new("b".to_string(), vec![Value::test_int(4)]),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/first.rs

@@ -44,13 +44,10 @@ impl Command for FirstDF {
                 description: "Return the first row of a dataframe",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new("a".to_string(), vec![Value::test_int(1)]),
-                            Column::new("a".to_string(), vec![Value::test_int(1)]),
+                        Column::new("b".to_string(), vec![Value::test_int(2)]),
-                            Column::new("b".to_string(), vec![Value::test_int(2)]),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -59,19 +56,16 @@ impl Command for FirstDF {
                 description: "Return the first two rows of a dataframe",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first 2",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
-                                vec![Value::test_int(1), Value::test_int(3)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/get.rs

@@ -36,13 +36,10 @@ impl Command for GetDF {
             description: "Returns the selected column",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr get a",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![Column::new(
-                    vec![Column::new(
+                    "a".to_string(),
-                        "a".to_string(),
+                    vec![Value::test_int(1), Value::test_int(3)],
-                        vec![Value::test_int(1), Value::test_int(3)],
+                )])
-                    )],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/eager/last.rs

@@ -40,13 +40,10 @@ impl Command for LastDF {
                 description: "Create new dataframe with last rows",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr last 1",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new("a".to_string(), vec![Value::test_int(3)]),
-                            Column::new("a".to_string(), vec![Value::test_int(3)]),
+                        Column::new("b".to_string(), vec![Value::test_int(4)]),
-                            Column::new("b".to_string(), vec![Value::test_int(4)]),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs

@@ -106,7 +106,7 @@ impl Command for MeltDF {
                             Value::test_string("c"),
                         ],
                     ),
-                ], None)
+                ])
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs

@@ -1,5 +1,4 @@
 mod append;
-mod cast;
 mod columns;
 mod drop;
 mod drop_duplicates;
@@ -16,7 +15,6 @@ mod open;
 mod query_df;
 mod rename;
 mod sample;
-mod schema;
 mod shape;
 mod slice;
 mod sql_context;
@@ -36,7 +34,6 @@ use nu_protocol::engine::StateWorkingSet;
 
 pub use self::open::OpenDataFrame;
 pub use append::AppendDF;
-pub use cast::CastDF;
 pub use columns::ColumnsDF;
 pub use drop::DropDF;
 pub use drop_duplicates::DropDuplicates;
@@ -52,10 +49,10 @@ pub use melt::MeltDF;
 pub use query_df::QueryDf;
 pub use rename::RenameDF;
 pub use sample::SampleDF;
-pub use schema::SchemaDF;
 pub use shape::ShapeDF;
 pub use slice::SliceDF;
 pub use sql_context::SQLContext;
+pub use sql_expr::parse_sql_expr;
 pub use summary::Summary;
 pub use take::TakeDF;
 pub use to_arrow::ToArrow;
@@ -80,7 +77,6 @@ pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
     // Dataframe commands
     bind_command!(
         AppendDF,
-        CastDF,
         ColumnsDF,
         DataTypes,
         Summary,
@@ -98,7 +94,6 @@ pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
         QueryDf,
         RenameDF,
         SampleDF,
-        SchemaDF,
         ShapeDF,
         SliceDF,
         TakeDF,

crates/nu-cmd-dataframe/src/dataframe/eager/open.rs

@@ -1,5 +1,3 @@
-use crate::dataframe::values::NuSchema;
-
 use super::super::values::{NuDataFrame, NuLazyFrame};
 use nu_engine::CallExt;
 use nu_protocol::{
@@ -72,12 +70,6 @@ impl Command for OpenDataFrame {
                 "Columns to be selected from csv file. CSV and Parquet file",
                 None,
             )
-            .named(
-                "schema",
-                SyntaxShape::Record(vec![]),
-                r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
-                Some('s')
-            )
             .input_output_type(Type::Any, Type::Custom("dataframe".into()))
             .category(Category::Custom("dataframe".into()))
     }
@@ -147,14 +139,14 @@ fn from_parquet(
     stack: &mut Stack,
     call: &Call,
 ) -> Result<Value, ShellError> {
-    if call.has_flag(engine_state, stack, "lazy")? {
+    if call.has_flag("lazy") {
         let file: String = call.req(engine_state, stack, 0)?;
         let args = ScanArgsParquet {
             n_rows: None,
             cache: true,
             parallel: ParallelStrategy::Auto,
             rechunk: false,
-            row_index: None,
+            row_count: None,
             low_memory: false,
             cloud_options: None,
             use_statistics: false,
@@ -246,13 +238,13 @@ fn from_ipc(
     stack: &mut Stack,
     call: &Call,
 ) -> Result<Value, ShellError> {
-    if call.has_flag(engine_state, stack, "lazy")? {
+    if call.has_flag("lazy") {
         let file: String = call.req(engine_state, stack, 0)?;
         let args = ScanArgsIpc {
             n_rows: None,
             cache: true,
             rechunk: false,
-            row_index: None,
+            row_count: None,
             memmap: true,
         };
 
@@ -313,19 +305,10 @@ fn from_json(
         help: None,
         inner: vec![],
     })?;
-    let maybe_schema = call
-        .get_flag(engine_state, stack, "schema")?
-        .map(|schema| NuSchema::try_from(&schema))
-        .transpose()?;
 
     let buf_reader = BufReader::new(file);
     let reader = JsonReader::new(buf_reader);
 
-    let reader = match maybe_schema {
-        Some(schema) => reader.with_schema(schema.into()),
-        None => reader,
-    };
-
     let df: NuDataFrame = reader
         .finish()
         .map_err(|e| ShellError::GenericError {
@@ -346,10 +329,6 @@ fn from_jsonl(
     call: &Call,
 ) -> Result<Value, ShellError> {
     let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
-    let maybe_schema = call
-        .get_flag(engine_state, stack, "schema")?
-        .map(|schema| NuSchema::try_from(&schema))
-        .transpose()?;
     let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
     let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
         error: "Error opening file".into(),
@@ -364,11 +343,6 @@ fn from_jsonl(
         .with_json_format(JsonFormat::JsonLines)
         .infer_schema_len(infer_schema);
 
-    let reader = match maybe_schema {
-        Some(schema) => reader.with_schema(schema.into()),
-        None => reader,
-    };
-
     let df: NuDataFrame = reader
         .finish()
         .map_err(|e| ShellError::GenericError {
@@ -389,17 +363,12 @@ fn from_csv(
     call: &Call,
 ) -> Result<Value, ShellError> {
     let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
-    let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
+    let no_header: bool = call.has_flag("no-header");
     let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
     let skip_rows: Option<usize> = call.get_flag(engine_state, stack, "skip-rows")?;
     let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
 
-    let maybe_schema = call
+    if call.has_flag("lazy") {
-        .get_flag(engine_state, stack, "schema")?
-        .map(|schema| NuSchema::try_from(&schema))
-        .transpose()?;
-
-    if call.has_flag(engine_state, stack, "lazy")? {
         let file: String = call.req(engine_state, stack, 0)?;
         let csv_reader = LazyCsvReader::new(file);
 
@@ -426,11 +395,6 @@ fn from_csv(
 
         let csv_reader = csv_reader.has_header(!no_header);
 
-        let csv_reader = match maybe_schema {
-            Some(schema) => csv_reader.with_schema(Some(schema.into())),
-            None => csv_reader,
-        };
-
         let csv_reader = match infer_schema {
             None => csv_reader,
             Some(r) => csv_reader.with_infer_schema_length(Some(r)),
@@ -488,11 +452,6 @@ fn from_csv(
 
         let csv_reader = csv_reader.has_header(!no_header);
 
-        let csv_reader = match maybe_schema {
-            Some(schema) => csv_reader.with_schema(Some(schema.into())),
-            None => csv_reader,
-        };
-
         let csv_reader = match infer_schema {
             None => csv_reader,
             Some(r) => csv_reader.infer_schema(Some(r)),

crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs

@@ -44,13 +44,10 @@ impl Command for QueryDf {
             description: "Query dataframe using SQL",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr query 'select a from df'",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![Column::new(
-                    vec![Column::new(
+                    "a".to_string(),
-                        "a".to_string(),
+                    vec![Value::test_int(1), Value::test_int(3)],
-                        vec![Value::test_int(1), Value::test_int(3)],
+                )])
-                    )],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs

@@ -46,18 +46,15 @@ impl Command for RenameDF {
                 description: "Renames a series",
                 example: "[5 6 7 8] | dfr into-df | dfr rename '0' new_name",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![Column::new(
-                        vec![Column::new(
+                        "new_name".to_string(),
-                            "new_name".to_string(),
+                        vec![
-                            vec![
+                            Value::test_int(5),
-                                Value::test_int(5),
+                            Value::test_int(6),
-                                Value::test_int(6),
+                            Value::test_int(7),
-                                Value::test_int(7),
+                            Value::test_int(8),
-                                Value::test_int(8),
+                        ],
-                            ],
+                    )])
-                        )],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -66,19 +63,16 @@ impl Command for RenameDF {
                 description: "Renames a dataframe column",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename a a_new",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a_new".to_string(),
-                                "a_new".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
-                                vec![Value::test_int(1), Value::test_int(3)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -87,19 +81,16 @@ impl Command for RenameDF {
                 description: "Renames two dataframe columns",
                 example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename [a b] [a_new b_new]",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a_new".to_string(),
-                                "a_new".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
-                                vec![Value::test_int(1), Value::test_int(3)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_new".to_string(),
-                                "b_new".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs

@@ -88,8 +88,8 @@ fn command(
     let seed: Option<u64> = call
         .get_flag::<i64>(engine_state, stack, "seed")?
         .map(|val| val as u64);
-    let replace: bool = call.has_flag(engine_state, stack, "replace")?;
+    let replace: bool = call.has_flag("replace");
-    let shuffle: bool = call.has_flag(engine_state, stack, "shuffle")?;
+    let shuffle: bool = call.has_flag("shuffle");
 
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 

crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs

@@ -1,117 +0,0 @@
-use super::super::values::NuDataFrame;
-use nu_engine::CallExt;
-use nu_protocol::{
-    ast::Call,
-    engine::{Command, EngineState, Stack},
-    record, Category, Example, PipelineData, ShellError, Signature, Span, Type, Value,
-};
-
-#[derive(Clone)]
-pub struct SchemaDF;
-
-impl Command for SchemaDF {
-    fn name(&self) -> &str {
-        "dfr schema"
-    }
-
-    fn usage(&self) -> &str {
-        "Show schema for a dataframe."
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .switch("datatype-list", "creates a lazy dataframe", Some('l'))
-            .input_output_type(
-                Type::Custom("dataframe".into()),
-                Type::Custom("dataframe".into()),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Dataframe schema",
-            example: r#"[[a b]; [1 "foo"] [3 "bar"]] | dfr into-df | dfr schema"#,
-            result: Some(Value::record(
-                record! {
-                    "a" => Value::string("i64", Span::test_data()),
-                    "b" => Value::string("str", Span::test_data()),
-                },
-                Span::test_data(),
-            )),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        if call.has_flag(engine_state, stack, "datatype-list")? {
-            Ok(PipelineData::Value(datatype_list(Span::unknown()), None))
-        } else {
-            command(engine_state, stack, call, input)
-        }
-    }
-}
-
-fn command(
-    _engine_state: &EngineState,
-    _stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-    let schema = df.schema();
-    let value: Value = schema.into();
-    Ok(PipelineData::Value(value, None))
-}
-
-fn datatype_list(span: Span) -> Value {
-    let types: Vec<Value> = [
-        ("null", ""),
-        ("bool", ""),
-        ("u8", ""),
-        ("u16", ""),
-        ("u32", ""),
-        ("u64", ""),
-        ("i8", ""),
-        ("i16", ""),
-        ("i32", ""),
-        ("i64", ""),
-        ("f32", ""),
-        ("f64", ""),
-        ("str", ""),
-        ("binary", ""),
-        ("date", ""),
-        ("datetime<time_unit: (ms, us, ns) timezone (optional)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns. Timezone wildcard is *. Other Timezone examples: UTC, America/Los_Angeles."),
-        ("duration<time_unit: (ms, us, ns)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns."),
-        ("time", ""),
-        ("object", ""),
-        ("unknown", ""),
-        ("list<dtype>", ""),
-    ]
-    .iter()
-    .map(|(dtype, note)| {
-        Value::record(record! {
-            "dtype" => Value::string(*dtype, span),
-            "note" => Value::string(*note, span),
-        },
-        span)
-    })
-    .collect();
-    Value::list(types, span)
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(SchemaDF {})])
-    }
-}

crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs

@@ -34,13 +34,10 @@ impl Command for ShapeDF {
             description: "Shows row and column shape",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr shape",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new("rows".to_string(), vec![Value::test_int(2)]),
-                        Column::new("rows".to_string(), vec![Value::test_int(2)]),
+                    Column::new("columns".to_string(), vec![Value::test_int(2)]),
-                        Column::new("columns".to_string(), vec![Value::test_int(2)]),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -73,7 +70,7 @@ fn command(
     let rows_col = Column::new("rows".to_string(), vec![rows]);
     let cols_col = Column::new("columns".to_string(), vec![cols]);
 
-    NuDataFrame::try_from_columns(vec![rows_col, cols_col], None)
+    NuDataFrame::try_from_columns(vec![rows_col, cols_col])
         .map(|df| PipelineData::Value(df.into_value(call.head), None))
 }
 

crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs

@@ -37,13 +37,10 @@ impl Command for SliceDF {
             description: "Create new dataframe from a slice of the rows",
             example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr slice 0 1",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new("a".to_string(), vec![Value::test_int(1)]),
-                        Column::new("a".to_string(), vec![Value::test_int(1)]),
+                    Column::new("b".to_string(), vec![Value::test_int(2)]),
-                        Column::new("b".to_string(), vec![Value::test_int(2)]),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs

@@ -13,7 +13,7 @@ fn map_sql_polars_datatype(data_type: &SQLDataType) -> Result<DataType> {
         | SQLDataType::Uuid
         | SQLDataType::Clob(_)
         | SQLDataType::Text
-        | SQLDataType::String(_) => DataType::String,
+        | SQLDataType::String(_) => DataType::Utf8,
         SQLDataType::Float(_) => DataType::Float32,
         SQLDataType::Real => DataType::Float32,
         SQLDataType::Double => DataType::Float64,
@@ -62,9 +62,7 @@ fn binary_op_(left: Expr, right: Expr, op: &SQLBinaryOperator) -> Result<Expr> {
         SQLBinaryOperator::Multiply => left * right,
         SQLBinaryOperator::Divide => left / right,
         SQLBinaryOperator::Modulo => left % right,
-        SQLBinaryOperator::StringConcat => {
+        SQLBinaryOperator::StringConcat => left.cast(DataType::Utf8) + right.cast(DataType::Utf8),
-            left.cast(DataType::String) + right.cast(DataType::String)
-        }
         SQLBinaryOperator::Gt => left.gt(right),
         SQLBinaryOperator::Lt => left.lt(right),
         SQLBinaryOperator::GtEq => left.gt_eq(right),

crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs

@@ -10,7 +10,7 @@ use polars::{
     chunked_array::ChunkedArray,
     prelude::{
         AnyValue, DataFrame, DataType, Float64Type, IntoSeries, NewChunkedArray,
-        QuantileInterpolOptions, Series, StringType,
+        QuantileInterpolOptions, Series, Utf8Type,
     },
 };
 
@@ -46,56 +46,53 @@ impl Command for Summary {
             description: "list dataframe descriptives",
             example: "[[a b]; [1 1] [1 1]] | dfr into-df | dfr summary",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "descriptor".to_string(),
-                            "descriptor".to_string(),
+                        vec![
-                            vec![
+                            Value::test_string("count"),
-                                Value::test_string("count"),
+                            Value::test_string("sum"),
-                                Value::test_string("sum"),
+                            Value::test_string("mean"),
-                                Value::test_string("mean"),
+                            Value::test_string("median"),
-                                Value::test_string("median"),
+                            Value::test_string("std"),
-                                Value::test_string("std"),
+                            Value::test_string("min"),
-                                Value::test_string("min"),
+                            Value::test_string("25%"),
-                                Value::test_string("25%"),
+                            Value::test_string("50%"),
-                                Value::test_string("50%"),
+                            Value::test_string("75%"),
-                                Value::test_string("75%"),
+                            Value::test_string("max"),
-                                Value::test_string("max"),
+                        ],
-                            ],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "a (i64)".to_string(),
-                            "a (i64)".to_string(),
+                        vec![
-                            vec![
+                            Value::test_float(2.0),
-                                Value::test_float(2.0),
+                            Value::test_float(2.0),
-                                Value::test_float(2.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(0.0),
-                                Value::test_float(0.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                        ],
-                            ],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b (i64)".to_string(),
-                            "b (i64)".to_string(),
+                        vec![
-                            vec![
+                            Value::test_float(2.0),
-                                Value::test_float(2.0),
+                            Value::test_float(2.0),
-                                Value::test_float(2.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(0.0),
-                                Value::test_float(0.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                            Value::test_float(1.0),
-                                Value::test_float(1.0),
+                        ],
-                            ],
+                    ),
-                        ),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -174,7 +171,7 @@ fn command(
 
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
-    let names = ChunkedArray::<StringType>::from_slice_options("descriptor", &labels).into_series();
+    let names = ChunkedArray::<Utf8Type>::from_slice_options("descriptor", &labels).into_series();
 
     let head = std::iter::once(names);
 
@@ -182,50 +179,42 @@ fn command(
         .as_ref()
         .get_columns()
         .iter()
-        .filter(|col| !matches!(col.dtype(), &DataType::Object("object", _)))
+        .filter(|col| col.dtype() != &DataType::Object("object"))
         .map(|col| {
             let count = col.len() as f64;
 
-            let sum = col.sum_as_series().ok().and_then(|series| {
+            let sum = col
-                series
+                .sum_as_series()
-                    .cast(&DataType::Float64)
+                .cast(&DataType::Float64)
-                    .ok()
+                .ok()
-                    .and_then(|ca| match ca.get(0) {
+                .and_then(|ca| match ca.get(0) {
-                        Ok(AnyValue::Float64(v)) => Some(v),
+                    Ok(AnyValue::Float64(v)) => Some(v),
-                        _ => None,
+                    _ => None,
-                    })
+                });
-            });
 
             let mean = match col.mean_as_series().get(0) {
                 Ok(AnyValue::Float64(v)) => Some(v),
                 _ => None,
             };
 
-            let median = match col.median_as_series() {
+            let median = match col.median_as_series().get(0) {
-                Ok(v) => match v.get(0) {
+                Ok(AnyValue::Float64(v)) => Some(v),
-                    Ok(AnyValue::Float64(v)) => Some(v),
-                    _ => None,
-                },
                 _ => None,
             };
 
-            let std = match col.std_as_series(0) {
+            let std = match col.std_as_series(0).get(0) {
-                Ok(v) => match v.get(0) {
+                Ok(AnyValue::Float64(v)) => Some(v),
-                    Ok(AnyValue::Float64(v)) => Some(v),
-                    _ => None,
-                },
                 _ => None,
             };
 
-            let min = col.min_as_series().ok().and_then(|series| {
+            let min = col
-                series
+                .min_as_series()
-                    .cast(&DataType::Float64)
+                .cast(&DataType::Float64)
-                    .ok()
+                .ok()
-                    .and_then(|ca| match ca.get(0) {
+                .and_then(|ca| match ca.get(0) {
-                        Ok(AnyValue::Float64(v)) => Some(v),
+                    Ok(AnyValue::Float64(v)) => Some(v),
-                        _ => None,
+                    _ => None,
-                    })
+                });
-            });
 
             let mut quantiles = quantiles
                 .clone()
@@ -241,15 +230,14 @@ fn command(
                 })
                 .collect::<Vec<Option<f64>>>();
 
-            let max = col.max_as_series().ok().and_then(|series| {
+            let max = col
-                series
+                .max_as_series()
-                    .cast(&DataType::Float64)
+                .cast(&DataType::Float64)
-                    .ok()
+                .ok()
-                    .and_then(|ca| match ca.get(0) {
+                .and_then(|ca| match ca.get(0) {
-                        Ok(AnyValue::Float64(v)) => Some(v),
+                    Ok(AnyValue::Float64(v)) => Some(v),
-                        _ => None,
+                    _ => None,
-                    })
+                });
-            });
 
             let mut descriptors = vec![Some(count), sum, mean, median, std, min];
             descriptors.append(&mut quantiles);

crates/nu-cmd-dataframe/src/dataframe/eager/take.rs

@@ -44,19 +44,16 @@ impl Command for TakeDF {
     let indices = ([0 2] | dfr into-df);
     $df | dfr take $indices"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(4), Value::test_int(4)],
-                                vec![Value::test_int(4), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
-                                vec![Value::test_int(1), Value::test_int(3)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -67,13 +64,10 @@ impl Command for TakeDF {
     let indices = ([0 2] | dfr into-df);
     $series | dfr take $indices"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![Column::new(
-                        vec![Column::new(
+                        "0".to_string(),
-                            "0".to_string(),
+                        vec![Value::test_int(4), Value::test_int(5)],
-                            vec![Value::test_int(4), Value::test_int(5)],
+                    )])
-                        )],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs

@@ -70,7 +70,7 @@ fn command(
 ) -> Result<PipelineData, ShellError> {
     let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
     let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
-    let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
+    let no_header: bool = call.has_flag("no-header");
 
     let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
 

crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs

@@ -1,14 +1,10 @@
-use crate::dataframe::values::NuSchema;
-
 use super::super::values::{Column, NuDataFrame};
 
-use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Type, Value,
+    Category, Example, PipelineData, ShellError, Signature, Span, Type, Value,
 };
-use polars::prelude::*;
 
 #[derive(Clone)]
 pub struct ToDataFrame;
@@ -24,12 +20,6 @@ impl Command for ToDataFrame {
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
-            .named(
-                "schema",
-                SyntaxShape::Record(vec![]),
-                r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
-                Some('s'),
-            )
             .input_output_type(Type::Any, Type::Custom("dataframe".into()))
             .category(Category::Custom("dataframe".into()))
     }
@@ -40,19 +30,16 @@ impl Command for ToDataFrame {
                 description: "Takes a dictionary and creates a dataframe",
                 example: "[[a b];[1 2] [3 4]] | dfr into-df",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
-                                vec![Value::test_int(1), Value::test_int(3)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -61,27 +48,24 @@ impl Command for ToDataFrame {
                 description: "Takes a list of tables and creates a dataframe",
                 example: "[[1 2 a] [3 4 b] [5 6 c]] | dfr into-df",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "0".to_string(),
-                                "0".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3), Value::test_int(5)],
-                                vec![Value::test_int(1), Value::test_int(3), Value::test_int(5)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "1".to_string(),
-                                "1".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4), Value::test_int(6)],
-                                vec![Value::test_int(2), Value::test_int(4), Value::test_int(6)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "2".to_string(),
-                                "2".to_string(),
+                            vec![
-                                vec![
+                                Value::test_string("a"),
-                                    Value::test_string("a"),
+                                Value::test_string("b"),
-                                    Value::test_string("b"),
+                                Value::test_string("c"),
-                                    Value::test_string("c"),
+                            ],
-                                ],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -90,17 +74,14 @@ impl Command for ToDataFrame {
                 description: "Takes a list and creates a dataframe",
                 example: "[a b c] | dfr into-df",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![Column::new(
-                        vec![Column::new(
+                        "0".to_string(),
-                            "0".to_string(),
+                        vec![
-                            vec![
+                            Value::test_string("a"),
-                                Value::test_string("a"),
+                            Value::test_string("b"),
-                                Value::test_string("b"),
+                            Value::test_string("c"),
-                                Value::test_string("c"),
+                        ],
-                            ],
+                    )])
-                        )],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -109,78 +90,30 @@ impl Command for ToDataFrame {
                 description: "Takes a list of booleans and creates a dataframe",
                 example: "[true true false] | dfr into-df",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![Column::new(
-                        vec![Column::new(
+                        "0".to_string(),
-                            "0".to_string(),
+                        vec![
-                            vec![
+                            Value::test_bool(true),
-                                Value::test_bool(true),
+                            Value::test_bool(true),
-                                Value::test_bool(true),
+                            Value::test_bool(false),
-                                Value::test_bool(false),
+                        ],
-                            ],
+                    )])
-                        )],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
             },
-            Example {
-                description: "Convert to a dataframe and provide a schema",
-                example: "{a: 1, b: {a: [1 2 3]}, c: [a b c]}| dfr into-df -s {a: u8, b: {a: list<u64>}, c: list<str>}",
-                result: Some(
-                    NuDataFrame::try_from_series(vec![
-                        Series::new("a", &[1u8]),
-                        {
-                            let dtype = DataType::Struct(vec![Field::new("a", DataType::List(Box::new(DataType::UInt64)))]);
-                            let vals = vec![AnyValue::StructOwned(
-                                Box::new((vec![AnyValue::List(Series::new("a", &[1u64, 2, 3]))], vec![Field::new("a", DataType::String)]))); 1];
-                            Series::from_any_values_and_dtype("b", &vals, &dtype, false)
-                                .expect("Struct series should not fail")
-                        },
-                        {
-                            let dtype = DataType::List(Box::new(DataType::String));
-                            let vals = vec![AnyValue::List(Series::new("c", &["a", "b", "c"]))];
-                            Series::from_any_values_and_dtype("c", &vals, &dtype, false)
-                                .expect("List series should not fail")
-                        }
-                    ], Span::test_data())
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Convert to a dataframe and provide a schema that adds a new column",
-                example: r#"[[a b]; [1 "foo"] [2 "bar"]] | dfr into-df -s {a: u8, b:str, c:i64} | dfr fill-null 3"#,
-                result: Some(NuDataFrame::try_from_series(vec![
-                        Series::new("a", [1u8, 2]),
-                        Series::new("b", ["foo", "bar"]),
-                        Series::new("c", [3i64, 3]),
-                    ], Span::test_data())
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            }
         ]
     }
 
     fn run(
         &self,
-        engine_state: &EngineState,
+        _engine_state: &EngineState,
-        stack: &mut Stack,
+        _stack: &mut Stack,
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        let maybe_schema = call
+        NuDataFrame::try_from_iter(input.into_iter())
-            .get_flag(engine_state, stack, "schema")?
+            .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
-            .map(|schema| NuSchema::try_from(&schema))
-            .transpose()?;
-
-        let df = NuDataFrame::try_from_iter(input.into_iter(), maybe_schema.clone())?;
-
-        Ok(PipelineData::Value(
-            NuDataFrame::into_value(df, call.head),
-            None,
-        ))
     }
 }
 

crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs

@@ -100,7 +100,7 @@ fn dataframe_command(
     input: Value,
 ) -> Result<PipelineData, ShellError> {
     let rows: Option<usize> = call.get_flag(engine_state, stack, "rows")?;
-    let tail: bool = call.has_flag(engine_state, stack, "tail")?;
+    let tail: bool = call.has_flag("tail");
 
     let df = NuDataFrame::try_from_value(input)?;
 

crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs

@@ -42,23 +42,20 @@ impl Command for WithColumn {
     | dfr into-df
     | dfr with-column ([5 6] | dfr into-df) --name c"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
-                                vec![Value::test_int(1), Value::test_int(3)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "c".to_string(),
-                                "c".to_string(),
+                            vec![Value::test_int(5), Value::test_int(6)],
-                                vec![Value::test_int(5), Value::test_int(6)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -73,27 +70,24 @@ impl Command for WithColumn {
       ]
     | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(3)],
-                                vec![Value::test_int(1), Value::test_int(3)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "c".to_string(),
-                                "c".to_string(),
+                            vec![Value::test_int(2), Value::test_int(6)],
-                                vec![Value::test_int(2), Value::test_int(6)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "d".to_string(),
-                                "d".to_string(),
+                            vec![Value::test_int(3), Value::test_int(9)],
-                                vec![Value::test_int(3), Value::test_int(9)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs

@@ -32,13 +32,10 @@ impl Command for ExprArgWhere {
             example: "let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df);
     $df | dfr select (dfr arg-where ((dfr col b) >= 2) | dfr as b_arg)",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![Column::new(
-                    vec![Column::new(
+                    "b_arg".to_string(),
-                        "b_arg".to_string(),
+                    vec![Value::test_int(1), Value::test_int(2)],
-                        vec![Value::test_int(1), Value::test_int(2)],
+                )])
-                    )],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs

@@ -41,30 +41,27 @@ impl Command for ExprConcatStr {
             example: r#"let df = ([[a b c]; [one two 1] [three four 2]] | dfr into-df);
     $df | dfr with-column ((dfr concat-str "-" [(dfr col a) (dfr col b) ((dfr col c) * 2)]) | dfr as concat)"#,
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_string("one"), Value::test_string("three")],
-                            vec![Value::test_string("one"), Value::test_string("three")],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_string("two"), Value::test_string("four")],
-                            vec![Value::test_string("two"), Value::test_string("four")],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "c".to_string(),
-                            "c".to_string(),
+                        vec![Value::test_int(1), Value::test_int(2)],
-                            vec![Value::test_int(1), Value::test_int(2)],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "concat".to_string(),
-                            "concat".to_string(),
+                        vec![
-                            vec![
+                            Value::test_string("one-two-2"),
-                                Value::test_string("one-two-2"),
+                            Value::test_string("three-four-4"),
-                                Value::test_string("three-four-4"),
+                        ],
-                            ],
+                    ),
-                        ),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -86,7 +83,7 @@ impl Command for ExprConcatStr {
         let value: Value = call.req(engine_state, stack, 1)?;
 
         let expressions = NuExpression::extract_exprs(value)?;
-        let expr: NuExpression = concat_str(expressions, &separator, false).into();
+        let expr: NuExpression = concat_str(expressions, &separator).into();
 
         Ok(PipelineData::Value(expr.into_value(call.head), None))
     }

crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs

@@ -9,11 +9,6 @@ use nu_protocol::{
     Category, Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Type,
     Value,
 };
-use polars::{
-    datatypes::{DataType, TimeUnit},
-    prelude::NamedFrom,
-    series::Series,
-};
 
 #[derive(Clone)]
 pub struct ExprDatePart;
@@ -52,13 +47,10 @@ impl Command for ExprDatePart {
                 description: "Creates an expression to capture the year date part",
                 example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" | dfr with-column [(dfr col datetime | dfr datepart year | dfr as datetime_year )]"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new("datetime".to_string(), vec![Value::test_date(dt)]),
-                            Column::new("datetime".to_string(), vec![Value::test_date(dt)]),
+                        Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]),
-                            Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -74,21 +66,16 @@ impl Command for ExprDatePart {
                 (dfr col datetime | dfr datepart second | dfr as datetime_second ),
                 (dfr col datetime | dfr datepart nanosecond | dfr as datetime_ns ) ]"#,
                 result: Some(
-                    NuDataFrame::try_from_series(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new("datetime".to_string(), vec![Value::test_date(dt)]),
-                            Series::new("datetime", &[dt.timestamp_nanos_opt()])
+                        Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]),
-                                .cast(&DataType::Datetime(TimeUnit::Nanoseconds, None))
+                        Column::new("datetime_month".to_string(), vec![Value::test_int(12)]),
-                                .expect("Error casting to datetime type"),
+                        Column::new("datetime_day".to_string(), vec![Value::test_int(30)]),
-                            Series::new("datetime_year", &[2021_i64]), // i32 was coerced to i64
+                        Column::new("datetime_hour".to_string(), vec![Value::test_int(1)]),
-                            Series::new("datetime_month", &[12_i8]),
+                        Column::new("datetime_minute".to_string(), vec![Value::test_int(2)]),
-                            Series::new("datetime_day", &[30_i8]),
+                        Column::new("datetime_second".to_string(), vec![Value::test_int(3)]),
-                            Series::new("datetime_hour", &[1_i8]),
+                        Column::new("datetime_ns".to_string(), vec![Value::test_int(123456789)]),
-                            Series::new("datetime_minute", &[2_i8]),
+                    ])
-                            Series::new("datetime_second", &[3_i8]),
-                            Series::new("datetime_ns", &[123456789_i64]), // i32 was coerced to i64
-                        ],
-                        Span::test_data(),
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs

@@ -179,18 +179,7 @@ macro_rules! lazy_expr_command {
                 let value = input.into_value(call.head);
                 if NuDataFrame::can_downcast(&value) {
                     let lazy = NuLazyFrame::try_from_value(value)?;
-                    let lazy = NuLazyFrame::new(
+                    let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func());
-                        lazy.from_eager,
-                        lazy.into_polars()
-                            .$func()
-                            .map_err(|e| ShellError::GenericError {
-                                error: "Dataframe Error".into(),
-                                msg: e.to_string(),
-                                help: None,
-                                span: None,
-                                inner: vec![],
-                            })?,
-                    );
 
                     Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
                 } else {
@@ -278,18 +267,7 @@ macro_rules! lazy_expr_command {
                 let value = input.into_value(call.head);
                 if NuDataFrame::can_downcast(&value) {
                     let lazy = NuLazyFrame::try_from_value(value)?;
-                    let lazy = NuLazyFrame::new(
+                    let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func($ddof));
-                        lazy.from_eager,
-                        lazy.into_polars()
-                            .$func($ddof)
-                            .map_err(|e| ShellError::GenericError {
-                                error: "Dataframe Error".into(),
-                                msg: e.to_string(),
-                                help: None,
-                                span: None,
-                                inner: vec![],
-                            })?,
-                    );
 
                     Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
                 } else {
@@ -407,13 +385,10 @@ lazy_expr_command!(
             description: "Max value from columns in a dataframe",
             example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr max",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new("a".to_string(), vec![Value::test_int(6)],),
-                        Column::new("a".to_string(), vec![Value::test_int(6)],),
+                    Column::new("b".to_string(), vec![Value::test_int(4)],),
-                        Column::new("b".to_string(), vec![Value::test_int(4)],),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -425,19 +400,16 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr max)"#,
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_string("one"), Value::test_string("two")],
-                            vec![Value::test_string("one"), Value::test_string("two")],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_int(4), Value::test_int(1)],
-                            vec![Value::test_int(4), Value::test_int(1)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -458,13 +430,10 @@ lazy_expr_command!(
             description: "Min value from columns in a dataframe",
             example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr min",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new("a".to_string(), vec![Value::test_int(1)],),
-                        Column::new("a".to_string(), vec![Value::test_int(1)],),
+                    Column::new("b".to_string(), vec![Value::test_int(1)],),
-                        Column::new("b".to_string(), vec![Value::test_int(1)],),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -476,19 +445,16 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr min)"#,
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_string("one"), Value::test_string("two")],
-                            vec![Value::test_string("one"), Value::test_string("two")],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_int(2), Value::test_int(1)],
-                            vec![Value::test_int(2), Value::test_int(1)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -509,13 +475,10 @@ lazy_expr_command!(
             description: "Sums all columns in a dataframe",
             example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr sum",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new("a".to_string(), vec![Value::test_int(11)],),
-                        Column::new("a".to_string(), vec![Value::test_int(11)],),
+                    Column::new("b".to_string(), vec![Value::test_int(7)],),
-                        Column::new("b".to_string(), vec![Value::test_int(7)],),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -527,19 +490,16 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr sum)"#,
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_string("one"), Value::test_string("two")],
-                            vec![Value::test_string("one"), Value::test_string("two")],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_int(6), Value::test_int(1)],
-                            vec![Value::test_int(6), Value::test_int(1)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -560,13 +520,10 @@ lazy_expr_command!(
             description: "Mean value from columns in a dataframe",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr mean",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new("a".to_string(), vec![Value::test_float(4.0)],),
-                        Column::new("a".to_string(), vec![Value::test_float(4.0)],),
+                    Column::new("b".to_string(), vec![Value::test_float(2.0)],),
-                        Column::new("b".to_string(), vec![Value::test_float(2.0)],),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -578,19 +535,16 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr mean)"#,
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_string("one"), Value::test_string("two")],
-                            vec![Value::test_string("one"), Value::test_string("two")],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_float(3.0), Value::test_float(1.0)],
-                            vec![Value::test_float(3.0), Value::test_float(1.0)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -613,19 +567,16 @@ expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr median)"#,
         result: Some(
-            NuDataFrame::try_from_columns(
+            NuDataFrame::try_from_columns(vec![
-                vec![
+                Column::new(
-                    Column::new(
+                    "a".to_string(),
-                        "a".to_string(),
+                    vec![Value::test_string("one"), Value::test_string("two")],
-                        vec![Value::test_string("one"), Value::test_string("two")],
+                ),
-                    ),
+                Column::new(
-                    Column::new(
+                    "b".to_string(),
-                        "b".to_string(),
+                    vec![Value::test_float(3.0), Value::test_float(1.0)],
-                        vec![Value::test_float(3.0), Value::test_float(1.0)],
+                ),
-                    ),
+            ])
-                ],
-                None
-            )
             .expect("simple df for test should not fail")
             .into_value(Span::test_data()),
         ),
@@ -645,13 +596,10 @@ lazy_expr_command!(
             description: "Std value from columns in a dataframe",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr std",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new("a".to_string(), vec![Value::test_float(2.0)],),
-                        Column::new("a".to_string(), vec![Value::test_float(2.0)],),
+                    Column::new("b".to_string(), vec![Value::test_float(0.0)],),
-                        Column::new("b".to_string(), vec![Value::test_float(0.0)],),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -663,19 +611,16 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr std)"#,
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_string("one"), Value::test_string("two")],
-                            vec![Value::test_string("one"), Value::test_string("two")],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_float(0.0), Value::test_float(0.0)],
-                            vec![Value::test_float(0.0), Value::test_float(0.0)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -698,13 +643,10 @@ lazy_expr_command!(
                 "Var value from columns in a dataframe or aggregates columns to their var value",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr var",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new("a".to_string(), vec![Value::test_float(4.0)],),
-                        Column::new("a".to_string(), vec![Value::test_float(4.0)],),
+                    Column::new("b".to_string(), vec![Value::test_float(0.0)],),
-                        Column::new("b".to_string(), vec![Value::test_float(0.0)],),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -716,19 +658,16 @@ lazy_expr_command!(
     | dfr group-by a
     | dfr agg (dfr col b | dfr var)"#,
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_string("one"), Value::test_string("two")],
-                            vec![Value::test_string("one"), Value::test_string("two")],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_float(0.0), Value::test_float(0.0)],
-                            vec![Value::test_float(0.0), Value::test_float(0.0)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/expressions/is_in.rs

@@ -39,31 +39,28 @@ impl Command for ExprIsIn {
             example: r#"let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df);
     $df | dfr with-column (dfr col a | dfr is-in [one two] | dfr as a_in)"#,
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![
-                            vec![
+                            Value::test_string("one"),
-                                Value::test_string("one"),
+                            Value::test_string("two"),
-                                Value::test_string("two"),
+                            Value::test_string("three"),
-                                Value::test_string("three"),
+                        ],
-                            ],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)],
-                            vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "a_in".to_string(),
-                            "a_in".to_string(),
+                        vec![
-                            vec![
+                            Value::test_bool(true),
-                                Value::test_bool(true),
+                            Value::test_bool(true),
-                                Value::test_bool(true),
+                            Value::test_bool(false),
-                                Value::test_bool(false),
+                        ],
-                            ],
+                    ),
-                        ),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),
@@ -84,8 +81,7 @@ impl Command for ExprIsIn {
         let list: Vec<Value> = call.req(engine_state, stack, 0)?;
         let expr = NuExpression::try_from_pipeline(input, call.head)?;
 
-        let values =
+        let values = NuDataFrame::try_from_columns(vec![Column::new("list".to_string(), list)])?;
-            NuDataFrame::try_from_columns(vec![Column::new("list".to_string(), list)], None)?;
         let list = values.as_series(call.head)?;
 
         if matches!(list.dtype(), DataType::Object(..)) {

crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs

@@ -54,27 +54,24 @@ impl Command for ExprOtherwise {
      )
    | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
-                                vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
-                                vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "c".to_string(),
-                                "c".to_string(),
+                            vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
-                                vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "d".to_string(),
-                                "d".to_string(),
+                            vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
-                                vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs

@@ -41,19 +41,16 @@ impl Command for ExprQuantile {
     | dfr group-by a
     | dfr agg (dfr col b | dfr quantile 0.5)"#,
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_string("one"), Value::test_string("two")],
-                            vec![Value::test_string("one"), Value::test_string("two")],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_float(4.0), Value::test_float(1.0)],
-                            vec![Value::test_float(4.0), Value::test_float(1.0)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs

@@ -62,27 +62,24 @@ impl Command for ExprWhen {
      )
    | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
-                                vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
-                                vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "c".to_string(),
-                                "c".to_string(),
+                            vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
-                                vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "d".to_string(),
-                                "d".to_string(),
+                            vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
-                                vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs

@@ -47,27 +47,24 @@ impl Command for LazyAggregate {
         (dfr col b | dfr sum | dfr as "b_sum")
      ]"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(2)],
-                                vec![Value::test_int(1), Value::test_int(2)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_min".to_string(),
-                                "b_min".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_max".to_string(),
-                                "b_max".to_string(),
+                            vec![Value::test_int(4), Value::test_int(6)],
-                                vec![Value::test_int(4), Value::test_int(6)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_sum".to_string(),
-                                "b_sum".to_string(),
+                            vec![Value::test_int(6), Value::test_int(10)],
-                                vec![Value::test_int(6), Value::test_int(10)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -84,27 +81,24 @@ impl Command for LazyAggregate {
      ]
     | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(2)],
-                                vec![Value::test_int(1), Value::test_int(2)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_min".to_string(),
-                                "b_min".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_max".to_string(),
-                                "b_max".to_string(),
+                            vec![Value::test_int(4), Value::test_int(6)],
-                                vec![Value::test_int(4), Value::test_int(6)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_sum".to_string(),
-                                "b_sum".to_string(),
+                            vec![Value::test_int(6), Value::test_int(10)],
-                                vec![Value::test_int(6), Value::test_int(10)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -166,7 +160,7 @@ fn get_col_name(expr: &Expr) -> Option<String> {
             | polars::prelude::AggExpr::Last(e)
             | polars::prelude::AggExpr::Mean(e)
             | polars::prelude::AggExpr::Implode(e)
-            | polars::prelude::AggExpr::Count(e, _)
+            | polars::prelude::AggExpr::Count(e)
             | polars::prelude::AggExpr::Sum(e)
             | polars::prelude::AggExpr::AggGroups(e)
             | polars::prelude::AggExpr::Std(e, _)
@@ -193,7 +187,7 @@ fn get_col_name(expr: &Expr) -> Option<String> {
         | Expr::Window { .. }
         | Expr::Wildcard
         | Expr::RenameAlias { .. }
-        | Expr::Len
+        | Expr::Count
         | Expr::Nth(_)
         | Expr::SubPlan(_, _)
         | Expr::Selector(_) => None,

crates/nu-cmd-dataframe/src/dataframe/lazy/collect.rs

@@ -33,19 +33,16 @@ impl Command for LazyCollect {
             description: "drop duplicates",
             example: "[[a b]; [1 2] [3 4]] | dfr into-lazy | dfr collect",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_int(1), Value::test_int(3)],
-                            vec![Value::test_int(1), Value::test_int(3)],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_int(2), Value::test_int(4)],
-                            vec![Value::test_int(2), Value::test_int(4)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs

@@ -69,7 +69,7 @@ impl Command for LazyExplode {
                             Value::test_string("Skiing"),
                             Value::test_string("Football"),
                         ]),
-                   ], None).expect("simple df for test should not fail")
+                   ]).expect("simple df for test should not fail")
                    .into_value(Span::test_data()),
                     )
             },
@@ -86,7 +86,7 @@ impl Command for LazyExplode {
                             Value::test_string("Skiing"),
                             Value::test_string("Football"),
                         ]),
-                   ], None).expect("simple df for test should not fail")
+                   ]).expect("simple df for test should not fail")
                    .into_value(Span::test_data()),
                     ),
             },

crates/nu-cmd-dataframe/src/dataframe/lazy/fetch.rs

@@ -38,19 +38,16 @@ impl Command for LazyFetch {
             description: "Fetch a rows from the dataframe",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr fetch 2",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_int(6), Value::test_int(4)],
-                            vec![Value::test_int(6), Value::test_int(4)],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_int(2), Value::test_int(2)],
-                            vec![Value::test_int(2), Value::test_int(2)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs

@@ -38,19 +38,16 @@ impl Command for LazyFillNA {
                 description: "Fills the NaN values with 0",
                 example: "[1 2 NaN 3 NaN] | dfr into-df | dfr fill-nan 0",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![Column::new(
-                        vec![Column::new(
+                        "0".to_string(),
-                            "0".to_string(),
+                        vec![
-                            vec![
+                            Value::test_int(1),
-                                Value::test_int(1),
+                            Value::test_int(2),
-                                Value::test_int(2),
+                            Value::test_int(0),
-                                Value::test_int(0),
+                            Value::test_int(3),
-                                Value::test_int(3),
+                            Value::test_int(0),
-                                Value::test_int(0),
+                        ],
-                            ],
+                    )])
-                        )],
-                        None,
-                    )
                     .expect("Df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -59,19 +56,16 @@ impl Command for LazyFillNA {
                 description: "Fills the NaN values of a whole dataframe",
                 example: "[[a b]; [0.2 1] [0.1 NaN]] | dfr into-df | dfr fill-nan 0",
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_float(0.2), Value::test_float(0.1)],
-                                vec![Value::test_float(0.2), Value::test_float(0.1)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![Value::test_int(1), Value::test_int(0)],
-                                vec![Value::test_int(1), Value::test_int(0)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("Df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -129,7 +123,7 @@ impl Command for LazyFillNA {
                 })
                 .collect::<Vec<Column>>();
             Ok(PipelineData::Value(
-                NuDataFrame::try_from_columns(dataframe, None)?.into_value(call.head),
+                NuDataFrame::try_from_columns(dataframe)?.into_value(call.head),
                 None,
             ))
         }

crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs

@@ -37,19 +37,16 @@ impl Command for LazyFillNull {
             description: "Fills the null values by 0",
             example: "[1 2 2 3 3] | dfr into-df | dfr shift 2 | dfr fill-null 0",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![Column::new(
-                    vec![Column::new(
+                    "0".to_string(),
-                        "0".to_string(),
+                    vec![
-                        vec![
+                        Value::test_int(0),
-                            Value::test_int(0),
+                        Value::test_int(0),
-                            Value::test_int(0),
+                        Value::test_int(1),
-                            Value::test_int(1),
+                        Value::test_int(2),
-                            Value::test_int(2),
+                        Value::test_int(2),
-                            Value::test_int(2),
+                    ],
-                        ],
+                )])
-                    )],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/lazy/filter.rs

@@ -38,19 +38,16 @@ impl Command for LazyFilter {
             description: "Filter dataframe using an expression",
             example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr filter ((dfr col a) >= 4)",
             result: Some(
-                NuDataFrame::try_from_columns(
+                NuDataFrame::try_from_columns(vec![
-                    vec![
+                    Column::new(
-                        Column::new(
+                        "a".to_string(),
-                            "a".to_string(),
+                        vec![Value::test_int(6), Value::test_int(4)],
-                            vec![Value::test_int(6), Value::test_int(4)],
+                    ),
-                        ),
+                    Column::new(
-                        Column::new(
+                        "b".to_string(),
-                            "b".to_string(),
+                        vec![Value::test_int(2), Value::test_int(2)],
-                            vec![Value::test_int(2), Value::test_int(2)],
+                    ),
-                        ),
+                ])
-                    ],
-                    None,
-                )
                 .expect("simple df for test should not fail")
                 .into_value(Span::test_data()),
             ),

crates/nu-cmd-dataframe/src/dataframe/lazy/flatten.rs

@@ -71,7 +71,7 @@ Example {
                             Value::test_string("Skiing"),
                             Value::test_string("Football"),
                         ]),
-                   ], None).expect("simple df for test should not fail")
+                   ]).expect("simple df for test should not fail")
                    .into_value(Span::test_data()),
                     )
             },
@@ -88,7 +88,7 @@ Example {
                             Value::test_string("Skiing"),
                             Value::test_string("Football"),
                         ]),
-                   ], None).expect("simple df for test should not fail")
+                   ]).expect("simple df for test should not fail")
                    .into_value(Span::test_data()),
                     ),
             },

crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs

@@ -46,27 +46,24 @@ impl Command for ToLazyGroupBy {
         (dfr col b | dfr sum | dfr as "b_sum")
      ]"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(2)],
-                                vec![Value::test_int(1), Value::test_int(2)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_min".to_string(),
-                                "b_min".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_max".to_string(),
-                                "b_max".to_string(),
+                            vec![Value::test_int(4), Value::test_int(6)],
-                                vec![Value::test_int(4), Value::test_int(6)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_sum".to_string(),
-                                "b_sum".to_string(),
+                            vec![Value::test_int(6), Value::test_int(10)],
-                                vec![Value::test_int(6), Value::test_int(10)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -83,27 +80,24 @@ impl Command for ToLazyGroupBy {
      ]
     | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![Value::test_int(1), Value::test_int(2)],
-                                vec![Value::test_int(1), Value::test_int(2)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_min".to_string(),
-                                "b_min".to_string(),
+                            vec![Value::test_int(2), Value::test_int(4)],
-                                vec![Value::test_int(2), Value::test_int(4)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_max".to_string(),
-                                "b_max".to_string(),
+                            vec![Value::test_int(4), Value::test_int(6)],
-                                vec![Value::test_int(4), Value::test_int(6)],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b_sum".to_string(),
-                                "b_sum".to_string(),
+                            vec![Value::test_int(6), Value::test_int(10)],
-                                vec![Value::test_int(6), Value::test_int(10)],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),

crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs

@@ -53,56 +53,53 @@ impl Command for LazyJoin {
     let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | dfr into-lazy);
     $df_a | dfr join $df_b a foo | dfr collect"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![
-                                vec![
+                                Value::test_int(1),
-                                    Value::test_int(1),
+                                Value::test_int(2),
-                                    Value::test_int(2),
+                                Value::test_int(1),
-                                    Value::test_int(1),
+                                Value::test_int(1),
-                                    Value::test_int(1),
+                            ],
-                                ],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![
-                                vec![
+                                Value::test_string("a"),
-                                    Value::test_string("a"),
+                                Value::test_string("b"),
-                                    Value::test_string("b"),
+                                Value::test_string("c"),
-                                    Value::test_string("c"),
+                                Value::test_string("c"),
-                                    Value::test_string("c"),
+                            ],
-                                ],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "c".to_string(),
-                                "c".to_string(),
+                            vec![
-                                vec![
+                                Value::test_int(0),
-                                    Value::test_int(0),
+                                Value::test_int(1),
-                                    Value::test_int(1),
+                                Value::test_int(2),
-                                    Value::test_int(2),
+                                Value::test_int(3),
-                                    Value::test_int(3),
+                            ],
-                                ],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "bar".to_string(),
-                                "bar".to_string(),
+                            vec![
-                                vec![
+                                Value::test_string("a"),
-                                    Value::test_string("a"),
+                                Value::test_string("c"),
-                                    Value::test_string("c"),
+                                Value::test_string("a"),
-                                    Value::test_string("a"),
+                                Value::test_string("a"),
-                                    Value::test_string("a"),
+                            ],
-                                ],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "ham".to_string(),
-                                "ham".to_string(),
+                            vec![
-                                vec![
+                                Value::test_string("let"),
-                                    Value::test_string("let"),
+                                Value::test_string("var"),
-                                    Value::test_string("var"),
+                                Value::test_string("let"),
-                                    Value::test_string("let"),
+                                Value::test_string("let"),
-                                    Value::test_string("let"),
+                            ],
-                                ],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -113,56 +110,53 @@ impl Command for LazyJoin {
     let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | dfr into-lazy);
     $df_a | dfr join $df_b a foo"#,
                 result: Some(
-                    NuDataFrame::try_from_columns(
+                    NuDataFrame::try_from_columns(vec![
-                        vec![
+                        Column::new(
-                            Column::new(
+                            "a".to_string(),
-                                "a".to_string(),
+                            vec![
-                                vec![
+                                Value::test_int(1),
-                                    Value::test_int(1),
+                                Value::test_int(2),
-                                    Value::test_int(2),
+                                Value::test_int(1),
-                                    Value::test_int(1),
+                                Value::test_int(1),
-                                    Value::test_int(1),
+                            ],
-                                ],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "b".to_string(),
-                                "b".to_string(),
+                            vec![
-                                vec![
+                                Value::test_string("a"),
-                                    Value::test_string("a"),
+                                Value::test_string("b"),
-                                    Value::test_string("b"),
+                                Value::test_string("c"),
-                                    Value::test_string("c"),
+                                Value::test_string("c"),
-                                    Value::test_string("c"),
+                            ],
-                                ],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "c".to_string(),
-                                "c".to_string(),
+                            vec![
-                                vec![
+                                Value::test_int(0),
-                                    Value::test_int(0),
+                                Value::test_int(1),
-                                    Value::test_int(1),
+                                Value::test_int(2),
-                                    Value::test_int(2),
+                                Value::test_int(3),
-                                    Value::test_int(3),
+                            ],
-                                ],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "bar".to_string(),
-                                "bar".to_string(),
+                            vec![
-                                vec![
+                                Value::test_string("a"),
-                                    Value::test_string("a"),
+                                Value::test_string("c"),
-                                    Value::test_string("c"),
+                                Value::test_string("a"),
-                                    Value::test_string("a"),
+                                Value::test_string("a"),
-                                    Value::test_string("a"),
+                            ],
-                                ],
+                        ),
-                            ),
+                        Column::new(
-                            Column::new(
+                            "ham".to_string(),
-                                "ham".to_string(),
+                            vec![
-                                vec![
+                                Value::test_string("let"),
-                                    Value::test_string("let"),
+                                Value::test_string("var"),
-                                    Value::test_string("var"),
+                                Value::test_string("let"),
-                                    Value::test_string("let"),
+                                Value::test_string("let"),
-                                    Value::test_string("let"),
+                            ],
-                                ],
+                        ),
-                            ),
+                    ])
-                        ],
-                        None,
-                    )
                     .expect("simple df for test should not fail")
                     .into_value(Span::test_data()),
                 ),
@@ -177,14 +171,14 @@ impl Command for LazyJoin {
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        let left = call.has_flag(engine_state, stack, "left")?;
+        let left = call.has_flag("left");
-        let outer = call.has_flag(engine_state, stack, "outer")?;
+        let outer = call.has_flag("outer");
-        let cross = call.has_flag(engine_state, stack, "cross")?;
+        let cross = call.has_flag("cross");
 
         let how = if left {
             JoinType::Left
         } else if outer {
-            JoinType::Outer { coalesce: true }
+            JoinType::Outer
         } else if cross {
             JoinType::Cross
         } else {

crates/nu-cmd-dataframe/src/dataframe/lazy/macro_commands.rs

@@ -112,70 +112,6 @@ macro_rules! lazy_command {
             }
         }
     };
-
-    ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident?, $test: ident) => {
-        #[derive(Clone)]
-        pub struct $command;
-
-        impl Command for $command {
-            fn name(&self) -> &str {
-                $name
-            }
-
-            fn usage(&self) -> &str {
-                $desc
-            }
-
-            fn signature(&self) -> Signature {
-                Signature::build(self.name())
-                    .input_output_type(
-                        Type::Custom("dataframe".into()),
-                        Type::Custom("dataframe".into()),
-                    )
-                    .category(Category::Custom("lazyframe".into()))
-            }
-
-            fn examples(&self) -> Vec<Example> {
-                $examples
-            }
-
-            fn run(
-                &self,
-                _engine_state: &EngineState,
-                _stack: &mut Stack,
-                call: &Call,
-                input: PipelineData,
-            ) -> Result<PipelineData, ShellError> {
-                let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?;
-
-                let lazy = NuLazyFrame::new(
-                    lazy.from_eager,
-                    lazy.into_polars()
-                        .$func()
-                        .map_err(|e| ShellError::GenericError {
-                            error: "Dataframe Error".into(),
-                            msg: e.to_string(),
-                            help: None,
-                            span: None,
-                            inner: vec![],
-                        })?,
-                );
-
-                Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
-            }
-        }
-
-        #[cfg(test)]
-        mod $test {
-            use super::super::super::test_dataframe::test_dataframe;
-            use super::*;
-
-            #[test]
-            fn test_examples() {
-                test_dataframe(vec![Box::new($command {})])
-            }
-        }
-    };
 }
 
 // LazyReverse command
@@ -188,19 +124,16 @@ lazy_command!(
         description: "Reverses the dataframe.",
         example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr reverse",
         result: Some(
-            NuDataFrame::try_from_columns(
+            NuDataFrame::try_from_columns(vec![
-                vec![
+                Column::new(
-                    Column::new(
+                    "a".to_string(),
-                        "a".to_string(),
+                    vec![Value::test_int(2), Value::test_int(4), Value::test_int(6),],
-                        vec![Value::test_int(2), Value::test_int(4), Value::test_int(6),],
+                ),
-                    ),
+                Column::new(
-                    Column::new(
+                    "b".to_string(),
-                        "b".to_string(),
+                    vec![Value::test_int(2), Value::test_int(2), Value::test_int(2),],
-                        vec![Value::test_int(2), Value::test_int(2), Value::test_int(2),],
+                ),
-                    ),
+            ])
-                ],
-                None
-            )
             .expect("simple df for test should not fail")
             .into_value(Span::test_data()),
         ),
@@ -234,17 +167,14 @@ lazy_command!(
         description: "Median value from columns in a dataframe",
         example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr median",
         result: Some(
-            NuDataFrame::try_from_columns(
+            NuDataFrame::try_from_columns(vec![
-                vec![
+                Column::new("a".to_string(), vec![Value::test_float(4.0)],),
-                    Column::new("a".to_string(), vec![Value::test_float(4.0)],),
+                Column::new("b".to_string(), vec![Value::test_float(2.0)],),
-                    Column::new("b".to_string(), vec![Value::test_float(2.0)],),
+            ])
-                ],
-                None
-            )
             .expect("simple df for test should not fail")
             .into_value(Span::test_data()),
         ),
     },],
-    median?,
+    median,
     test_median
 );

crates/nu-cmd-dataframe/src/dataframe/lazy/mod.rs

@@ -20,7 +20,7 @@ use crate::dataframe::lazy::aggregate::LazyAggregate;
 pub use crate::dataframe::lazy::collect::LazyCollect;
 use crate::dataframe::lazy::fetch::LazyFetch;
 use crate::dataframe::lazy::fill_nan::LazyFillNA;
-pub use crate::dataframe::lazy::fill_null::LazyFillNull;
+use crate::dataframe::lazy::fill_null::LazyFillNull;
 use crate::dataframe::lazy::filter::LazyFilter;
 use crate::dataframe::lazy::groupby::ToLazyGroupBy;
 use crate::dataframe::lazy::join::LazyJoin;