Bump to 0.63 (#5627)

* Allow env vars to be kept from removed overlay

* Rename --keep to --keep-custom; Add new test

* Rename some symbols

* (WIP) Start working on --keep for defs and aliases

* Fix decls/aliases not melting properly

* Use id instead of the whole cloned overlay

* Rewrite overlay remove for no reason

Doesn't fix the bug but at least looks better.

* Rename variable

* Fix adding overlay env vars

* Add more tests; Fmt + Clippy

.github/pull_request_template.md

@@ -7,5 +7,5 @@
 Make sure you've run and fixed any issues with these commands:
 
 - [ ] `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes)
-- [ ] `cargo clippy --all --all-features -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style
+- [ ] `cargo clippy --workspace --features=extra -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style
-- [ ] `cargo build; cargo test --all --all-features` to check that all the tests pass
+- [ ] `cargo test --workspace --features=extra` to check that all the tests pass

.github/workflows/ci.yml

@@ -7,26 +7,17 @@ on:
 name: continuous-integration
 
 jobs:
-  build-clippy:
+  nu-fmt-clippy:
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         platform: [windows-latest, macos-latest, ubuntu-latest]
-        style: [all, default]
         rust:
           - stable
-        include:
-          - style: all
-            flags: "--all-features"
-          - style: default
-            flags: ""
-        exclude:
-          - platform: windows-latest
-            style: default
-          - platform: macos-latest
-            style: default
 
     runs-on: ${{ matrix.platform }}
+    env:
+      NUSHELL_CARGO_TARGET: ci
 
     steps:
       - uses: actions/checkout@v2
@@ -41,7 +32,7 @@ jobs:
 
       - uses: Swatinem/rust-cache@v1
         with:
-          key: ${{ matrix.style }}v1 # increment this to bust the cache if needed
+          key: "v2" # increment this to bust the cache if needed
 
       - name: Rustfmt
         uses: actions-rs/cargo@v1
@@ -49,29 +40,26 @@ jobs:
           command: fmt
           args: --all -- --check
 
-      - name: Build Nushell
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace ${{ matrix.flags }}
-
       - name: Clippy
         uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --workspace ${{ matrix.flags }} -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect
+          args: --features=extra --workspace --exclude nu_plugin_* -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect
+
+  nu-tests:
+    env:
+      NUSHELL_CARGO_TARGET: ci
 
-  test:
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         platform: [windows-latest, macos-latest, ubuntu-latest]
-        style: [all, default]
+        style: [extra, default]
         rust:
           - stable
         include:
-          - style: all
+          - style: extra
-            flags: "--all-features"
+            flags: "--features=extra"
           - style: default
             flags: ""
         exclude:
@@ -91,29 +79,23 @@ jobs:
           profile: minimal
           toolchain: ${{ matrix.rust }}
           override: true
-          components: rustfmt, clippy
 
       - uses: Swatinem/rust-cache@v1
         with:
-          key: ${{ matrix.style }}v1 # increment this to bust the cache if needed
+          key: ${{ matrix.style }}v3 # increment this to bust the cache if needed
-
-      - uses: taiki-e/install-action@nextest
 
       - name: Tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: nextest
-          args: run --all ${{ matrix.flags }}
-
-      - name: Doctests
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --workspace --doc ${{ matrix.flags }}
+          args: --workspace --profile ci --exclude nu_plugin_* ${{ matrix.flags }}
 
   python-virtualenv:
+    env:
+      NUSHELL_CARGO_TARGET: ci
+
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
         rust:
@@ -135,13 +117,13 @@ jobs:
 
       - uses: Swatinem/rust-cache@v1
         with:
-          key: "1" # increment this to bust the cache if needed
+          key: "2" # increment this to bust the cache if needed
 
       - name: Install Nushell
         uses: actions-rs/cargo@v1
         with:
           command: install
-          args: --path=. --no-default-features --debug
+          args: --path=. --profile ci --no-default-features
 
       - name: Setup Python
         uses: actions/setup-python@v2
@@ -161,9 +143,14 @@ jobs:
         run: cd virtualenv && tox -e ${{ matrix.py }} -- -k nushell
         shell: bash
 
+  # Build+test plugins on their own, without the rest of Nu. This helps with CI parallelization and
+  # also helps test that the plugins build without any feature unification shenanigans
   plugins:
+    env:
+      NUSHELL_CARGO_TARGET: ci
+
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         platform: [windows-latest, macos-latest, ubuntu-latest]
         rust:
@@ -181,29 +168,14 @@ jobs:
           toolchain: ${{ matrix.rust }}
           override: true
 
-      # This job does not use rust-cache because 1) we have limited cache space, 2) even
+      - name: Clippy
-      # without caching, it's not the slowest job. Revisit if those facts change.
-
-      - name: Build nu_plugin_example
         uses: actions-rs/cargo@v1
         with:
-          command: build
+          command: clippy
-          args: --package nu_plugin_example
+          args: --package nu_plugin_* ${{ matrix.flags }} -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect
 
-      - name: Build nu_plugin_gstat
+      - name: Tests
         uses: actions-rs/cargo@v1
         with:
-          command: build
+          command: test
-          args: --package nu_plugin_gstat
+          args: --profile ci --package nu_plugin_*
-
-      - name: Build nu_plugin_inc
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --package nu_plugin_inc
-
-      - name: Build nu_plugin_query
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --package nu_plugin_query

.gitignore

@@ -23,4 +23,5 @@ debian/nu/
 .vscode/*
 
 # Helix configuration folder
+.helix/*
 .helix

Cargo.lock

@@ -62,6 +62,12 @@ dependencies = [
  "alloc-no-stdlib",
 ]
 
+[[package]]
+name = "alphanumeric-sort"
+version = "1.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77e9c9abb82613923ec78d7a461595d52491ba7240f3c64c0bbe0e6d98e0fce0"
+
 [[package]]
 name = "ansi-parser"
 version = "0.8.0"
@@ -135,9 +141,9 @@ dependencies = [
 
 [[package]]
 name = "arrow2"
-version = "0.10.1"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e387b20dd573a96f36b173d9027483898f944d696521afd74e2caa3c813d86e"
+checksum = "b040061368d1314b0fd8b8f1fde0671eba1afc63a1c61a4dafaf2d4fc10c96f9"
 dependencies = [
  "arrow-format",
  "base64",
@@ -2401,7 +2407,7 @@ dependencies = [
 
 [[package]]
 name = "nu"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "assert_cmd",
  "chrono",
@@ -2428,6 +2434,7 @@ dependencies = [
  "nu-table",
  "nu-term-grid",
  "nu-test-support",
+ "nu-utils",
  "openssl",
  "pretty_assertions",
  "pretty_env_logger",
@@ -2450,7 +2457,7 @@ dependencies = [
 
 [[package]]
 name = "nu-cli"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "crossterm",
  "fuzzy-matcher",
@@ -2472,7 +2479,7 @@ dependencies = [
 
 [[package]]
 name = "nu-color-config"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "nu-ansi-term",
  "nu-json",
@@ -2483,9 +2490,10 @@ dependencies = [
 
 [[package]]
 name = "nu-command"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "Inflector",
+ "alphanumeric-sort",
  "base64",
  "bytesize",
  "calamine",
@@ -2532,6 +2540,7 @@ dependencies = [
  "num 0.4.0",
  "pathdiff",
  "polars",
+ "powierza-coefficient",
  "quick-xml 0.22.0",
  "quickcheck",
  "quickcheck_macros",
@@ -2568,18 +2577,19 @@ dependencies = [
 
 [[package]]
 name = "nu-engine"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "chrono",
  "nu-glob",
  "nu-path",
  "nu-protocol",
+ "nu-utils",
  "sysinfo",
 ]
 
 [[package]]
 name = "nu-glob"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "doc-comment",
  "tempdir",
@@ -2587,7 +2597,7 @@ dependencies = [
 
 [[package]]
 name = "nu-json"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "lazy_static",
  "linked-hash-map",
@@ -2600,7 +2610,7 @@ dependencies = [
 
 [[package]]
 name = "nu-parser"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "chrono",
  "log",
@@ -2614,7 +2624,7 @@ dependencies = [
 
 [[package]]
 name = "nu-path"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "dirs-next",
  "dunce",
@@ -2623,7 +2633,7 @@ dependencies = [
 
 [[package]]
 name = "nu-plugin"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "capnp",
  "nu-engine",
@@ -2634,7 +2644,7 @@ dependencies = [
 
 [[package]]
 name = "nu-pretty-hex"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "heapless 0.7.10",
  "nu-ansi-term",
@@ -2643,7 +2653,7 @@ dependencies = [
 
 [[package]]
 name = "nu-protocol"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "byte-unit",
  "chrono",
@@ -2651,6 +2661,7 @@ dependencies = [
  "indexmap",
  "miette 4.5.0",
  "nu-json",
+ "nu-utils",
  "num-format",
  "regex",
  "serde",
@@ -2662,7 +2673,7 @@ dependencies = [
 
 [[package]]
 name = "nu-system"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "chrono",
  "errno",
@@ -2676,7 +2687,7 @@ dependencies = [
 
 [[package]]
 name = "nu-table"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "ansi-str",
  "atty",
@@ -2689,7 +2700,7 @@ dependencies = [
 
 [[package]]
 name = "nu-term-grid"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "strip-ansi-escapes",
  "unicode-width",
@@ -2697,7 +2708,7 @@ dependencies = [
 
 [[package]]
 name = "nu-test-support"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "getset",
  "hamcrest2",
@@ -2709,14 +2720,14 @@ dependencies = [
 
 [[package]]
 name = "nu-utils"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "crossterm_winapi",
 ]
 
 [[package]]
 name = "nu_plugin_example"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "nu-plugin",
  "nu-protocol",
@@ -2724,7 +2735,7 @@ dependencies = [
 
 [[package]]
 name = "nu_plugin_gstat"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "git2",
  "nu-engine",
@@ -2734,7 +2745,7 @@ dependencies = [
 
 [[package]]
 name = "nu_plugin_inc"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "nu-plugin",
  "nu-protocol",
@@ -2743,7 +2754,7 @@ dependencies = [
 
 [[package]]
 name = "nu_plugin_query"
-version = "0.62.0"
+version = "0.63.0"
 dependencies = [
  "gjson",
  "nu-engine",
@@ -2972,15 +2983,6 @@ dependencies = [
  "vcpkg",
 ]
 
-[[package]]
-name = "ordered-float"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7"
-dependencies = [
- "num-traits",
-]
-
 [[package]]
 name = "output_vt100"
 version = "0.1.3"
@@ -3052,22 +3054,20 @@ dependencies = [
 
 [[package]]
 name = "parquet-format-async-temp"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03abc2f9c83fe9ceec83f47c76cc071bfd56caba33794340330f35623ab1f544"
+checksum = "488c8b5f43521d019fade4bcc0ce88cce5da5fd26eb1d38b933807041f5930bf"
 dependencies = [
  "async-trait",
- "byteorder",
  "futures",
  "integer-encoding",
- "ordered-float",
 ]
 
 [[package]]
 name = "parquet2"
-version = "0.10.3"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b085f9e78e4842865151b693f6d94bdf7b280af66daa6e3587adeb3106a07e9"
+checksum = "98f99f9724402d81faadd9cfa1e8dc78055fd0ddfdbefb7adab3a3a13e893408"
 dependencies = [
  "async-stream",
  "bitpacking",
@@ -3239,33 +3239,35 @@ dependencies = [
 
 [[package]]
 name = "polars"
-version = "0.20.0"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "656db3b86c338a8a717476eb29436a380ebdf74915a71cff6ecce78d52173e53"
+checksum = "b140da767e129c60c41c8e1968ffab5f114bcf823182edb7fa900464a31bf421"
 dependencies = [
  "polars-core",
  "polars-io",
  "polars-lazy",
+ "polars-ops",
  "polars-time",
 ]
 
 [[package]]
 name = "polars-arrow"
-version = "0.20.0"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcedf44a7b15b60c69e811c9d343ac459788e961dc4136f002ed1b68a1fada07"
+checksum = "6d27df11ee28956bd6f5aed54e7e05ce87b886871995e1da501134627ec89077"
 dependencies = [
  "arrow2",
  "hashbrown 0.12.0",
  "num 0.4.0",
+ "serde",
  "thiserror",
 ]
 
 [[package]]
 name = "polars-core"
-version = "0.20.0"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6dfed0e21ac4d4c85df45b5864a68cfc5b2a97e9fba8a981be7b09c6f02a7eaa"
+checksum = "fdf8d12cb7ec278516228fc86469f98c62ab81ca31e4e76d2c0ccf5a09c70491"
 dependencies = [
  "ahash",
  "anyhow",
@@ -3276,8 +3278,8 @@ dependencies = [
  "indexmap",
  "lazy_static",
  "num 0.4.0",
- "num_cpus",
  "polars-arrow",
+ "polars-utils",
  "rand 0.8.5",
  "rand_distr",
  "rayon",
@@ -3289,9 +3291,9 @@ dependencies = [
 
 [[package]]
 name = "polars-io"
-version = "0.20.0"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8770fb4233ab88affac80c410be090dc7a2c044a9e4e7b942132e94ceeb732b"
+checksum = "fdd4b762e5694f359ded21ca0627b5bc95b6eb49f6b330569afc1d20f0564b01"
 dependencies = [
  "ahash",
  "anyhow",
@@ -3303,21 +3305,22 @@ dependencies = [
  "memchr",
  "memmap2",
  "num 0.4.0",
- "num_cpus",
  "polars-arrow",
  "polars-core",
+ "polars-time",
  "polars-utils",
  "rayon",
  "regex",
+ "serde",
  "serde_json",
  "simdutf8",
 ]
 
 [[package]]
 name = "polars-lazy"
-version = "0.20.0"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4eca1fed3b88ae1bb9b7f1d7b2958f1655d9c1aed33495d6ba30ff84a0c1e9e9"
+checksum = "eedc21001f05611e41bb7439b38d0f4ef9406aa49c17f3b289b5f57d8fa40c59"
 dependencies = [
  "ahash",
  "glob",
@@ -3328,24 +3331,36 @@ dependencies = [
  "polars-time",
  "polars-utils",
  "rayon",
+ "serde",
 ]
 
 [[package]]
-name = "polars-time"
+name = "polars-ops"
-version = "0.20.0"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fe48c759ca778a8b6fb30f70e9a81b56f0987a82dc71e61c5b2d3c236b6b8d6"
+checksum = "86fae68f0992955f224f09d1f15648a6fb76d8e3b962efac2f97ccc2aa58977a"
 dependencies = [
- "chrono",
- "polars-arrow",
  "polars-core",
 ]
 
 [[package]]
-name = "polars-utils"
+name = "polars-time"
-version = "0.20.0"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71011e8ed52f123ce23d110b496c8704d0a59c5fd4115cd938e7ff19d4bcb7ca"
+checksum = "be499f73749e820f96689c5f9ec59669b7cdd551d864358e2bdaebb5944e4bfb"
+dependencies = [
+ "chrono",
+ "lexical",
+ "polars-arrow",
+ "polars-core",
+ "serde",
+]
+
+[[package]]
+name = "polars-utils"
+version = "0.21.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7f4cd569d383f5f000abbd6d5146550e6cb4e43fac30d1af98699499a440d56"
 dependencies = [
  "parking_lot 0.12.0",
  "rayon",
@@ -3360,6 +3375,12 @@ dependencies = [
  "nom 7.1.1",
 ]
 
+[[package]]
+name = "powierza-coefficient"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9caa43783252cf8c4c66dd1cc381a5929cc95f6530da7abd1f9cdb97e2065842"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.16"
@@ -3715,9 +3736,9 @@ dependencies = [
 
 [[package]]
 name = "reedline"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d96c36021d0668f3b4f8c054fce3a9b9b0aa83fc60aa6c59df0e2165f9980763"
+checksum = "422f144c06f679da4ab4f082a6d1d43e28bfabb68d009100e6e5520728f99fec"
 dependencies = [
  "chrono",
  "crossterm",
@@ -4177,7 +4198,6 @@ checksum = "f47e98e36909e951f4da3908f4475f969bec92a41734dd92e883aaa11c10294b"
 dependencies = [
  "chrono",
  "const_format",
- "git2",
  "is_debug",
 ]
 
@@ -4713,9 +4733,9 @@ dependencies = [
 
 [[package]]
 name = "trash"
-version = "2.0.4"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a2ed4369f59214865022230fb397ad71353101fe87bfef0f0cf887c43eaa094"
+checksum = "115b3303b13438787fbe4e813a3b16bb4f2928840aa41d80a593c347d0425192"
 dependencies = [
  "chrono",
  "libc",
@@ -4777,9 +4797,9 @@ checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c"
 
 [[package]]
 name = "umask"
-version = "1.0.1"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efb3f38a494193b563eb215c43cb635a4fda1dfcd885fe3906b215bc6a9fb6b8"
+checksum = "46b0c16eadfb312c7acd6970fc97d1f3152eb536714a2ff72ca09a92cae6fa67"
 
 [[package]]
 name = "uncased"
@@ -5320,18 +5340,18 @@ dependencies = [
 
 [[package]]
 name = "zstd"
-version = "0.10.0+zstd.1.5.2"
+version = "0.11.1+zstd.1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b1365becbe415f3f0fcd024e2f7b45bacfb5bdd055f0dc113571394114e7bdd"
+checksum = "77a16b8414fde0414e90c612eba70985577451c4c504b99885ebed24762cb81a"
 dependencies = [
  "zstd-safe",
 ]
 
 [[package]]
 name = "zstd-safe"
-version = "4.1.4+zstd.1.5.2"
+version = "5.0.1+zstd.1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f7cd17c9af1a4d6c24beb1cc54b17e2ef7b593dc92f19e9d9acad8b182bbaee"
+checksum = "7c12659121420dd6365c5c3de4901f97145b79651fb1d25814020ed2ed0585ae"
 dependencies = [
  "libc",
  "zstd-sys",
@@ -5339,9 +5359,9 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "1.6.3+zstd.1.5.2"
+version = "2.0.1+zstd.1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8"
+checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b"
 dependencies = [
  "cc",
  "libc",

Cargo.toml

@@ -10,8 +10,8 @@ license = "MIT"
 name = "nu"
 readme = "README.md"
 repository = "https://github.com/nushell/nushell"
-rust-version = "1.59"
+rust-version = "1.60"
-version = "0.62.0"
+version = "0.63.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -38,27 +38,28 @@ ctrlc = "3.2.1"
 log = "0.4"
 miette = "4.5.0"
 nu-ansi-term = "0.45.1"
-nu-cli = { path="./crates/nu-cli", version = "0.62.0"  }
+nu-cli = { path="./crates/nu-cli", version = "0.63.0"  }
-nu-color-config = { path = "./crates/nu-color-config", version = "0.62.0"  }
+nu-color-config = { path = "./crates/nu-color-config", version = "0.63.0"  }
-nu-command = { path="./crates/nu-command", version = "0.62.0"  }
+nu-command = { path="./crates/nu-command", version = "0.63.0"  }
-nu-engine = { path="./crates/nu-engine", version = "0.62.0"  }
+nu-engine = { path="./crates/nu-engine", version = "0.63.0"  }
-nu-json = { path="./crates/nu-json", version = "0.62.0"  }
+nu-json = { path="./crates/nu-json", version = "0.63.0"  }
-nu-parser = { path="./crates/nu-parser", version = "0.62.0"  }
+nu-parser = { path="./crates/nu-parser", version = "0.63.0"  }
-nu-path = { path="./crates/nu-path", version = "0.62.0"  }
+nu-path = { path="./crates/nu-path", version = "0.63.0"  }
-nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.62.0"  }
+nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.63.0"  }
-nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.62.0"  }
+nu-pretty-hex = { path = "./crates/nu-pretty-hex", version = "0.63.0"  }
-nu-protocol = { path = "./crates/nu-protocol", version = "0.62.0"  }
+nu-protocol = { path = "./crates/nu-protocol", version = "0.63.0"  }
-nu-system = { path = "./crates/nu-system", version = "0.62.0" }
+nu-system = { path = "./crates/nu-system", version = "0.63.0" }
-nu-table = { path = "./crates/nu-table", version = "0.62.0"  }
+nu-table = { path = "./crates/nu-table", version = "0.63.0"  }
-nu-term-grid = { path = "./crates/nu-term-grid", version = "0.62.0"  }
+nu-term-grid = { path = "./crates/nu-term-grid", version = "0.63.0"  }
+nu-utils = { path = "./crates/nu-utils", version = "0.63.0"  }
 openssl = { version = "0.10.38", features = ["vendored"], optional = true }
 pretty_env_logger = "0.4.0"
 rayon = "1.5.1"
-reedline = { version = "0.5.0", features = ["bashisms"]}
+reedline = { version = "0.6.0", features = ["bashisms"]}
 is_executable = "1.0.1"
 
 [dev-dependencies]
-nu-test-support = { path="./crates/nu-test-support", version = "0.62.0"  }
+nu-test-support = { path="./crates/nu-test-support", version = "0.63.0"  }
 tempfile = "3.2.0"
 assert_cmd = "2.0.2"
 pretty_assertions = "1.0.0"
@@ -103,6 +104,13 @@ inherits = "release"
 strip = false
 debug = true
 
+# build with `cargo build --profile ci`
+# to analyze performance with tooling like linux perf
+[profile.ci]
+inherits = "dev"
+strip = false
+debug = false
+
 # Main nu binary
 [[bin]]
 name = "nu"

README.md

@@ -43,7 +43,7 @@ You can also find information on more specific topics in our [cookbook](https://
 
 Up-to-date installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/installation.html). **Windows users**: please note that Nu works on Windows 10 and does not currently have Windows 7/8.1 support.
 
-To build Nu, you will need to use the **latest stable (1.59 or later)** version of the compiler.
+To build Nu, you will need to use the **latest stable (1.60 or later)** version of the compiler.
 
 Required dependencies:
 

assets/nushell.rc

@@ -1,10 +1,10 @@
 #include <winver.h>
 
-#define VER_FILEVERSION 0,59,1,0
+#define VER_FILEVERSION 0,62,1,0
-#define VER_FILEVERSION_STR "0.59.1"
+#define VER_FILEVERSION_STR "0.62.1"
 
-#define VER_PRODUCTVERSION 0,59,1,0
+#define VER_PRODUCTVERSION 0,62,1,0
-#define VER_PRODUCTVERSION_STR "0.59.1"
+#define VER_PRODUCTVERSION_STR "0.62.1"
 
 #ifdef RC_INVOKED
 

crates/nu-cli/Cargo.toml

@@ -4,21 +4,21 @@ description = "CLI-related functionality for Nushell"
 edition = "2021"
 license = "MIT"
 name = "nu-cli"
-version = "0.62.0"
+version = "0.63.0"
 
 [dev-dependencies]
-nu-test-support = { path="../nu-test-support", version = "0.62.0"  }
+nu-test-support = { path="../nu-test-support", version = "0.63.0"  }
-nu-command = { path = "../nu-command", version = "0.62.0" }
+nu-command = { path = "../nu-command", version = "0.63.0" }
 
 [dependencies]
-nu-engine = { path = "../nu-engine", version = "0.62.0"  }
+nu-engine = { path = "../nu-engine", version = "0.63.0"  }
-nu-path = { path = "../nu-path", version = "0.62.0"  }
+nu-path = { path = "../nu-path", version = "0.63.0"  }
-nu-parser = { path = "../nu-parser", version = "0.62.0"  }
+nu-parser = { path = "../nu-parser", version = "0.63.0"  }
-nu-protocol = { path = "../nu-protocol", version = "0.62.0"  }
+nu-protocol = { path = "../nu-protocol", version = "0.63.0"  }
-nu-utils = { path = "../nu-utils", version = "0.62.0"  }
+nu-utils = { path = "../nu-utils", version = "0.63.0"  }
 nu-ansi-term = "0.45.1"
-nu-color-config = { path = "../nu-color-config", version = "0.62.0"  }
+nu-color-config = { path = "../nu-color-config", version = "0.63.0"  }
-reedline = { version = "0.5.0", features = ["bashisms"]}
+reedline = { version = "0.6.0", features = ["bashisms"]}
 crossterm = "0.23.0"
 miette = { version = "4.5.0", features = ["fancy"] }
 thiserror = "1.0.29"

crates/nu-cli/src/commands.rs

@@ -6,7 +6,7 @@ use nu_parser::parse;
 use nu_protocol::engine::Stack;
 use nu_protocol::{
     engine::{EngineState, StateDelta, StateWorkingSet},
-    PipelineData, Spanned,
+    PipelineData, Spanned, Value,
 };
 use std::path::Path;
 
@@ -17,9 +17,16 @@ pub fn evaluate_commands(
     stack: &mut Stack,
     input: PipelineData,
     is_perf_true: bool,
+    table_mode: Option<Value>,
 ) -> Result<()> {
     // Run a command (or commands) given to us by the user
     let (block, delta) = {
+        if let Some(ref t_mode) = table_mode {
+            let mut config = engine_state.get_config().clone();
+            config.table_mode = t_mode.as_string()?;
+            engine_state.set_config(&config);
+        }
+
         let mut working_set = StateWorkingSet::new(engine_state);
 
         let (output, err) = parse(&mut working_set, None, commands.item.as_bytes(), false, &[]);
@@ -37,12 +44,17 @@ pub fn evaluate_commands(
         report_error(&working_set, &err);
     }
 
-    let config = engine_state.get_config().clone();
+    let mut config = engine_state.get_config().clone();
+    if let Some(t_mode) = table_mode {
+        config.table_mode = t_mode.as_string()?;
+    }
 
     // Merge the delta in case env vars changed in the config
     match nu_engine::env::current_dir(engine_state, stack) {
         Ok(cwd) => {
-            if let Err(e) = engine_state.merge_delta(StateDelta::new(), Some(stack), cwd) {
+            if let Err(e) =
+                engine_state.merge_delta(StateDelta::new(engine_state), Some(stack), cwd)
+            {
                 let working_set = StateWorkingSet::new(engine_state);
                 report_error(&working_set, &e);
                 std::process::exit(1);
@@ -64,7 +76,7 @@ pub fn evaluate_commands(
 
     match eval_block(engine_state, stack, &block, input, false, false) {
         Ok(pipeline_data) => {
-            crate::eval_file::print_table_or_error(engine_state, stack, pipeline_data, &config)
+            crate::eval_file::print_table_or_error(engine_state, stack, pipeline_data, &mut config)
         }
         Err(err) => {
             let working_set = StateWorkingSet::new(engine_state);

crates/nu-cli/src/completions/command_completions.rs

@@ -1,7 +1,5 @@
-use crate::completions::{
+use crate::completions::{Completer, CompletionOptions, MatchAlgorithm, SortBy};
-    file_completions::file_path_completion, Completer, CompletionOptions, MatchAlgorithm, SortBy,
+use nu_parser::FlatShape;
-};
-use nu_parser::{unescape_unquote_string, FlatShape};
 use nu_protocol::{
     engine::{EngineState, StateWorkingSet},
     Span,
@@ -12,7 +10,6 @@ use std::sync::Arc;
 pub struct CommandCompletion {
     engine_state: Arc<EngineState>,
     flattened: Vec<(Span, FlatShape)>,
-    flat_idx: usize,
     flat_shape: FlatShape,
 }
 
@@ -21,13 +18,11 @@ impl CommandCompletion {
         engine_state: Arc<EngineState>,
         _: &StateWorkingSet,
         flattened: Vec<(Span, FlatShape)>,
-        flat_idx: usize,
         flat_shape: FlatShape,
     ) -> Self {
         Self {
             engine_state,
             flattened,
-            flat_idx,
             flat_shape,
         }
     }
@@ -39,7 +34,7 @@ impl CommandCompletion {
     ) -> Vec<String> {
         let mut executables = vec![];
 
-        let paths = self.engine_state.env_vars.get("PATH");
+        let paths = self.engine_state.get_env_var("PATH");
 
         if let Some(paths) = paths {
             if let Ok(paths) = paths.as_list() {
@@ -161,7 +156,7 @@ impl Completer for CommandCompletion {
     fn fetch(
         &mut self,
         working_set: &StateWorkingSet,
-        prefix: Vec<u8>,
+        _prefix: Vec<u8>,
         span: Span,
         offset: usize,
         pos: usize,
@@ -214,66 +209,8 @@ impl Completer for CommandCompletion {
             vec![]
         };
 
-        let cwd = if let Some(d) = self.engine_state.env_vars.get("PWD") {
+        subcommands
-            match d.as_string() {
-                Ok(s) => s,
-                Err(_) => "".to_string(),
-            }
-        } else {
-            "".to_string()
-        };
-
-        let preceding_byte = if span.start > offset {
-            working_set
-                .get_span_contents(Span {
-                    start: span.start - 1,
-                    end: span.start,
-                })
-                .to_vec()
-        } else {
-            vec![]
-        };
-        // let prefix = working_set.get_span_contents(flat.0);
-        let prefix = String::from_utf8_lossy(&prefix).to_string();
-
-        file_path_completion(span, &prefix, &cwd, options.match_algorithm)
             .into_iter()
-            .map(move |x| {
-                if self.flat_idx == 0 {
-                    // We're in the command position
-                    if (x.1.starts_with('"') || x.1.starts_with('\'') || x.1.starts_with('`'))
-                        && !matches!(preceding_byte.get(0), Some(b'^'))
-                    {
-                        let (trimmed, _) = unescape_unquote_string(x.1.as_bytes(), span);
-                        let expanded = nu_path::canonicalize_with(trimmed, &cwd);
-
-                        if let Ok(expanded) = expanded {
-                            if is_executable::is_executable(expanded) {
-                                (x.0, format!("^{}", x.1))
-                            } else {
-                                (x.0, x.1)
-                            }
-                        } else {
-                            (x.0, x.1)
-                        }
-                    } else {
-                        (x.0, x.1)
-                    }
-                } else {
-                    (x.0, x.1)
-                }
-            })
-            .map(move |x| Suggestion {
-                value: x.1,
-                description: None,
-                extra: None,
-                span: reedline::Span {
-                    start: x.0.start - offset,
-                    end: x.0.end - offset,
-                },
-                append_whitespace: false,
-            })
-            .chain(subcommands.into_iter())
             .chain(commands.into_iter())
             .collect::<Vec<_>>()
     }

crates/nu-cli/src/completions/completer.rs

@@ -56,6 +56,7 @@ impl NuCompleter {
     fn completion_helper(&mut self, line: &str, pos: usize) -> Vec<Suggestion> {
         let mut working_set = StateWorkingSet::new(&self.engine_state);
         let offset = working_set.next_span_start();
+        let initial_line = line.to_string();
         let mut line = line.to_string();
         line.insert(pos, 'a');
         let pos = offset + pos;
@@ -150,7 +151,7 @@ impl NuCompleter {
                                     self.engine_state.clone(),
                                     self.stack.clone(),
                                     *decl_id,
-                                    line,
+                                    initial_line,
                                 );
 
                                 return self.process_completion(
@@ -175,37 +176,39 @@ impl NuCompleter {
                                     pos,
                                 );
                             }
-                            FlatShape::Filepath
-                            | FlatShape::GlobPattern
-                            | FlatShape::ExternalArg => {
-                                let mut completer = FileCompletion::new(self.engine_state.clone());
-
-                                return self.process_completion(
-                                    &mut completer,
-                                    &working_set,
-                                    prefix,
-                                    new_span,
-                                    offset,
-                                    pos,
-                                );
-                            }
                             flat_shape => {
                                 let mut completer = CommandCompletion::new(
                                     self.engine_state.clone(),
                                     &working_set,
                                     flattened.clone(),
-                                    flat_idx,
+                                    // flat_idx,
                                     flat_shape.clone(),
                                 );
 
-                                return self.process_completion(
+                                let out: Vec<_> = self.process_completion(
                                     &mut completer,
                                     &working_set,
-                                    prefix,
+                                    prefix.clone(),
                                     new_span,
                                     offset,
                                     pos,
                                 );
+
+                                if out.is_empty() {
+                                    let mut completer =
+                                        FileCompletion::new(self.engine_state.clone());
+
+                                    return self.process_completion(
+                                        &mut completer,
+                                        &working_set,
+                                        prefix,
+                                        new_span,
+                                        offset,
+                                        pos,
+                                    );
+                                }
+
+                                return out;
                             }
                         };
                     }

crates/nu-cli/src/completions/directory_completions.rs

@@ -4,10 +4,11 @@ use nu_protocol::{
     levenshtein_distance, Span,
 };
 use reedline::Suggestion;
+use std::fs;
 use std::path::Path;
 use std::sync::Arc;
 
-use super::{partial_from, prepend_base_dir, MatchAlgorithm};
+use super::{partial_from, prepend_base_dir};
 
 const SEP: char = std::path::MAIN_SEPARATOR;
 
@@ -32,7 +33,7 @@ impl Completer for DirectoryCompletion {
         _: usize,
         options: &CompletionOptions,
     ) -> Vec<Suggestion> {
-        let cwd = if let Some(d) = self.engine_state.env_vars.get("PWD") {
+        let cwd = if let Some(d) = self.engine_state.get_env_var("PWD") {
             match d.as_string() {
                 Ok(s) => s,
                 Err(_) => "".to_string(),
@@ -43,7 +44,7 @@ impl Completer for DirectoryCompletion {
         let partial = String::from_utf8_lossy(&prefix).to_string();
 
         // Filter only the folders
-        let output: Vec<_> = directory_completion(span, &partial, &cwd, options.match_algorithm)
+        let output: Vec<_> = directory_completion(span, &partial, &cwd, options)
             .into_iter()
             .map(move |x| Suggestion {
                 value: x.1,
@@ -102,7 +103,7 @@ pub fn directory_completion(
     span: nu_protocol::Span,
     partial: &str,
     cwd: &str,
-    match_algorithm: MatchAlgorithm,
+    options: &CompletionOptions,
 ) -> Vec<(nu_protocol::Span, String)> {
     let original_input = partial;
 
@@ -120,10 +121,10 @@ pub fn directory_completion(
         return result
             .filter_map(|entry| {
                 entry.ok().and_then(|entry| {
-                    if let Ok(metadata) = entry.metadata() {
+                    if let Ok(metadata) = fs::metadata(entry.path()) {
                         if metadata.is_dir() {
                             let mut file_name = entry.file_name().to_string_lossy().into_owned();
-                            if matches(&partial, &file_name, match_algorithm) {
+                            if matches(&partial, &file_name, options) {
                                 let mut path = if prepend_base_dir(original_input, &base_dir_name) {
                                     format!("{}{}", base_dir_name, file_name)
                                 } else {

crates/nu-cli/src/completions/dotnu_completions.rs

@@ -37,7 +37,7 @@ impl Completer for DotNuCompletion {
 
         // Fetch the lib dirs
         let lib_dirs: Vec<String> =
-            if let Some(lib_dirs) = self.engine_state.env_vars.get("NU_LIB_DIRS") {
+            if let Some(lib_dirs) = self.engine_state.get_env_var("NU_LIB_DIRS") {
                 lib_dirs
                     .as_list()
                     .into_iter()
@@ -58,7 +58,7 @@ impl Completer for DotNuCompletion {
             };
 
         // Check if the base_dir is a folder
-        if base_dir != "./" {
+        if base_dir != format!(".{}", SEP) {
             // Add the base dir into the directories to be searched
             search_dirs.push(base_dir.clone());
 
@@ -70,7 +70,7 @@ impl Completer for DotNuCompletion {
             partial = base_dir_partial;
         } else {
             // Fetch the current folder
-            let current_folder = if let Some(d) = self.engine_state.env_vars.get("PWD") {
+            let current_folder = if let Some(d) = self.engine_state.get_env_var("PWD") {
                 match d.as_string() {
                     Ok(s) => s,
                     Err(_) => "".to_string(),
@@ -91,7 +91,7 @@ impl Completer for DotNuCompletion {
         let output: Vec<Suggestion> = search_dirs
             .into_iter()
             .flat_map(|it| {
-                file_path_completion(span, &partial, &it, options.match_algorithm)
+                file_path_completion(span, &partial, &it, options)
                     .into_iter()
                     .filter(|it| {
                         // Different base dir, so we list the .nu files or folders

crates/nu-cli/src/completions/file_completions.rs

@@ -1,4 +1,4 @@
-use crate::completions::{Completer, CompletionOptions, MatchAlgorithm};
+use crate::completions::{Completer, CompletionOptions};
 use nu_protocol::{
     engine::{EngineState, StateWorkingSet},
     levenshtein_distance, Span,
@@ -30,7 +30,7 @@ impl Completer for FileCompletion {
         _: usize,
         options: &CompletionOptions,
     ) -> Vec<Suggestion> {
-        let cwd = if let Some(d) = self.engine_state.env_vars.get("PWD") {
+        let cwd = if let Some(d) = self.engine_state.get_env_var("PWD") {
             match d.as_string() {
                 Ok(s) => s,
                 Err(_) => "".to_string(),
@@ -39,7 +39,7 @@ impl Completer for FileCompletion {
             "".to_string()
         };
         let prefix = String::from_utf8_lossy(&prefix).to_string();
-        let output: Vec<_> = file_path_completion(span, &prefix, &cwd, options.match_algorithm)
+        let output: Vec<_> = file_path_completion(span, &prefix, &cwd, options)
             .into_iter()
             .map(move |x| Suggestion {
                 value: x.1,
@@ -112,7 +112,7 @@ pub fn file_path_completion(
     span: nu_protocol::Span,
     partial: &str,
     cwd: &str,
-    match_algorithm: MatchAlgorithm,
+    options: &CompletionOptions,
 ) -> Vec<(nu_protocol::Span, String)> {
     let original_input = partial;
     let (base_dir_name, partial) = partial_from(partial);
@@ -129,7 +129,7 @@ pub fn file_path_completion(
             .filter_map(|entry| {
                 entry.ok().and_then(|entry| {
                     let mut file_name = entry.file_name().to_string_lossy().into_owned();
-                    if matches(&partial, &file_name, match_algorithm) {
+                    if matches(&partial, &file_name, options) {
                         let mut path = if prepend_base_dir(original_input, &base_dir_name) {
                             format!("{}{}", base_dir_name, file_name)
                         } else {
@@ -158,8 +158,15 @@ pub fn file_path_completion(
     Vec::new()
 }
 
-pub fn matches(partial: &str, from: &str, match_algorithm: MatchAlgorithm) -> bool {
+pub fn matches(partial: &str, from: &str, options: &CompletionOptions) -> bool {
-    match_algorithm.matches_str(&from.to_ascii_lowercase(), &partial.to_ascii_lowercase())
+    // Check for case sensitive
+    if !options.case_sensitive {
+        return options
+            .match_algorithm
+            .matches_str(&from.to_ascii_lowercase(), &partial.to_ascii_lowercase());
+    }
+
+    options.match_algorithm.matches_str(from, partial)
 }
 
 /// Returns whether the base_dir should be prepended to the file path

crates/nu-cli/src/completions/variable_completions.rs

@@ -11,7 +11,7 @@ use std::sync::Arc;
 
 #[derive(Clone)]
 pub struct VariableCompletion {
-    engine_state: Arc<EngineState>,
+    engine_state: Arc<EngineState>, // TODO: Is engine state necessary? It's already a part of working set in fetch()
     stack: Stack,
     var_context: (Vec<u8>, Vec<Vec<u8>>), // tuple with $var and the sublevels (.b.c.d)
 }
@@ -143,24 +143,39 @@ impl Completer for VariableCompletion {
             }
         }
 
+        // TODO: The following can be refactored (see find_commands_by_predicate() used in
+        // command_completions).
+        let mut removed_overlays = vec![];
         // Working set scope vars
-        for scope in &working_set.delta.scope {
+        for scope_frame in working_set.delta.scope.iter().rev() {
-            for v in &scope.vars {
+            for overlay_frame in scope_frame
-                if options.match_algorithm.matches_u8(v.0, &prefix) {
+                .active_overlays(&mut removed_overlays)
-                    output.push(Suggestion {
+                .iter()
-                        value: String::from_utf8_lossy(v.0).to_string(),
+                .rev()
-                        description: None,
+            {
-                        extra: None,
+                for v in &overlay_frame.vars {
-                        span: current_span,
+                    if options.match_algorithm.matches_u8(v.0, &prefix) {
-                        append_whitespace: false,
+                        output.push(Suggestion {
-                    });
+                            value: String::from_utf8_lossy(v.0).to_string(),
+                            description: None,
+                            extra: None,
+                            span: current_span,
+                            append_whitespace: false,
+                        });
+                    }
                 }
             }
         }
 
         // Permanent state vars
-        for scope in &self.engine_state.scope {
+        // for scope in &self.engine_state.scope {
-            for v in &scope.vars {
+        for overlay_frame in self
+            .engine_state
+            .active_overlays(&removed_overlays)
+            .iter()
+            .rev()
+        {
+            for v in &overlay_frame.vars {
                 if options.match_algorithm.matches_u8(v.0, &prefix) {
                     output.push(Suggestion {
                         value: String::from_utf8_lossy(v.0).to_string(),
@@ -173,7 +188,7 @@ impl Completer for VariableCompletion {
             }
         }
 
-        output.dedup();
+        output.dedup(); // TODO: Removes only consecutive duplicates, is it intended?
 
         output
     }

crates/nu-cli/src/config_files.rs

@@ -69,7 +69,9 @@ pub fn eval_config_contents(
             // Merge the delta in case env vars changed in the config
             match nu_engine::env::current_dir(engine_state, stack) {
                 Ok(cwd) => {
-                    if let Err(e) = engine_state.merge_delta(StateDelta::new(), Some(stack), cwd) {
+                    if let Err(e) =
+                        engine_state.merge_delta(StateDelta::new(engine_state), Some(stack), cwd)
+                    {
                         let working_set = StateWorkingSet::new(engine_state);
                         report_error(&working_set, &e);
                     }

crates/nu-cli/src/eval_file.rs

@@ -9,7 +9,7 @@ use nu_protocol::{
     engine::{EngineState, Stack, StateWorkingSet},
     Config, PipelineData, Span, Value,
 };
-use std::io::Write;
+use nu_utils::stdout_write_all_and_flush;
 
 /// Main function used when a file path is found as argument for nu
 pub fn evaluate_file(
@@ -61,17 +61,20 @@ pub fn evaluate_file(
 }
 
 pub fn print_table_or_error(
-    engine_state: &EngineState,
+    engine_state: &mut EngineState,
     stack: &mut Stack,
     mut pipeline_data: PipelineData,
-    config: &Config,
+    config: &mut Config,
 ) {
     let exit_code = match &mut pipeline_data {
         PipelineData::ExternalStream { exit_code, .. } => exit_code.take(),
         _ => None,
     };
 
-    match engine_state.find_decl("table".as_bytes()) {
+    // Change the engine_state config to use the passed in configuration
+    engine_state.set_config(config);
+
+    match engine_state.find_decl("table".as_bytes(), &[]) {
         Some(decl_id) => {
             let table = engine_state.get_decl(decl_id).run(
                 engine_state,
@@ -83,8 +86,6 @@ pub fn print_table_or_error(
             match table {
                 Ok(table) => {
                     for item in table {
-                        let stdout = std::io::stdout();
-
                         if let Value::Error { error } = item {
                             let working_set = StateWorkingSet::new(engine_state);
 
@@ -96,10 +97,7 @@ pub fn print_table_or_error(
                         let mut out = item.into_string("\n", config);
                         out.push('\n');
 
-                        match stdout.lock().write_all(out.as_bytes()) {
+                        let _ = stdout_write_all_and_flush(out).map_err(|err| eprintln!("{}", err));
-                            Ok(_) => (),
-                            Err(err) => eprintln!("{}", err),
-                        };
                     }
                 }
                 Err(error) => {
@@ -113,8 +111,6 @@ pub fn print_table_or_error(
         }
         None => {
             for item in pipeline_data {
-                let stdout = std::io::stdout();
-
                 if let Value::Error { error } = item {
                     let working_set = StateWorkingSet::new(engine_state);
 
@@ -126,10 +122,7 @@ pub fn print_table_or_error(
                 let mut out = item.into_string("\n", config);
                 out.push('\n');
 
-                match stdout.lock().write_all(out.as_bytes()) {
+                let _ = stdout_write_all_and_flush(out).map_err(|err| eprintln!("{}", err));
-                    Ok(_) => (),
-                    Err(err) => eprintln!("{}", err),
-                };
             }
         }
     };

crates/nu-cli/src/menus/description_menu.rs

@@ -586,7 +586,7 @@ impl Menu for DescriptionMenu {
                         } else {
                             self.example_index = Some(self.examples.len().saturating_sub(1));
                         }
-                    } else {
+                    } else if !self.examples.is_empty() {
                         self.example_index = Some(0);
                     }
                 }
@@ -598,7 +598,7 @@ impl Menu for DescriptionMenu {
                         } else {
                             self.example_index = Some(0);
                         }
-                    } else {
+                    } else if !self.examples.is_empty() {
                         self.example_index = Some(0);
                     }
                 }

crates/nu-cli/src/print.rs

@@ -16,6 +16,11 @@ impl Command for Print {
     fn signature(&self) -> Signature {
         Signature::build("print")
             .rest("rest", SyntaxShape::Any, "the values to print")
+            .switch(
+                "no_newline",
+                "print without inserting a newline for the line ending",
+                Some('n'),
+            )
             .category(Category::Strings)
     }
 
@@ -31,10 +36,12 @@ impl Command for Print {
         _input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
         let args: Vec<Value> = call.rest(engine_state, stack, 0)?;
+        let no_newline = call.has_flag("no_newline");
         let head = call.head;
 
         for arg in args {
-            arg.into_pipeline_data().print(engine_state, stack)?;
+            arg.into_pipeline_data()
+                .print(engine_state, stack, no_newline)?;
         }
 
         Ok(PipelineData::new(head))

crates/nu-cli/src/prompt.rs

@@ -7,9 +7,6 @@ use {
     std::borrow::Cow,
 };
 
-const PROMPT_MARKER_BEFORE_PS1: &str = "\x1b]133;A\x1b\\"; // OSC 133;A ST
-const PROMPT_MARKER_BEFORE_PS2: &str = "\x1b]133;A;k=s\x1b\\"; // OSC 133;A;k=s ST
-
 /// Nushell prompt definition
 #[derive(Clone)]
 pub struct NushellPrompt {
@@ -19,7 +16,6 @@ pub struct NushellPrompt {
     default_vi_insert_prompt_indicator: Option<String>,
     default_vi_normal_prompt_indicator: Option<String>,
     default_multiline_indicator: Option<String>,
-    shell_integration: bool,
 }
 
 impl Default for NushellPrompt {
@@ -37,7 +33,6 @@ impl NushellPrompt {
             default_vi_insert_prompt_indicator: None,
             default_vi_normal_prompt_indicator: None,
             default_multiline_indicator: None,
-            shell_integration: false,
         }
     }
 
@@ -87,34 +82,20 @@ impl NushellPrompt {
     fn default_wrapped_custom_string(&self, str: String) -> String {
         format!("({})", str)
     }
-
-    pub(crate) fn enable_shell_integration(&mut self) {
-        self.shell_integration = true
-    }
 }
 
 impl Prompt for NushellPrompt {
     fn render_prompt_left(&self) -> Cow<str> {
-        // Just before starting to draw the PS1 prompt send the escape code (see
+        if let Some(prompt_string) = &self.left_prompt_string {
-        // https://sw.kovidgoyal.net/kitty/shell-integration/#notes-for-shell-developers)
+            prompt_string.replace('\n', "\r\n").into()
-        let mut prompt = if self.shell_integration {
-            String::from(PROMPT_MARKER_BEFORE_PS1)
         } else {
-            String::new()
+            let default = DefaultPrompt::new();
-        };
+            default
-
+                .render_prompt_left()
-        prompt.push_str(&match &self.left_prompt_string {
+                .to_string()
-            Some(prompt_string) => prompt_string.replace('\n', "\r\n"),
+                .replace('\n', "\r\n")
-            None => {
+                .into()
-                let default = DefaultPrompt::new();
+        }
-                default
-                    .render_prompt_left()
-                    .to_string()
-                    .replace('\n', "\r\n")
-            }
-        });
-
-        prompt.into()
     }
 
     fn render_prompt_right(&self) -> Cow<str> {
@@ -155,21 +136,10 @@ impl Prompt for NushellPrompt {
     }
 
     fn render_prompt_multiline_indicator(&self) -> Cow<str> {
-        // Just before starting to draw the PS1 prompt send the escape code (see
+        match &self.default_multiline_indicator {
-        // https://sw.kovidgoyal.net/kitty/shell-integration/#notes-for-shell-developers)
+            Some(indicator) => indicator.as_str().into(),
-        let mut prompt = if self.shell_integration {
+            None => "::: ".into(),
-            String::from(PROMPT_MARKER_BEFORE_PS2)
+        }
-        } else {
-            String::new()
-        };
-
-        prompt.push_str(
-            self.default_multiline_indicator
-                .as_ref()
-                .unwrap_or(&String::from("::: ")),
-        );
-
-        prompt.into()
     }
 
     fn render_prompt_history_search_indicator(

crates/nu-cli/src/prompt_update.rs

@@ -147,10 +147,6 @@ pub(crate) fn update_prompt<'prompt>(
         (prompt_vi_insert_string, prompt_vi_normal_string),
     );
 
-    if config.shell_integration {
-        nu_prompt.enable_shell_integration();
-    }
-
     let ret_val = nu_prompt as &dyn Prompt;
     if is_perf_true {
         info!("update_prompt {}:{}:{}", file!(), line!(), column!());

crates/nu-cli/src/reedline_config.rs

@@ -24,7 +24,7 @@ const DEFAULT_COMPLETION_MENU: &str = r#"
   type: {
       layout: columnar
       columns: 4
-      col_width: 20   
+      col_width: 20
       col_padding: 2
   }
   style: {
@@ -58,7 +58,7 @@ const DEFAULT_HELP_MENU: &str = r#"
   type: {
       layout: description
       columns: 4
-      col_width: 20   
+      col_width: 20
       col_padding: 2
       selection_rows: 4
       description_rows: 10
@@ -501,14 +501,16 @@ fn add_menu_keybindings(keybindings: &mut Keybindings) {
         ReedlineEvent::MenuPrevious,
     );
 
-    // History menu keybinding
+    keybindings.add_binding(
+        KeyModifiers::CONTROL,
+        KeyCode::Char('r'),
+        ReedlineEvent::Menu("history_menu".to_string()),
+    );
+
     keybindings.add_binding(
         KeyModifiers::CONTROL,
         KeyCode::Char('x'),
-        ReedlineEvent::UntilFound(vec![
+        ReedlineEvent::MenuPageNext,
-            ReedlineEvent::Menu("history_menu".to_string()),
-            ReedlineEvent::MenuPageNext,
-        ]),
     );
 
     keybindings.add_binding(
@@ -522,8 +524,8 @@ fn add_menu_keybindings(keybindings: &mut Keybindings) {
 
     // Help menu keybinding
     keybindings.add_binding(
-        KeyModifiers::CONTROL,
+        KeyModifiers::NONE,
-        KeyCode::Char('q'),
+        KeyCode::F(1),
         ReedlineEvent::Menu("help_menu".to_string()),
     );
 }

crates/nu-cli/src/repl.rs

@@ -1,28 +1,26 @@
-use crate::reedline_config::add_menus;
-use crate::{completions::NuCompleter, NuHighlighter, NuValidator, NushellPrompt};
-use crate::{prompt_update, reedline_config};
 use crate::{
-    reedline_config::KeybindingsMode,
+    completions::NuCompleter,
+    prompt_update,
+    reedline_config::{add_menus, create_keybindings, KeybindingsMode},
     util::{eval_source, report_error},
+    NuHighlighter, NuValidator, NushellPrompt,
 };
-use log::info;
+use log::{info, trace};
-use log::trace;
 use miette::{IntoDiagnostic, Result};
 use nu_color_config::get_color_config;
-use nu_engine::convert_env_values;
+use nu_engine::{convert_env_values, eval_block};
 use nu_parser::lex;
-use nu_protocol::engine::Stack;
-use nu_protocol::PipelineData;
 use nu_protocol::{
-    engine::{EngineState, StateWorkingSet},
+    engine::{EngineState, Stack, StateWorkingSet},
-    ShellError, Span, Value,
+    BlockId, PipelineData, PositionalArg, ShellError, Span, Value,
 };
 use reedline::{DefaultHinter, Emacs, Vi};
 use std::io::{self, Write};
 use std::path::PathBuf;
 use std::{sync::atomic::Ordering, time::Instant};
 
-const PROMPT_MARKER_BEFORE_CMD: &str = "\x1b]133;C\x1b\\"; // OSC 133;C ST
+const PRE_EXECUTE_MARKER: &str = "\x1b]133;A\x1b\\";
+const PRE_PROMPT_MARKER: &str = "\x1b]133;C\x1b\\";
 const RESET_APPLICATION_MODE: &str = "\x1b[?1l";
 
 pub fn evaluate_repl(
@@ -161,7 +159,26 @@ pub fn evaluate_repl(
             }
         };
 
-        line_editor = line_editor.with_buffer_editor(config.buffer_editor.clone(), "nu".into());
+        let buffer_editor = if !config.buffer_editor.is_empty() {
+            Some(config.buffer_editor.clone())
+        } else {
+            stack
+                .get_env_var(engine_state, "EDITOR")
+                .map(|v| v.as_string().unwrap_or_default())
+                .filter(|v| !v.is_empty())
+                .or_else(|| {
+                    stack
+                        .get_env_var(engine_state, "VISUAL")
+                        .map(|v| v.as_string().unwrap_or_default())
+                        .filter(|v| !v.is_empty())
+                })
+        };
+
+        line_editor = if let Some(buffer_editor) = buffer_editor {
+            line_editor.with_buffer_editor(buffer_editor, "nu".into())
+        } else {
+            line_editor
+        };
 
         if config.sync_history_on_enter {
             if is_perf_true {
@@ -175,7 +192,7 @@ pub fn evaluate_repl(
         }
 
         // Changing the line editor based on the found keybindings
-        line_editor = match reedline_config::create_keybindings(config) {
+        line_editor = match create_keybindings(config) {
             Ok(keybindings) => match keybindings {
                 KeybindingsMode::Emacs(keybindings) => {
                     let edit_mode = Box::new(Emacs::new(keybindings));
@@ -200,6 +217,65 @@ pub fn evaluate_repl(
             info!("prompt_update {}:{}:{}", file!(), line!(), column!());
         }
 
+        // Right before we start our prompt and take input from the user,
+        // fire the "pre_prompt" hook
+        if let Some(hook) = &config.hooks.pre_prompt {
+            if let Err(err) = run_hook(engine_state, stack, vec![], hook) {
+                let working_set = StateWorkingSet::new(engine_state);
+                report_error(&working_set, &err);
+            }
+        }
+
+        // Next, check all the environment variables they ask for
+        // fire the "env_change" hook
+        if let Some(hook) = config.hooks.env_change.clone() {
+            match hook {
+                Value::Record {
+                    cols, vals: blocks, ..
+                } => {
+                    for (idx, env_var) in cols.iter().enumerate() {
+                        let before = engine_state
+                            .previous_env_vars
+                            .get(env_var)
+                            .cloned()
+                            .unwrap_or_default();
+                        let after = stack.get_env_var(engine_state, env_var).unwrap_or_default();
+                        if before != after {
+                            if let Err(err) = run_hook(
+                                engine_state,
+                                stack,
+                                vec![before, after.clone()],
+                                &blocks[idx],
+                            ) {
+                                let working_set = StateWorkingSet::new(engine_state);
+                                report_error(&working_set, &err);
+                            }
+
+                            engine_state
+                                .previous_env_vars
+                                .insert(env_var.to_string(), after);
+                        }
+                    }
+                }
+                x => {
+                    let working_set = StateWorkingSet::new(engine_state);
+                    report_error(
+                        &working_set,
+                        &ShellError::TypeMismatch(
+                            "record for 'env_change' hook".to_string(),
+                            x.span().unwrap_or_else(|_| Span::new(0, 0)),
+                        ),
+                    )
+                }
+            }
+        }
+
+        config = engine_state.get_config();
+
+        if config.shell_integration {
+            run_ansi_sequence(PRE_EXECUTE_MARKER)?;
+        }
+
         let prompt =
             prompt_update::update_prompt(config, engine_state, stack, &mut nu_prompt, is_perf_true);
 
@@ -215,10 +291,39 @@ pub fn evaluate_repl(
         }
 
         let input = line_editor.read_line(prompt);
-        let use_shell_integration = config.shell_integration;
 
         match input {
             Ok(Signal::Success(s)) => {
+                // Right before we start running the code the user gave us,
+                // fire the "pre_execution" hook
+                if let Some(hook) = &config.hooks.pre_execution {
+                    if let Err(err) = run_hook(engine_state, stack, vec![], hook) {
+                        let working_set = StateWorkingSet::new(engine_state);
+                        report_error(&working_set, &err);
+                    }
+                }
+
+                if config.shell_integration {
+                    run_ansi_sequence(RESET_APPLICATION_MODE)?;
+                    run_ansi_sequence(PRE_PROMPT_MARKER)?;
+                    if let Some(cwd) = stack.get_env_var(engine_state, "PWD") {
+                        let path = cwd.as_string()?;
+                        // Try to abbreviate string for windows title
+                        let maybe_abbrev_path = if let Some(p) = nu_path::home_dir() {
+                            path.replace(&p.as_path().display().to_string(), "~")
+                        } else {
+                            path
+                        };
+
+                        // Set window title too
+                        // https://tldp.org/HOWTO/Xterm-Title-3.html
+                        // ESC]0;stringBEL -- Set icon name and window title to string
+                        // ESC]1;stringBEL -- Set icon name to string
+                        // ESC]2;stringBEL -- Set window title to string
+                        run_ansi_sequence(&format!("\x1b]2;{}\x07", maybe_abbrev_path))?;
+                    }
+                }
+
                 let start_time = Instant::now();
                 let tokens = lex(s.as_bytes(), 0, &[], &[], false);
                 // Check if this is a single call to a directory, if so auto-cd
@@ -244,7 +349,6 @@ pub fn evaluate_repl(
                                 &ShellError::DirectoryNotFound(tokens.0[0].span, None),
                             );
                         }
-
                         let path = nu_path::canonicalize_with(path, &cwd)
                             .expect("internal error: cannot canonicalize known path");
                         (path.to_string_lossy().to_string(), tokens.0[0].span)
@@ -290,37 +394,22 @@ pub fn evaluate_repl(
                         &format!("entry #{}", entry_num),
                         PipelineData::new(Span::new(0, 0)),
                     );
-
-                    stack.add_env_var(
-                        "CMD_DURATION_MS".into(),
-                        Value::String {
-                            val: format!("{}", start_time.elapsed().as_millis()),
-                            span: Span { start: 0, end: 0 },
-                        },
-                    );
                 }
+
+                stack.add_env_var(
+                    "CMD_DURATION_MS".into(),
+                    Value::String {
+                        val: format!("{}", start_time.elapsed().as_millis()),
+                        span: Span { start: 0, end: 0 },
+                    },
+                );
+
                 // FIXME: permanent state changes like this hopefully in time can be removed
                 // and be replaced by just passing the cwd in where needed
                 if let Some(cwd) = stack.get_env_var(engine_state, "PWD") {
                     let path = cwd.as_string()?;
                     let _ = std::env::set_current_dir(path);
-                    engine_state.env_vars.insert("PWD".into(), cwd);
+                    engine_state.add_env_var("PWD".into(), cwd);
-                }
-
-                if use_shell_integration {
-                    // Just before running a command/program, send the escape code (see
-                    // https://sw.kovidgoyal.net/kitty/shell-integration/#notes-for-shell-developers)
-                    let mut ansi_escapes = String::from(PROMPT_MARKER_BEFORE_CMD);
-                    ansi_escapes.push_str(RESET_APPLICATION_MODE);
-                    if let Some(cwd) = stack.get_env_var(engine_state, "PWD") {
-                        let path = cwd.as_string()?;
-                        ansi_escapes.push_str(&format!("\x1b]2;{}\x07", path));
-                    }
-                    // print!("{}", ansi_escapes);
-                    match io::stdout().write_all(ansi_escapes.as_bytes()) {
-                        Ok(it) => it,
-                        Err(err) => print!("error: {}", err),
-                    };
                 }
             }
             Ok(Signal::CtrlC) => {
@@ -342,3 +431,87 @@ pub fn evaluate_repl(
 
     Ok(())
 }
+
+fn run_ansi_sequence(seq: &str) -> Result<(), ShellError> {
+    match io::stdout().write_all(seq.as_bytes()) {
+        Ok(it) => it,
+        Err(err) => {
+            return Err(ShellError::GenericError(
+                "Error writing ansi sequence".into(),
+                err.to_string(),
+                Some(Span { start: 0, end: 0 }),
+                None,
+                Vec::new(),
+            ));
+        }
+    };
+    io::stdout().flush().map_err(|e| {
+        ShellError::GenericError(
+            "Error flushing stdio".into(),
+            e.to_string(),
+            Some(Span { start: 0, end: 0 }),
+            None,
+            Vec::new(),
+        )
+    })
+}
+
+pub fn run_hook(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    arguments: Vec<Value>,
+    value: &Value,
+) -> Result<(), ShellError> {
+    match value {
+        Value::List { vals, .. } => {
+            for val in vals {
+                run_hook(engine_state, stack, arguments.clone(), val)?
+            }
+            Ok(())
+        }
+        Value::Block {
+            val: block_id,
+            span,
+            ..
+        } => run_hook_block(engine_state, stack, *block_id, arguments, *span),
+        x => match x.span() {
+            Ok(span) => Err(ShellError::MissingConfigValue(
+                "block for hook in config".into(),
+                span,
+            )),
+            _ => Err(ShellError::MissingConfigValue(
+                "block for hook in config".into(),
+                Span { start: 0, end: 0 },
+            )),
+        },
+    }
+}
+
+pub fn run_hook_block(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    block_id: BlockId,
+    arguments: Vec<Value>,
+    span: Span,
+) -> Result<(), ShellError> {
+    let block = engine_state.get_block(block_id);
+    let input = PipelineData::new(span);
+
+    let mut callee_stack = stack.gather_captures(&block.captures);
+
+    for (idx, PositionalArg { var_id, .. }) in
+        block.signature.required_positional.iter().enumerate()
+    {
+        if let Some(var_id) = var_id {
+            callee_stack.add_var(*var_id, arguments[idx].clone())
+        }
+    }
+
+    match eval_block(engine_state, &mut callee_stack, block, input, false, false) {
+        Ok(pipeline_data) => match pipeline_data.into_value(span) {
+            Value::Error { error } => Err(error),
+            _ => Ok(()),
+        },
+        Err(err) => Err(err),
+    }
+}

crates/nu-cli/src/util.rs

@@ -179,7 +179,7 @@ fn gather_env_vars(vars: impl Iterator<Item = (String, String)>, engine_state: &
             };
 
             // stack.add_env_var(name, value);
-            engine_state.env_vars.insert(name, value);
+            engine_state.add_env_var(name, value);
         }
     }
 }
@@ -211,8 +211,8 @@ pub fn eval_source(
         (output, working_set.render())
     };
 
-    let cwd = match nu_engine::env::current_dir_str(engine_state, stack) {
+    let cwd = match nu_engine::env::current_dir(engine_state, stack) {
-        Ok(p) => PathBuf::from(p),
+        Ok(p) => p,
         Err(e) => {
             let working_set = StateWorkingSet::new(engine_state);
             report_error(&working_set, &e);
@@ -220,10 +220,7 @@ pub fn eval_source(
         }
     };
 
-    if let Err(err) = engine_state.merge_delta(delta, Some(stack), &cwd) {
+    let _ = engine_state.merge_delta(delta, Some(stack), &cwd);
-        let working_set = StateWorkingSet::new(engine_state);
-        report_error(&working_set, &err);
-    }
 
     match eval_block(engine_state, stack, &block, input, false, false) {
         Ok(mut pipeline_data) => {
@@ -237,7 +234,7 @@ pub fn eval_source(
                 set_last_exit_code(stack, 0);
             }
 
-            if let Err(err) = pipeline_data.print(engine_state, stack) {
+            if let Err(err) = pipeline_data.print(engine_state, stack, false) {
                 let working_set = StateWorkingSet::new(engine_state);
 
                 report_error(&working_set, &err);
@@ -319,12 +316,18 @@ mod test {
             &mut engine_state,
         );
 
-        let env = engine_state.env_vars;
+        let env = engine_state.render_env_vars();
 
-        assert!(matches!(env.get("FOO"), Some(Value::String { val, .. }) if val == "foo"));
+        assert!(
-        assert!(matches!(env.get("SYMBOLS"), Some(Value::String { val, .. }) if val == symbols));
+            matches!(env.get(&"FOO".to_string()), Some(&Value::String { val, .. }) if val == "foo")
-        assert!(matches!(env.get(symbols), Some(Value::String { val, .. }) if val == "symbols"));
+        );
-        assert!(env.get("PWD").is_some());
+        assert!(
+            matches!(env.get(&"SYMBOLS".to_string()), Some(&Value::String { val, .. }) if val == symbols)
+        );
+        assert!(
+            matches!(env.get(&symbols.to_string()), Some(&Value::String { val, .. }) if val == "symbols")
+        );
+        assert!(env.get(&"PWD".to_string()).is_some());
         assert_eq!(env.len(), 4);
     }
 }

crates/nu-cli/tests/custom_completions.rs

@@ -0,0 +1,29 @@
+pub mod support;
+
+use nu_cli::NuCompleter;
+use reedline::Completer;
+use support::{match_suggestions, new_engine};
+
+#[test]
+fn variables_completions() {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = r#"def animals [] { ["cat", "dog", "eel" ] }
+    def my-command [animal: string@animals] { print $animal }"#;
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for $nu
+    let suggestions = completer.complete("my-command ".into(), 11);
+
+    assert_eq!(3, suggestions.len());
+
+    let expected: Vec<String> = vec!["cat".into(), "dog".into(), "eel".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+}

crates/nu-cli/tests/dotnu_completions.rs

@@ -0,0 +1,28 @@
+pub mod support;
+
+use nu_cli::NuCompleter;
+use reedline::Completer;
+use support::new_engine;
+
+#[test]
+fn dotnu_completions() {
+    // Create a new engine
+    let (_, _, engine, stack) = new_engine();
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test source completion
+    let completion_str = "source ".to_string();
+    let suggestions = completer.complete(&completion_str, completion_str.len());
+
+    assert_eq!(1, suggestions.len());
+    assert_eq!("custom_completion.nu", suggestions.get(0).unwrap().value);
+
+    // Test use completion
+    let completion_str = "use ".to_string();
+    let suggestions = completer.complete(&completion_str, completion_str.len());
+
+    assert_eq!(1, suggestions.len());
+    assert_eq!("custom_completion.nu", suggestions.get(0).unwrap().value);
+}

crates/nu-cli/tests/file_completions.rs

@@ -0,0 +1,42 @@
+pub mod support;
+
+use nu_cli::NuCompleter;
+use reedline::Completer;
+use support::{file, folder, match_suggestions, new_engine};
+
+#[test]
+fn file_completions() {
+    // Create a new engine
+    let (dir, dir_str, engine, stack) = new_engine();
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for the current folder
+    let target_dir = format!("cp {}", dir_str);
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        file(dir.join("nushell")),
+        folder(dir.join("test_a")),
+        folder(dir.join("test_b")),
+        folder(dir.join("another")),
+        file(dir.join("custom_completion.nu")),
+        file(dir.join(".hidden_file")),
+        folder(dir.join(".hidden_folder")),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+
+    // Test completions for the completions/another folder
+    let target_dir = format!("cd {}", folder(dir.join("another")));
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![file(dir.join("another").join("newfile"))];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+}

crates/nu-cli/tests/flag_completions.rs

@@ -0,0 +1,36 @@
+pub mod support;
+
+use nu_cli::NuCompleter;
+use reedline::Completer;
+use support::{match_suggestions, new_engine};
+
+#[test]
+fn flag_completions() {
+    // Create a new engine
+    let (_, _, engine, stack) = new_engine();
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+    // Test completions for the 'ls' flags
+    let suggestions = completer.complete("ls -".into(), 4);
+
+    assert_eq!(12, suggestions.len());
+
+    let expected: Vec<String> = vec![
+        "--all".into(),
+        "--du".into(),
+        "--full-paths".into(),
+        "--help".into(),
+        "--long".into(),
+        "--short-names".into(),
+        "-a".into(),
+        "-d".into(),
+        "-f".into(),
+        "-h".into(),
+        "-l".into(),
+        "-s".into(),
+    ];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+}

crates/nu-cli/tests/folder_completions.rs

@@ -0,0 +1,29 @@
+pub mod support;
+
+use nu_cli::NuCompleter;
+use reedline::Completer;
+use support::{folder, match_suggestions, new_engine};
+
+#[test]
+fn folder_completions() {
+    // Create a new engine
+    let (dir, dir_str, engine, stack) = new_engine();
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for the current folder
+    let target_dir = format!("cd {}", dir_str);
+    let suggestions = completer.complete(&target_dir, target_dir.len());
+
+    // Create the expected values
+    let expected_paths: Vec<String> = vec![
+        folder(dir.join("test_a")),
+        folder(dir.join("test_b")),
+        folder(dir.join("another")),
+        folder(dir.join(".hidden_folder")),
+    ];
+
+    // Match the results
+    match_suggestions(expected_paths, suggestions);
+}

crates/nu-cli/tests/support/completions_helpers.rs

@@ -0,0 +1,107 @@
+use std::path::PathBuf;
+
+use nu_command::create_default_context;
+use nu_engine::eval_block;
+use nu_parser::parse;
+use nu_protocol::{
+    engine::{EngineState, Stack, StateDelta, StateWorkingSet},
+    PipelineData, ShellError, Span, Value,
+};
+use nu_test_support::fs;
+use reedline::Suggestion;
+const SEP: char = std::path::MAIN_SEPARATOR;
+
+// creates a new engine with the current path into the completions fixtures folder
+pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
+    // Target folder inside assets
+    let dir = fs::fixtures().join("completions");
+    let mut dir_str = dir
+        .clone()
+        .into_os_string()
+        .into_string()
+        .unwrap_or_default();
+    dir_str.push(SEP);
+
+    // Create a new engine with default context
+    let mut engine_state = create_default_context(&dir);
+
+    // New stack
+    let mut stack = Stack::new();
+
+    // New delta state
+    let delta = StateDelta::new(&engine_state);
+
+    // Add pwd as env var
+    stack.add_env_var(
+        "PWD".to_string(),
+        Value::String {
+            val: dir_str.clone(),
+            span: nu_protocol::Span {
+                start: 0,
+                end: dir_str.len(),
+            },
+        },
+    );
+
+    // Merge delta
+    let merge_result = engine_state.merge_delta(delta, Some(&mut stack), &dir);
+    assert!(merge_result.is_ok());
+
+    (dir, dir_str, engine_state, stack)
+}
+
+// match a list of suggestions with the expected values
+pub fn match_suggestions(expected: Vec<String>, suggestions: Vec<Suggestion>) {
+    expected.iter().zip(suggestions).for_each(|it| {
+        assert_eq!(it.0, &it.1.value);
+    });
+}
+
+// append the separator to the converted path
+pub fn folder(path: PathBuf) -> String {
+    let mut converted_path = file(path);
+    converted_path.push(SEP);
+
+    converted_path
+}
+
+// convert a given path to string
+pub fn file(path: PathBuf) -> String {
+    path.into_os_string().into_string().unwrap_or_default()
+}
+
+// merge_input executes the given input into the engine
+// and merges the state
+pub fn merge_input(
+    input: &[u8],
+    engine_state: &mut EngineState,
+    stack: &mut Stack,
+    dir: PathBuf,
+) -> Result<(), ShellError> {
+    let (block, delta) = {
+        let mut working_set = StateWorkingSet::new(&engine_state);
+
+        let (block, err) = parse(&mut working_set, None, input, false, &[]);
+
+        assert!(err.is_none());
+
+        (block, working_set.render())
+    };
+    assert!(eval_block(
+        &engine_state,
+        stack,
+        &block,
+        PipelineData::Value(
+            Value::Nothing {
+                span: Span { start: 0, end: 0 },
+            },
+            None
+        ),
+        false,
+        false
+    )
+    .is_ok());
+
+    // Merge delta
+    engine_state.merge_delta(delta, Some(stack), &dir)
+}

crates/nu-cli/tests/support/mod.rs

@@ -0,0 +1,3 @@
+pub mod completions_helpers;
+
+pub use completions_helpers::{file, folder, match_suggestions, merge_input, new_engine};

crates/nu-cli/tests/test_completions.rs

@@ -1,107 +0,0 @@
-use std::path::PathBuf;
-
-use nu_cli::NuCompleter;
-use nu_command::create_default_context;
-use nu_protocol::engine::{EngineState, Stack};
-use nu_test_support::fs;
-use reedline::{Completer, Suggestion};
-const SEP: char = std::path::MAIN_SEPARATOR;
-
-#[test]
-fn file_completions() {
-    // Create a new engine
-    let (dir, dir_str, engine) = new_engine();
-
-    let stack = Stack::new();
-
-    // Instatiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    // Test completions for the current folder
-    let target_dir = format!("cp {}", dir_str);
-    let suggestions = completer.complete(&target_dir, target_dir.len());
-
-    // Create the expected values
-    let expected_paths: Vec<String> = vec![
-        file(dir.join("nushell")),
-        folder(dir.join("test_a")),
-        folder(dir.join("test_b")),
-        folder(dir.join("another")),
-        file(dir.join(".hidden_file")),
-        folder(dir.join(".hidden_folder")),
-    ];
-
-    // Match the results
-    match_suggestions(expected_paths, suggestions);
-
-    // Test completions for the completions/another folder
-    let target_dir = format!("cd {}", folder(dir.join("another")));
-    let suggestions = completer.complete(&target_dir, target_dir.len());
-
-    // Create the expected values
-    let expected_paths: Vec<String> = vec![file(dir.join("another").join("newfile"))];
-
-    // Match the results
-    match_suggestions(expected_paths, suggestions);
-}
-
-#[test]
-fn folder_completions() {
-    // Create a new engine
-    let (dir, dir_str, engine) = new_engine();
-
-    let stack = Stack::new();
-
-    // Instatiate a new completer
-    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
-
-    // Test completions for the current folder
-    let target_dir = format!("cd {}", dir_str);
-    let suggestions = completer.complete(&target_dir, target_dir.len());
-
-    // Create the expected values
-    let expected_paths: Vec<String> = vec![
-        folder(dir.join("test_a")),
-        folder(dir.join("test_b")),
-        folder(dir.join("another")),
-        folder(dir.join(".hidden_folder")),
-    ];
-
-    // Match the results
-    match_suggestions(expected_paths, suggestions);
-}
-
-// creates a new engine with the current path into the completions fixtures folder
-pub fn new_engine() -> (PathBuf, String, EngineState) {
-    // Target folder inside assets
-    let dir = fs::fixtures().join("completions");
-    let mut dir_str = dir
-        .clone()
-        .into_os_string()
-        .into_string()
-        .unwrap_or_default();
-    dir_str.push(SEP);
-
-    // Create a default engine
-    (dir.clone(), dir_str, create_default_context(dir))
-}
-
-// match a list of suggestions with the expected values
-pub fn match_suggestions(expected: Vec<String>, suggestions: Vec<Suggestion>) {
-    expected.iter().zip(suggestions).for_each(|it| {
-        assert_eq!(it.0, &it.1.value);
-    });
-}
-
-// append the separator to the converted path
-pub fn folder(path: PathBuf) -> String {
-    let mut converted_path = file(path);
-    converted_path.push(SEP);
-
-    converted_path
-}
-
-// convert a given path to string
-pub fn file(path: PathBuf) -> String {
-    path.into_os_string().into_string().unwrap_or_default()
-}

crates/nu-cli/tests/variables_completions.rs

@@ -0,0 +1,57 @@
+pub mod support;
+
+use nu_cli::NuCompleter;
+use reedline::Completer;
+use support::{match_suggestions, new_engine};
+
+#[test]
+fn variables_completions() {
+    // Create a new engine
+    let (dir, _, mut engine, mut stack) = new_engine();
+
+    // Add record value as example
+    let record = "let actor = { name: 'Tom Hardy', age: 44 }";
+    assert!(support::merge_input(record.as_bytes(), &mut engine, &mut stack, dir).is_ok());
+
+    // Instatiate a new completer
+    let mut completer = NuCompleter::new(std::sync::Arc::new(engine), stack);
+
+    // Test completions for $nu
+    let suggestions = completer.complete("$nu.".into(), 4);
+
+    assert_eq!(8, suggestions.len());
+
+    let expected: Vec<String> = vec![
+        "config-path".into(),
+        "env-path".into(),
+        "history-path".into(),
+        "home-path".into(),
+        "os-info".into(),
+        "pid".into(),
+        "scope".into(),
+        "temp-path".into(),
+    ];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for custom var
+    let suggestions = completer.complete("$actor.".into(), 7);
+
+    assert_eq!(2, suggestions.len());
+
+    let expected: Vec<String> = vec!["age".into(), "name".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+
+    // Test completions for $env
+    let suggestions = completer.complete("$env.".into(), 5);
+
+    assert_eq!(1, suggestions.len());
+
+    let expected: Vec<String> = vec!["PWD".into()];
+
+    // Match results
+    match_suggestions(expected, suggestions);
+}

crates/nu-color-config/Cargo.toml

@@ -4,11 +4,11 @@ description = "Color configuration code used by Nushell"
 edition = "2021"
 license = "MIT"
 name = "nu-color-config"
-version = "0.62.0"
+version = "0.63.0"
 
 [dependencies]
-nu-protocol = { path = "../nu-protocol", version = "0.62.0"  }
+nu-protocol = { path = "../nu-protocol", version = "0.63.0"  }
 nu-ansi-term = "0.45.1"
-nu-json = { path = "../nu-json", version = "0.62.0"  }
+nu-json = { path = "../nu-json", version = "0.63.0"  }
-nu-table = { path = "../nu-table", version = "0.62.0"  }
+nu-table = { path = "../nu-table", version = "0.63.0"  }
 serde = { version="1.0.123", features=["derive"] }

crates/nu-command/Cargo.toml

@@ -4,28 +4,29 @@ description = "Nushell's built-in commands"
 edition = "2021"
 license = "MIT"
 name = "nu-command"
-version = "0.62.0"
+version = "0.63.0"
 build = "build.rs"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nu-color-config = { path = "../nu-color-config", version = "0.62.0"  }
+nu-color-config = { path = "../nu-color-config", version = "0.63.0"  }
-nu-engine = { path = "../nu-engine", version = "0.62.0"  }
+nu-engine = { path = "../nu-engine", version = "0.63.0"  }
-nu-glob = { path = "../nu-glob", version = "0.62.0" }
+nu-glob = { path = "../nu-glob", version = "0.63.0" }
-nu-json = { path = "../nu-json", version = "0.62.0"  }
+nu-json = { path = "../nu-json", version = "0.63.0"  }
-nu-parser = { path = "../nu-parser", version = "0.62.0"  }
+nu-parser = { path = "../nu-parser", version = "0.63.0"  }
-nu-path = { path = "../nu-path", version = "0.62.0"  }
+nu-path = { path = "../nu-path", version = "0.63.0"  }
-nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.62.0"  }
+nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.63.0"  }
-nu-protocol = { path = "../nu-protocol", version = "0.62.0"  }
+nu-protocol = { path = "../nu-protocol", version = "0.63.0"  }
-nu-system = { path = "../nu-system", version = "0.62.0"  }
+nu-system = { path = "../nu-system", version = "0.63.0"  }
-nu-table = { path = "../nu-table", version = "0.62.0"  }
+nu-table = { path = "../nu-table", version = "0.63.0"  }
-nu-term-grid = { path = "../nu-term-grid", version = "0.62.0"  }
+nu-term-grid = { path = "../nu-term-grid", version = "0.63.0"  }
-nu-test-support = { path = "../nu-test-support", version = "0.62.0"  }
+nu-test-support = { path = "../nu-test-support", version = "0.63.0"  }
-nu-utils = { path = "../nu-utils", version = "0.62.0" }
+nu-utils = { path = "../nu-utils", version = "0.63.0" }
 nu-ansi-term = "0.45.1"
 
 # Potential dependencies for extras
+alphanumeric-sort = "1.4.4"
 base64 = "0.13.0"
 bytesize = "1.1.0"
 calamine = "0.18.0"
@@ -56,6 +57,7 @@ mime = "0.3.16"
 notify = "4.0.17"
 num = { version = "0.4.0", optional = true }
 pathdiff = "0.2.1"
+powierza-coefficient = "1.0"
 quick-xml = "0.22"
 rand = "0.8"
 rayon = "1.5.1"
@@ -68,7 +70,8 @@ serde_ini = "0.2.0"
 serde_urlencoded = "0.7.0"
 serde_yaml = "0.8.16"
 sha2 = "0.10.0"
-shadow-rs = "0.11.0"
+# Disable default features b/c the default features build Git (very slow to compile)
+shadow-rs = { version = "0.11.0", default-features = false }
 strip-ansi-escapes = "0.1.1"
 sysinfo = "0.23.5"
 terminal_size = "0.1.17"
@@ -79,27 +82,29 @@ unicode-segmentation = "1.8.0"
 url = "2.2.1"
 uuid = { version = "0.8.2", features = ["v4"] }
 which = { version = "4.2.2", optional = true }
-reedline = { version = "0.5.0", features = ["bashisms"]}
+reedline = { version = "0.6.0", features = ["bashisms"]}
 wax = { version =  "0.4.0", features = ["diagnostics"] }
 rusqlite = { version = "0.27.0", features = ["bundled"], optional = true }
 sqlparser = { version = "0.16.0", features = ["serde"], optional = true }
 
 [target.'cfg(unix)'.dependencies]
-umask = "1.0.0"
+umask = "2.0.0"
 users = "0.11.0"
 
 [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies.trash]
-version = "2.0.2"
+version = "2.1.3"
 optional = true
 
 [dependencies.polars]
-version = "0.20.0"
+version = "0.21.1"
+# path = "../../../../polars/polars"
 optional = true
 features = [
-	"default", "parquet", "json", "serde", "object",
+	"default", "to_dummies", "parquet", "json", "serde", "serde-lazy",
-	"checked_arithmetic", "strings", "cum_agg", "is_in",
+	"object", "checked_arithmetic", "strings", "cum_agg", "is_in",
 	"rolling_window", "strings", "rows", "random",
-    "dtype-datetime"
+  "dtype-datetime", "dtype-struct", "lazy", "cross_join",
+	"dynamic_groupby"
 ]
 
 [features]
@@ -110,7 +115,7 @@ dataframe = ["polars", "num"]
 database = ["sqlparser", "rusqlite"]
 
 [build-dependencies]
-shadow-rs = "0.11.0"
+shadow-rs = { version = "0.11.0", default-features = false }
 
 [dev-dependencies]
 hamcrest2 = "0.3.0"

crates/nu-command/build.rs

@@ -1,3 +1,18 @@
+use std::process::Command;
+
 fn main() -> shadow_rs::SdResult<()> {
+    // Look up the current Git commit ourselves instead of relying on shadow_rs,
+    // because shadow_rs does it in a really slow-to-compile way (it builds libgit2)
+    let hash = get_git_hash().expect("failed to get latest git commit hash");
+    println!("cargo:rustc-env=NU_COMMIT_HASH={}", hash);
+
     shadow_rs::new()
 }
+
+fn get_git_hash() -> Result<String, std::io::Error> {
+    let out = Command::new("git").args(["rev-parse", "HEAD"]).output()?;
+    Ok(String::from_utf8(out.stdout)
+        .expect("could not convert stdout to string")
+        .trim()
+        .to_string())
+}

crates/nu-command/src/charting/hashable_value.rs

@@ -0,0 +1,317 @@
+use chrono::{DateTime, FixedOffset};
+use nu_protocol::{ShellError, Span, Value};
+use std::hash::{Hash, Hasher};
+
+/// A subset of [Value](crate::Value), which is hashable.
+/// And it means that we can put the value into something like [HashMap](std::collections::HashMap) or [HashSet](std::collections::HashSet)
+/// for further usage like value statistics.
+///
+/// For now the main way to crate a [HashableValue] is using [from_value](HashableValue::from_value)
+///
+/// Please note that although each variant contains `span` field, but during hashing, this field will not be concerned.
+/// Which means that the following will be true:
+/// ```text
+/// assert_eq!(HashableValue::Bool {val: true, span: Span{start: 0, end: 1}}, HashableValue::Bool {val: true, span: Span{start: 90, end: 1000}})
+/// ```
+#[derive(Eq, Debug)]
+pub enum HashableValue {
+    Bool {
+        val: bool,
+        span: Span,
+    },
+    Int {
+        val: i64,
+        span: Span,
+    },
+    Float {
+        val: [u8; 8], // because f64 is not hashable, we save it as [u8;8] array to make it hashable.
+        span: Span,
+    },
+    Filesize {
+        val: i64,
+        span: Span,
+    },
+    Duration {
+        val: i64,
+        span: Span,
+    },
+    Date {
+        val: DateTime<FixedOffset>,
+        span: Span,
+    },
+    String {
+        val: String,
+        span: Span,
+    },
+    Binary {
+        val: Vec<u8>,
+        span: Span,
+    },
+}
+
+impl Default for HashableValue {
+    fn default() -> Self {
+        HashableValue::Bool {
+            val: false,
+            span: Span { start: 0, end: 0 },
+        }
+    }
+}
+
+impl HashableValue {
+    /// Try to convert from `value` to self
+    ///
+    /// A `span` is required because when there is an error in value, it may not contain `span` field.
+    ///
+    /// If the given value is not hashable(mainly because of it is structured data), an error will returned.
+    pub fn from_value(value: Value, span: Span) -> Result<Self, ShellError> {
+        match value {
+            Value::Bool { val, span } => Ok(HashableValue::Bool { val, span }),
+            Value::Int { val, span } => Ok(HashableValue::Int { val, span }),
+            Value::Filesize { val, span } => Ok(HashableValue::Filesize { val, span }),
+            Value::Duration { val, span } => Ok(HashableValue::Duration { val, span }),
+            Value::Date { val, span } => Ok(HashableValue::Date { val, span }),
+            Value::Float { val, span } => Ok(HashableValue::Float {
+                val: val.to_ne_bytes(),
+                span,
+            }),
+            Value::String { val, span } => Ok(HashableValue::String { val, span }),
+            Value::Binary { val, span } => Ok(HashableValue::Binary { val, span }),
+
+            _ => {
+                let input_span = value.span().unwrap_or(span);
+                Err(ShellError::UnsupportedInput(
+                    format!("input value {value:?} is not hashable"),
+                    input_span,
+                ))
+            }
+        }
+    }
+
+    /// Convert from self to nu's core data type `Value`.
+    pub fn into_value(self) -> Value {
+        match self {
+            HashableValue::Bool { val, span } => Value::Bool { val, span },
+            HashableValue::Int { val, span } => Value::Int { val, span },
+            HashableValue::Filesize { val, span } => Value::Filesize { val, span },
+            HashableValue::Duration { val, span } => Value::Duration { val, span },
+            HashableValue::Date { val, span } => Value::Date { val, span },
+            HashableValue::Float { val, span } => Value::Float {
+                val: f64::from_ne_bytes(val),
+                span,
+            },
+            HashableValue::String { val, span } => Value::String { val, span },
+            HashableValue::Binary { val, span } => Value::Binary { val, span },
+        }
+    }
+}
+
+impl Hash for HashableValue {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        match self {
+            HashableValue::Bool { val, .. } => val.hash(state),
+            HashableValue::Int { val, .. } => val.hash(state),
+            HashableValue::Filesize { val, .. } => val.hash(state),
+            HashableValue::Duration { val, .. } => val.hash(state),
+            HashableValue::Date { val, .. } => val.hash(state),
+            HashableValue::Float { val, .. } => val.hash(state),
+            HashableValue::String { val, .. } => val.hash(state),
+            HashableValue::Binary { val, .. } => val.hash(state),
+        }
+    }
+}
+
+impl PartialEq for HashableValue {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (HashableValue::Bool { val: lhs, .. }, HashableValue::Bool { val: rhs, .. }) => {
+                lhs == rhs
+            }
+            (HashableValue::Int { val: lhs, .. }, HashableValue::Int { val: rhs, .. }) => {
+                lhs == rhs
+            }
+            (
+                HashableValue::Filesize { val: lhs, .. },
+                HashableValue::Filesize { val: rhs, .. },
+            ) => lhs == rhs,
+            (
+                HashableValue::Duration { val: lhs, .. },
+                HashableValue::Duration { val: rhs, .. },
+            ) => lhs == rhs,
+            (HashableValue::Date { val: lhs, .. }, HashableValue::Date { val: rhs, .. }) => {
+                lhs == rhs
+            }
+            (HashableValue::Float { val: lhs, .. }, HashableValue::Float { val: rhs, .. }) => {
+                lhs == rhs
+            }
+            (HashableValue::String { val: lhs, .. }, HashableValue::String { val: rhs, .. }) => {
+                lhs == rhs
+            }
+            (HashableValue::Binary { val: lhs, .. }, HashableValue::Binary { val: rhs, .. }) => {
+                lhs == rhs
+            }
+            _ => false,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use nu_protocol::ast::{CellPath, PathMember};
+    use std::collections::{HashMap, HashSet};
+
+    #[test]
+    fn from_value() {
+        let span = Span::test_data();
+        let values = vec![
+            (
+                Value::Bool { val: true, span },
+                HashableValue::Bool { val: true, span },
+            ),
+            (
+                Value::Int { val: 1, span },
+                HashableValue::Int { val: 1, span },
+            ),
+            (
+                Value::Filesize { val: 1, span },
+                HashableValue::Filesize { val: 1, span },
+            ),
+            (
+                Value::Duration { val: 1, span },
+                HashableValue::Duration { val: 1, span },
+            ),
+            (
+                Value::Date {
+                    val: DateTime::<FixedOffset>::parse_from_rfc2822(
+                        "Wed, 18 Feb 2015 23:16:09 GMT",
+                    )
+                    .unwrap(),
+                    span,
+                },
+                HashableValue::Date {
+                    val: DateTime::<FixedOffset>::parse_from_rfc2822(
+                        "Wed, 18 Feb 2015 23:16:09 GMT",
+                    )
+                    .unwrap(),
+                    span,
+                },
+            ),
+            (
+                Value::String {
+                    val: "1".to_string(),
+                    span,
+                },
+                HashableValue::String {
+                    val: "1".to_string(),
+                    span,
+                },
+            ),
+            (
+                Value::Binary { val: vec![1], span },
+                HashableValue::Binary { val: vec![1], span },
+            ),
+        ];
+        for (val, expect_hashable_val) in values.into_iter() {
+            assert_eq!(
+                HashableValue::from_value(val, Span { start: 0, end: 0 }).unwrap(),
+                expect_hashable_val
+            );
+        }
+    }
+
+    #[test]
+    fn from_unhashable_value() {
+        let span = Span::test_data();
+        let values = [
+            Value::List {
+                vals: vec![Value::Bool { val: true, span }],
+                span,
+            },
+            Value::Block {
+                val: 0,
+                captures: HashMap::new(),
+                span,
+            },
+            Value::Nothing { span },
+            Value::Error {
+                error: ShellError::DidYouMean("what?".to_string(), span),
+            },
+            Value::CellPath {
+                val: CellPath {
+                    members: vec![PathMember::Int { val: 0, span }],
+                },
+                span,
+            },
+        ];
+        for v in values {
+            assert!(HashableValue::from_value(v, Span { start: 0, end: 0 }).is_err())
+        }
+    }
+
+    #[test]
+    fn from_to_tobe_same() {
+        let span = Span::test_data();
+        let values = vec![
+            Value::Bool { val: true, span },
+            Value::Int { val: 1, span },
+            Value::Filesize { val: 1, span },
+            Value::Duration { val: 1, span },
+            Value::String {
+                val: "1".to_string(),
+                span,
+            },
+            Value::Binary { val: vec![1], span },
+        ];
+        for val in values.into_iter() {
+            let expected_val = val.clone();
+            assert_eq!(
+                HashableValue::from_value(val, Span { start: 0, end: 0 })
+                    .unwrap()
+                    .into_value(),
+                expected_val
+            );
+        }
+    }
+
+    #[test]
+    fn hashable_value_eq_without_concern_span() {
+        assert_eq!(
+            HashableValue::Bool {
+                val: true,
+                span: Span { start: 0, end: 1 }
+            },
+            HashableValue::Bool {
+                val: true,
+                span: Span {
+                    start: 90,
+                    end: 1000
+                }
+            }
+        )
+    }
+
+    #[test]
+    fn put_to_hashset() {
+        let span = Span::test_data();
+        let mut set = HashSet::new();
+        set.insert(HashableValue::Bool { val: true, span });
+        assert!(set.contains(&HashableValue::Bool { val: true, span }));
+
+        // hashable value doesn't care about span.
+        let diff_span = Span { start: 1, end: 2 };
+        set.insert(HashableValue::Bool {
+            val: true,
+            span: diff_span,
+        });
+        assert!(set.contains(&HashableValue::Bool { val: true, span }));
+        assert!(set.contains(&HashableValue::Bool {
+            val: true,
+            span: diff_span
+        }));
+        assert_eq!(set.len(), 1);
+
+        set.insert(HashableValue::Int { val: 2, span });
+        assert_eq!(set.len(), 2);
+    }
+}

crates/nu-command/src/charting/histogram.rs

@@ -0,0 +1,256 @@
+use super::hashable_value::HashableValue;
+use nu_engine::CallExt;
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape,
+    Value,
+};
+use std::collections::HashMap;
+use std::iter;
+
+#[derive(Clone)]
+pub struct Histogram;
+
+enum PercentageCalcMethod {
+    Normalize,
+    Relative,
+}
+
+impl Command for Histogram {
+    fn name(&self) -> &str {
+        "histogram"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("histogram")
+            .optional("column-name", SyntaxShape::String, "column name to calc frequency, no need to provide if input is just a list")
+            .optional("frequency-column-name", SyntaxShape::String, "histogram's frequency column, default to be frequency column output")
+            .named("percentage-type", SyntaxShape::String, "percentage calculate method, can be 'normalize' or 'relative', in 'normalize', defaults to be 'normalize'", Some('t'))
+    }
+
+    fn usage(&self) -> &str {
+        "Creates a new table with a histogram based on the column name passed in."
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Get a histogram for the types of files",
+                example: "ls | histogram type",
+                result: None,
+            },
+            Example {
+                description:
+                    "Get a histogram for the types of files, with frequency column named freq",
+                example: "ls | histogram type freq",
+                result: None,
+            },
+            Example {
+                description: "Get a histogram for a list of numbers",
+                example: "echo [1 2 3 1 1 1 2 2 1 1] | histogram",
+                result: None,
+            },
+            Example {
+                description: "Get a histogram for a list of numbers, and percentage is based on the maximum value",
+                example: "echo [1 2 3 1 1 1 2 2 1 1] | histogram --percentage-type relative",
+                result: None,
+            }
+        ]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        // input check.
+        let column_name: Option<Spanned<String>> = call.opt(engine_state, stack, 0)?;
+        let frequency_name_arg = call.opt::<Spanned<String>>(engine_state, stack, 1)?;
+        let frequency_column_name = match frequency_name_arg {
+            Some(inner) => {
+                let span = inner.span;
+                if ["value", "count", "quantile", "percentage"].contains(&inner.item.as_str()) {
+                    return Err(ShellError::UnsupportedInput(
+                        "frequency-column-name can't be 'value', 'count' or 'percentage'"
+                            .to_string(),
+                        span,
+                    ));
+                }
+                inner.item
+            }
+            None => "frequency".to_string(),
+        };
+
+        let calc_method: Option<Spanned<String>> =
+            call.get_flag(engine_state, stack, "percentage-type")?;
+        let calc_method = match calc_method {
+            None => PercentageCalcMethod::Normalize,
+            Some(inner) => match inner.item.as_str() {
+                "normalize" => PercentageCalcMethod::Normalize,
+                "relative" => PercentageCalcMethod::Relative,
+                _ => {
+                    return Err(ShellError::UnsupportedInput(
+                        "calc method can only be 'normalize' or 'relative'".to_string(),
+                        inner.span,
+                    ))
+                }
+            },
+        };
+
+        let span = call.head;
+        let data_as_value = input.into_value(span);
+        // `input` is not a list, here we can return an error.
+        match data_as_value.as_list() {
+            Ok(list_value) => run_histogram(
+                list_value.to_vec(),
+                column_name,
+                frequency_column_name,
+                calc_method,
+                span,
+            ),
+            Err(e) => Err(e),
+        }
+    }
+}
+
+fn run_histogram(
+    values: Vec<Value>,
+    column_name: Option<Spanned<String>>,
+    freq_column: String,
+    calc_method: PercentageCalcMethod,
+    head_span: Span,
+) -> Result<PipelineData, ShellError> {
+    let mut inputs = vec![];
+    // convert from inputs to hashable values.
+    match column_name {
+        None => {
+            // some invalid input scenario needs to handle:
+            // Expect input is a list of hashable value, if one value is not hashable, throw out error.
+            for v in values {
+                let current_span = v.span().unwrap_or(head_span);
+                inputs.push(HashableValue::from_value(v, head_span).map_err(|_| {
+                    ShellError::UnsupportedInput(
+                        "--column-name is not provided, can only support a list of simple value."
+                            .to_string(),
+                        current_span,
+                    )
+                })?);
+            }
+        }
+        Some(ref col) => {
+            // some invalid input scenario needs to handle:
+            // * item in `input` is not a record, just skip it.
+            // * a record doesn't contain specific column, just skip it.
+            // * all records don't contain specific column, throw out error, indicate at least one row should contains specific column.
+            // * a record contain a value which can't be hashed, skip it.
+            let col_name = &col.item;
+            for v in values {
+                match v {
+                    // parse record, and fill valid value to actual input.
+                    Value::Record { cols, vals, .. } => {
+                        for (c, v) in iter::zip(cols, vals) {
+                            if &c == col_name {
+                                if let Ok(v) = HashableValue::from_value(v, head_span) {
+                                    inputs.push(v);
+                                }
+                            }
+                        }
+                    }
+                    _ => continue,
+                }
+            }
+
+            if inputs.is_empty() {
+                return Err(ShellError::UnsupportedInput(
+                    format!("expect input is table, and inputs doesn't contain any value which has {col_name} column"),
+                    head_span,
+                ));
+            }
+        }
+    }
+
+    let value_column_name = column_name
+        .map(|x| x.item)
+        .unwrap_or_else(|| "value".to_string());
+    Ok(histogram_impl(
+        inputs,
+        &value_column_name,
+        calc_method,
+        &freq_column,
+        head_span,
+    ))
+}
+
+fn histogram_impl(
+    inputs: Vec<HashableValue>,
+    value_column_name: &str,
+    calc_method: PercentageCalcMethod,
+    freq_column: &str,
+    span: Span,
+) -> PipelineData {
+    // here we can make sure that inputs is not empty, and every elements
+    // is a simple val and ok to make count.
+    let mut counter = HashMap::new();
+    let mut max_cnt = 0;
+    let total_cnt = inputs.len();
+    for i in inputs {
+        let new_cnt = *counter.get(&i).unwrap_or(&0) + 1;
+        counter.insert(i, new_cnt);
+        if new_cnt > max_cnt {
+            max_cnt = new_cnt;
+        }
+    }
+
+    let mut result = vec![];
+    let result_cols = vec![
+        value_column_name.to_string(),
+        "count".to_string(),
+        "quantile".to_string(),
+        "percentage".to_string(),
+        freq_column.to_string(),
+    ];
+    const MAX_FREQ_COUNT: f64 = 100.0;
+    for (val, count) in counter.into_iter() {
+        let quantile = match calc_method {
+            PercentageCalcMethod::Normalize => (count as f64 / total_cnt as f64),
+            PercentageCalcMethod::Relative => (count as f64 / max_cnt as f64),
+        };
+
+        let percentage = format!("{:.2}%", quantile * 100_f64);
+        let freq = "*".repeat((MAX_FREQ_COUNT * quantile).floor() as usize);
+
+        result.push(Value::Record {
+            cols: result_cols.clone(),
+            vals: vec![
+                val.into_value(),
+                Value::Int { val: count, span },
+                Value::Float {
+                    val: quantile,
+                    span,
+                },
+                Value::String {
+                    val: percentage,
+                    span,
+                },
+                Value::String { val: freq, span },
+            ],
+            span,
+        });
+    }
+    Value::List { vals: result, span }.into_pipeline_data()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        use crate::test_examples;
+
+        test_examples(Histogram)
+    }
+}

crates/nu-command/src/charting/mod.rs

@@ -0,0 +1,4 @@
+mod hashable_value;
+mod histogram;
+
+pub use histogram::Histogram;

crates/nu-command/src/conversions/into/datetime.rs

@@ -24,13 +24,13 @@ enum Zone {
     Local,
     East(u8),
     West(u8),
-    Error, // we want the nullshell to cast it instead of rust
+    Error, // we want Nushell to cast it instead of Rust
 }
 
 impl Zone {
     fn new(i: i64) -> Self {
         if i.abs() <= 12 {
-            // guanranteed here
+            // guaranteed here
             if i >= 0 {
                 Self::East(i as u8) // won't go out of range
             } else {
@@ -59,29 +59,29 @@ impl Command for SubCommand {
 
     fn signature(&self) -> Signature {
         Signature::build("into datetime")
-            .switch(
-                "list",
-                "lists strftime cheatsheet",
-                Some('l'),
-                )
             .named(
                 "timezone",
                 SyntaxShape::String,
-                "Specify timezone if the input is timestamp, like 'UTC/u' or 'LOCAL/l'",
+                "Specify timezone if the input is a Unix timestamp. Valid options: 'UTC' ('u') or 'LOCAL' ('l')",
                 Some('z'),
             )
             .named(
                 "offset",
                 SyntaxShape::Int,
-                "Specify timezone by offset if the input is timestamp, like '+8', '-4', prior than timezone",
+                "Specify timezone by offset from UTC if the input is a Unix timestamp, like '+8', '-4'",
                 Some('o'),
             )
             .named(
                 "format",
                 SyntaxShape::String,
-                "Specify date and time formatting",
+                "Specify an expected format for parsing strings to datetimes. Use --list to see all possible options",
                 Some('f'),
             )
+            .switch(
+                "list",
+                "Show all possible variables for use with the --format flag",
+                Some('l'),
+                )
             .rest(
             "rest",
                 SyntaxShape::CellPath,
@@ -112,28 +112,40 @@ impl Command for SubCommand {
         vec![
             Example {
                 description: "Convert to datetime",
-                example: "'16.11.1984 8:00 am +0000' | into datetime",
+                example: "'27.02.2021 1:55 pm +0000' | into datetime",
-                result: None,
+                result: Some(Value::Date {
+                    val: Utc.timestamp(1614434100, 0).into(),
+                    span: Span::test_data(),
+                }),
             },
             Example {
                 description: "Convert to datetime",
-                example: "'2020-08-04T16:39:18+00:00' | into datetime",
+                example: "'2021-02-27T13:55:40+00:00' | into datetime",
-                result: None,
+                result: Some(Value::Date {
+                    val: Utc.timestamp(1614434140, 0).into(),
+                    span: Span::test_data(),
+                }),
             },
             Example {
                 description: "Convert to datetime using a custom format",
-                example: "'20200904_163918+0000' | into datetime -f '%Y%m%d_%H%M%S%z'",
+                example: "'20210227_135540+0000' | into datetime -f '%Y%m%d_%H%M%S%z'",
-                result: None,
+                result: Some(Value::Date {
+                    val: Utc.timestamp(1614434140, 0).into(),
+                    span: Span::test_data(),
+                }),
             },
             Example {
-                description: "Convert timestamp (no larger than 8e+12) to datetime using a specified timezone",
+                description: "Convert timestamp (no larger than 8e+12) to a UTC datetime",
-                example: "'1614434140' | into datetime -z 'UTC'",
+                example: "1614434140 | into datetime",
-                result: None,
+                result: Some(Value::Date {
+                    val: Utc.timestamp(1614434140, 0).into(),
+                    span: Span::test_data(),
+                }),
             },
             Example {
                 description:
                     "Convert timestamp (no larger than 8e+12) to datetime using a specified timezone offset (between -12 and 12)",
-                example: "'1614434140' | into datetime -o +9",
+                example: "1614434140 | into datetime -o +9",
                 result: None,
             },
         ]
@@ -209,58 +221,78 @@ fn action(
     dateformat: &Option<DatetimeFormat>,
     head: Span,
 ) -> Value {
-    match input {
+    // Check to see if input looks like a Unix timestamp (i.e. can it be parsed to an int?)
-        Value::String { val: s, span } => {
+    let timestamp = match input {
-            let ts = s.parse::<i64>();
+        Value::Int { val, .. } => Ok(*val),
-            // if timezone if specified, first check if the input is a timestamp.
+        Value::String { val, .. } => val.parse::<i64>(),
-            if let Some(tz) = timezone {
+        other => {
-                const TIMESTAMP_BOUND: i64 = 8.2e+12 as i64;
+            return Value::Error {
-                // Since the timestamp method of chrono itself don't throw an error (it just panicked)
+                error: ShellError::UnsupportedInput(
-                // We have to manually guard it.
+                    format!("Expected string or int, got {} instead", other.get_type()),
-                if let Ok(t) = ts {
+                    head,
-                    if t.abs() > TIMESTAMP_BOUND {
+                ),
-                        return Value::Error{error: ShellError::UnsupportedInput(
-                            "Given timestamp is out of range, it should between -8e+12 and 8e+12".to_string(),
-                            head,
-                        )};
-                    }
-                    const HOUR: i32 = 3600;
-                    let stampout = match tz.item {
-                        Zone::Utc => Value::Date {
-                            val: Utc.timestamp(t, 0).into(),
-                            span: head,
-                        },
-                        Zone::Local => Value::Date {
-                            val: Local.timestamp(t, 0).into(),
-                            span: head,
-                        },
-                        Zone::East(i) => {
-                            let eastoffset = FixedOffset::east((i as i32) * HOUR);
-                            Value::Date {
-                                val: eastoffset.timestamp(t, 0),
-                                span: head,
-                            }
-                        }
-                        Zone::West(i) => {
-                            let westoffset = FixedOffset::west((i as i32) * HOUR);
-                            Value::Date {
-                                val: westoffset.timestamp(t, 0),
-                                span: head,
-                            }
-                        }
-                        Zone::Error => Value::Error {
-                            error: ShellError::UnsupportedInput(
-                                "Cannot convert given timezone or offset to timestamp".to_string(),
-                                tz.span,
-                            ),
-                        },
-                    };
-                    return stampout;
-                }
             };
-            // if it's not, continue and default to the system's local timezone.
+        }
-            let out = match dateformat {
+    };
-                Some(dt) => match DateTime::parse_from_str(s, &dt.0) {
+
+    if let Ok(ts) = timestamp {
+        const TIMESTAMP_BOUND: i64 = 8.2e+12 as i64;
+        const HOUR: i32 = 3600;
+
+        if ts.abs() > TIMESTAMP_BOUND {
+            return Value::Error {
+                error: ShellError::UnsupportedInput(
+                    "Given timestamp is out of range, it should between -8e+12 and 8e+12"
+                        .to_string(),
+                    head,
+                ),
+            };
+        }
+
+        return match timezone {
+            // default to UTC
+            None => Value::Date {
+                val: Utc.timestamp(ts, 0).into(),
+                span: head,
+            },
+            Some(Spanned { item, span }) => match item {
+                Zone::Utc => Value::Date {
+                    val: Utc.timestamp(ts, 0).into(),
+                    span: head,
+                },
+                Zone::Local => Value::Date {
+                    val: Local.timestamp(ts, 0).into(),
+                    span: head,
+                },
+                Zone::East(i) => {
+                    let eastoffset = FixedOffset::east((*i as i32) * HOUR);
+                    Value::Date {
+                        val: eastoffset.timestamp(ts, 0),
+                        span: head,
+                    }
+                }
+                Zone::West(i) => {
+                    let westoffset = FixedOffset::west((*i as i32) * HOUR);
+                    Value::Date {
+                        val: westoffset.timestamp(ts, 0),
+                        span: head,
+                    }
+                }
+                Zone::Error => Value::Error {
+                    error: ShellError::UnsupportedInput(
+                        "Cannot convert given timezone or offset to timestamp".to_string(),
+                        *span,
+                    ),
+                },
+            },
+        };
+    }
+
+    // If input is not a timestamp, try parsing it as a string
+    match input {
+        Value::String { val, span } => {
+            match dateformat {
+                Some(dt) => match DateTime::parse_from_str(val, &dt.0) {
                     Ok(d) => Value::Date { val: d, span: head },
                     Err(reason) => {
                         return Value::Error {
@@ -276,23 +308,21 @@ fn action(
                 // Tries to automatically parse the date
                 // (i.e. without a format string)
                 // and assumes the system's local timezone if none is specified
-                None => match parse_date_from_string(s, *span) {
+                None => match parse_date_from_string(val, *span) {
                     Ok(date) => Value::Date {
                         val: date,
                         span: *span,
                     },
                     Err(err) => err,
                 },
-            };
-
-            out
-        }
-        other => {
-            let got = format!("Expected string, got {} instead", other.get_type());
-            Value::Error {
-                error: ShellError::UnsupportedInput(got, head),
             }
         }
+        other => Value::Error {
+            error: ShellError::UnsupportedInput(
+                format!("Expected string, got {} instead", other.get_type()),
+                head,
+            ),
+        },
     }
 }
 
@@ -351,6 +381,23 @@ mod tests {
         assert_eq!(actual, expected)
     }
 
+    #[test]
+    fn takes_timestamp_offset_as_int() {
+        let date_int = Value::test_int(1614434140);
+        let timezone_option = Some(Spanned {
+            item: Zone::East(8),
+            span: Span::test_data(),
+        });
+        let actual = action(&date_int, &timezone_option, &None, Span::test_data());
+        let expected = Value::Date {
+            val: DateTime::parse_from_str("2021-02-27 21:55:40 +08:00", "%Y-%m-%d %H:%M:%S %z")
+                .unwrap(),
+            span: Span::test_data(),
+        };
+
+        assert_eq!(actual, expected)
+    }
+
     #[test]
     fn takes_timestamp() {
         let date_str = Value::test_string("1614434140");
@@ -367,6 +414,20 @@ mod tests {
         assert_eq!(actual, expected)
     }
 
+    #[test]
+    fn takes_timestamp_without_timezone() {
+        let date_str = Value::test_string("1614434140");
+        let timezone_option = None;
+        let actual = action(&date_str, &timezone_option, &None, Span::test_data());
+
+        let expected = Value::Date {
+            val: Utc.timestamp(1614434140, 0).into(),
+            span: Span::test_data(),
+        };
+
+        assert_eq!(actual, expected)
+    }
+
     #[test]
     fn takes_invalid_timestamp() {
         let date_str = Value::test_string("10440970000000");

crates/nu-command/src/conversions/into/filesize.rs

@@ -133,6 +133,10 @@ pub fn action(input: &Value, span: Span) -> Value {
                 },
                 Err(error) => Value::Error { error },
             },
+            Value::Nothing { .. } => Value::Filesize {
+                val: 0,
+                span: value_span,
+            },
             _ => Value::Error {
                 error: ShellError::UnsupportedInput(
                     "'into filesize' for unsupported type".into(),

crates/nu-command/src/conversions/into/int.rs

@@ -85,6 +85,11 @@ impl Command for SubCommand {
                     span: Span::test_data(),
                 }),
             },
+            Example {
+                description: "Convert date to integer (Unix timestamp)",
+                example: "2022-02-02 | into int",
+                result: Some(Value::test_int(1643760000)),
+            },
             Example {
                 description: "Convert to integer from binary",
                 example: "'1101' | into int -r 2",
@@ -181,6 +186,10 @@ pub fn action(input: &Value, span: Span, radix: u32) -> Value {
                 Value::Int { val: 0, span }
             }
         }
+        Value::Date { val, .. } => Value::Int {
+            val: val.timestamp(),
+            span,
+        },
         _ => Value::Error {
             error: ShellError::UnsupportedInput("'into int' for unsupported type".into(), span),
         },

crates/nu-command/src/conversions/into/string.rs

@@ -248,7 +248,7 @@ pub fn action(
             span,
         },
         Value::Nothing { .. } => Value::String {
-            val: "nothing".to_string(),
+            val: "".to_string(),
             span,
         },
         Value::Record {

crates/nu-command/src/core_commands/error_make.rs

@@ -36,7 +36,7 @@ impl Command for ErrorMake {
         let arg: Option<Value> = call.opt(engine_state, stack, 0)?;
 
         if let Some(arg) = arg {
-            Ok(make_error(&arg)
+            Ok(make_error(&arg, span)
                 .map(|err| Value::Error { error: err })
                 .unwrap_or_else(|| Value::Error {
                     error: ShellError::GenericError(
@@ -51,7 +51,7 @@ impl Command for ErrorMake {
         } else {
             input.map(
                 move |value| {
-                    make_error(&value)
+                    make_error(&value, span)
                         .map(|err| Value::Error { error: err })
                         .unwrap_or_else(|| Value::Error {
                             error: ShellError::GenericError(
@@ -89,7 +89,7 @@ impl Command for ErrorMake {
     }
 }
 
-fn make_error(value: &Value) -> Option<ShellError> {
+fn make_error(value: &Value, throw_span: Span) -> Option<ShellError> {
     if let Value::Record { .. } = &value {
         let msg = value.get_data_by_key("msg");
         let label = value.get_data_by_key("label");
@@ -117,13 +117,26 @@ fn make_error(value: &Value) -> Option<ShellError> {
                         None,
                         Vec::new(),
                     )),
+                    (
+                        None,
+                        None,
+                        Some(Value::String {
+                            val: label_text, ..
+                        }),
+                    ) => Some(ShellError::GenericError(
+                        message,
+                        label_text,
+                        Some(throw_span),
+                        None,
+                        Vec::new(),
+                    )),
                     _ => None,
                 }
             }
             (Some(Value::String { val: message, .. }), None) => Some(ShellError::GenericError(
                 message,
-                "".to_string(),
+                "originates from here".to_string(),
-                None,
+                Some(throw_span),
                 None,
                 Vec::new(),
             )),

crates/nu-command/src/core_commands/hide.rs

@@ -63,23 +63,23 @@ https://www.nushell.sh/book/thinking_in_nushell.html#parsing-and-evaluation-are-
             return Err(ShellError::NonUtf8(import_pattern.head.span));
         };
 
-        if let Some(overlay_id) = engine_state.find_overlay(&import_pattern.head.name) {
+        if let Some(module_id) = engine_state.find_module(&import_pattern.head.name, &[]) {
             // The first word is a module
-            let overlay = engine_state.get_overlay(overlay_id);
+            let module = engine_state.get_module(module_id);
 
             let env_vars_to_hide = if import_pattern.members.is_empty() {
-                overlay.env_vars_with_head(&import_pattern.head.name)
+                module.env_vars_with_head(&import_pattern.head.name)
             } else {
                 match &import_pattern.members[0] {
-                    ImportPatternMember::Glob { .. } => overlay.env_vars(),
+                    ImportPatternMember::Glob { .. } => module.env_vars(),
                     ImportPatternMember::Name { name, span } => {
                         let mut output = vec![];
 
                         if let Some((name, id)) =
-                            overlay.env_var_with_head(name, &import_pattern.head.name)
+                            module.env_var_with_head(name, &import_pattern.head.name)
                         {
                             output.push((name, id));
-                        } else if !(overlay.has_alias(name) || overlay.has_decl(name)) {
+                        } else if !(module.has_alias(name) || module.has_decl(name)) {
                             return Err(ShellError::EnvVarNotFoundAtRuntime(
                                 String::from_utf8_lossy(name).into(),
                                 *span,
@@ -93,10 +93,10 @@ https://www.nushell.sh/book/thinking_in_nushell.html#parsing-and-evaluation-are-
 
                         for (name, span) in names {
                             if let Some((name, id)) =
-                                overlay.env_var_with_head(name, &import_pattern.head.name)
+                                module.env_var_with_head(name, &import_pattern.head.name)
                             {
                                 output.push((name, id));
-                            } else if !(overlay.has_alias(name) || overlay.has_decl(name)) {
+                            } else if !(module.has_alias(name) || module.has_decl(name)) {
                                 return Err(ShellError::EnvVarNotFoundAtRuntime(
                                     String::from_utf8_lossy(name).into(),
                                     *span,

crates/nu-command/src/core_commands/mod.rs

@@ -22,6 +22,7 @@ mod ignore;
 mod let_;
 mod metadata;
 mod module;
+pub(crate) mod overlay;
 mod source;
 mod tutor;
 mod use_;
@@ -51,6 +52,7 @@ pub use ignore::Ignore;
 pub use let_::Let;
 pub use metadata::Metadata;
 pub use module::Module;
+pub use overlay::*;
 pub use source::Source;
 pub use tutor::Tutor;
 pub use use_::Use;

crates/nu-command/src/core_commands/overlay/add.rs

@@ -0,0 +1,148 @@
+use nu_engine::{eval_block, CallExt};
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape};
+
+use std::path::Path;
+
+#[derive(Clone)]
+pub struct OverlayAdd;
+
+impl Command for OverlayAdd {
+    fn name(&self) -> &str {
+        "overlay add"
+    }
+
+    fn usage(&self) -> &str {
+        "Add definitions from a module as an overlay"
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("overlay add")
+            .required(
+                "name",
+                SyntaxShape::String,
+                "Module name to create overlay for",
+            )
+            // TODO:
+            // .switch(
+            //     "prefix",
+            //     "Prepend module name to the imported symbols",
+            //     Some('p'),
+            // )
+            .category(Category::Core)
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check
+https://www.nushell.sh/book/thinking_in_nushell.html#parsing-and-evaluation-are-different-stages"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let name_arg: Spanned<String> = call.req(engine_state, stack, 0)?;
+
+        let maybe_overlay_name = if engine_state
+            .find_overlay(name_arg.item.as_bytes())
+            .is_some()
+        {
+            Some(name_arg.item.clone())
+        } else if let Some(os_str) = Path::new(&name_arg.item).file_stem() {
+            os_str.to_str().map(|name| name.to_string())
+        } else {
+            None
+        };
+
+        if let Some(overlay_name) = maybe_overlay_name {
+            if let Some(overlay_id) = engine_state.find_overlay(overlay_name.as_bytes()) {
+                let old_module_id = engine_state.get_overlay(overlay_id).origin;
+
+                stack.add_overlay(overlay_name.clone());
+
+                if let Some(new_module_id) = engine_state.find_module(overlay_name.as_bytes(), &[])
+                {
+                    if !stack.has_env_overlay(&overlay_name, engine_state)
+                        || (old_module_id != new_module_id)
+                    {
+                        // Add environment variables only if:
+                        // a) adding a new overlay
+                        // b) refreshing an active overlay (the origin module changed)
+                        let module = engine_state.get_module(new_module_id);
+
+                        for (name, block_id) in module.env_vars() {
+                            let name = if let Ok(s) = String::from_utf8(name.clone()) {
+                                s
+                            } else {
+                                return Err(ShellError::NonUtf8(call.head));
+                            };
+
+                            let block = engine_state.get_block(block_id);
+
+                            let val = eval_block(
+                                engine_state,
+                                stack,
+                                block,
+                                PipelineData::new(call.head),
+                                false,
+                                true,
+                            )?
+                            .into_value(call.head);
+
+                            stack.add_env_var(name, val);
+                        }
+                    }
+                }
+            } else {
+                return Err(ShellError::OverlayNotFoundAtRuntime(
+                    name_arg.item,
+                    name_arg.span,
+                ));
+            }
+        } else {
+            return Err(ShellError::OverlayNotFoundAtRuntime(
+                name_arg.item,
+                name_arg.span,
+            ));
+        }
+
+        Ok(PipelineData::new(call.head))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Create an overlay from a module",
+                example: r#"module spam { export def foo [] { "foo" } }
+    overlay add spam"#,
+                result: None,
+            },
+            Example {
+                description: "Create an overlay from a file",
+                example: r#"echo 'export env FOO { "foo" }' | save spam.nu
+    overlay add spam.nu"#,
+                result: None,
+            },
+        ]
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        use crate::test_examples;
+
+        test_examples(OverlayAdd {})
+    }
+}

crates/nu-command/src/core_commands/overlay/command.rs

@@ -0,0 +1,58 @@
+use nu_engine::get_full_help;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, IntoPipelineData, PipelineData, Signature, Value,
+};
+
+#[derive(Clone)]
+pub struct Overlay;
+
+impl Command for Overlay {
+    fn name(&self) -> &str {
+        "overlay"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build("overlay").category(Category::Core)
+    }
+
+    fn usage(&self) -> &str {
+        "Commands for manipulating overlays."
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check
+https://www.nushell.sh/book/thinking_in_nushell.html#parsing-and-evaluation-are-different-stages"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
+        Ok(Value::String {
+            val: get_full_help(&Overlay.signature(), &[], engine_state, stack),
+            span: call.head,
+        }
+        .into_pipeline_data())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_examples() {
+        use crate::test_examples;
+
+        test_examples(Overlay {})
+    }
+}

crates/nu-command/src/core_commands/overlay/list.rs

@@ -0,0 +1,85 @@
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, Value,
+};
+
+use log::trace;
+
+#[derive(Clone)]
+pub struct OverlayList;
+
+impl Command for OverlayList {
+    fn name(&self) -> &str {
+        "overlay list"
+    }
+
+    fn usage(&self) -> &str {
+        "List all active overlays"
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("overlay list").category(Category::Core)
+    }
+
+    fn extra_usage(&self) -> &str {
+        "The overlays are listed in the order they were activated."
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let active_overlays_parser: Vec<Value> = engine_state
+            .active_overlay_names(&[])
+            .iter()
+            .map(|s| Value::string(String::from_utf8_lossy(s), call.head))
+            .collect();
+
+        let active_overlays_engine: Vec<Value> = stack
+            .active_overlays
+            .iter()
+            .map(|s| Value::string(s, call.head))
+            .collect();
+
+        // Check if the overlays in the engine match the overlays in the parser
+        if (active_overlays_parser.len() != active_overlays_engine.len())
+            || active_overlays_parser
+                .iter()
+                .zip(active_overlays_engine.iter())
+                .any(|(op, oe)| op != oe)
+        {
+            trace!("parser overlays: {:?}", active_overlays_parser);
+            trace!("engine overlays: {:?}", active_overlays_engine);
+
+            return Err(ShellError::NushellFailedSpannedHelp(
+                "Overlay mismatch".into(),
+                "Active overlays do not match between the engine and the parser.".into(),
+                call.head,
+                "Run Nushell with --log-level=trace to see what went wrong.".into(),
+            ));
+        }
+
+        Ok(Value::List {
+            vals: active_overlays_engine,
+            span: call.head,
+        }
+        .into_pipeline_data())
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Get the last activated overlay",
+            example: r#"module spam { export def foo [] { "foo" } }
+    overlay add spam
+    overlay list | last"#,
+            result: Some(Value::String {
+                val: "spam".to_string(),
+                span: Span::test_data(),
+            }),
+        }]
+    }
+}

crates/nu-command/src/core_commands/overlay/mod.rs

@@ -0,0 +1,9 @@
+mod add;
+mod command;
+mod list;
+mod remove;
+
+pub use add::OverlayAdd;
+pub use command::Overlay;
+pub use list::OverlayList;
+pub use remove::OverlayRemove;

crates/nu-command/src/core_commands/overlay/remove.rs

@@ -0,0 +1,117 @@
+use nu_engine::CallExt;
+use nu_protocol::ast::Call;
+use nu_protocol::engine::{Command, EngineState, Stack};
+use nu_protocol::{
+    Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape, Value,
+};
+
+#[derive(Clone)]
+pub struct OverlayRemove;
+
+impl Command for OverlayRemove {
+    fn name(&self) -> &str {
+        "overlay remove"
+    }
+
+    fn usage(&self) -> &str {
+        "Remove an active overlay"
+    }
+
+    fn signature(&self) -> nu_protocol::Signature {
+        Signature::build("overlay remove")
+            .optional("name", SyntaxShape::String, "Overlay to remove")
+            .switch(
+                "keep-custom",
+                "Keep newly added symbols within the next activated overlay",
+                Some('k'),
+            )
+            .category(Category::Core)
+    }
+
+    fn extra_usage(&self) -> &str {
+        r#"This command is a parser keyword. For details, check
+https://www.nushell.sh/book/thinking_in_nushell.html#parsing-and-evaluation-are-different-stages"#
+    }
+
+    fn is_parser_keyword(&self) -> bool {
+        true
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<nu_protocol::PipelineData, nu_protocol::ShellError> {
+        let overlay_name: Spanned<String> = if let Some(name) = call.opt(engine_state, stack, 0)? {
+            name
+        } else {
+            Spanned {
+                item: stack.last_overlay_name()?,
+                span: call.head,
+            }
+        };
+
+        if !stack.is_overlay_active(&overlay_name.item) {
+            return Err(ShellError::OverlayNotFoundAtRuntime(
+                overlay_name.item,
+                overlay_name.span,
+            ));
+        }
+
+        if call.has_flag("keep-custom") {
+            if let Some(overlay_id) = engine_state.find_overlay(overlay_name.item.as_bytes()) {
+                let overlay_frame = engine_state.get_overlay(overlay_id);
+                let origin_module = engine_state.get_module(overlay_frame.origin);
+
+                let env_vars_to_keep: Vec<(String, Value)> = stack
+                    .get_overlay_env_vars(engine_state, &overlay_name.item)
+                    .into_iter()
+                    .filter(|(name, _)| !origin_module.has_env_var(name.as_bytes()))
+                    .collect();
+
+                stack.remove_overlay(&overlay_name.item);
+
+                for (name, val) in env_vars_to_keep {
+                    stack.add_env_var(name, val);
+                }
+            } else {
+                return Err(ShellError::OverlayNotFoundAtRuntime(
+                    overlay_name.item,
+                    overlay_name.span,
+                ));
+            }
+        } else {
+            stack.remove_overlay(&overlay_name.item);
+        }
+
+        Ok(PipelineData::new(call.head))
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![
+            Example {
+                description: "Remove an overlay created from a module",
+                example: r#"module spam { export def foo [] { "foo" } }
+    overlay add spam
+    overlay remove spam"#,
+                result: None,
+            },
+            Example {
+                description: "Remove an overlay created from a file",
+                example: r#"echo 'export alias f = "foo"' | save spam.nu
+    overlay add spam.nu
+    overlay remove spam"#,
+                result: None,
+            },
+            Example {
+                description: "Remove the last activated overlay",
+                example: r#"module spam { export env FOO { "foo" } }
+    overlay add spam
+    overlay remove"#,
+                result: None,
+            },
+        ]
+    }
+}

crates/nu-command/src/core_commands/tutor.rs

@@ -74,6 +74,7 @@ fn tutor(
 
     let search: Option<String> = call.opt(engine_state, stack, 0).unwrap_or(None);
     let find: Option<String> = call.get_flag(engine_state, stack, "find")?;
+    let notes = "You can learn about a topic using `tutor` followed by the name of the topic.\nFor example: `tutor table` to open the table topic.\n\n";
 
     let search_space = [
         (vec!["begin"], begin_tutor()),
@@ -100,7 +101,6 @@ fn tutor(
             vec!["var", "vars", "variable", "variables"],
             variable_tutor(),
         ),
-        (vec!["engine-q", "e-q"], engineq_tutor()),
         (vec!["block", "blocks"], block_tutor()),
         (vec!["shorthand", "shorthands"], shorthand_tutor()),
     ];
@@ -113,13 +113,22 @@ fn tutor(
             }
         }
 
-        let message = format!("You can find '{}' in the following topics:\n{}\n\nYou can learn about a topic using `tutor` followed by the name of the topic.\nFor example: `tutor table` to open the table topic.\n\n",
+        let message = format!(
-            find,
+            "You can find '{find}' in the following topics:\n\n{}\n\n{notes}",
             results.into_iter().map(|x| format!("- {}", x)).join("\n")
         );
 
         return Ok(display(&message, engine_state, stack, span));
     } else if let Some(search) = search {
+        if search == "list" {
+            let results = search_space.map(|s| s.0[0].to_string());
+            let message = format!(
+                "This tutorial contains the following topics:\n\n{}\n\n{notes}",
+                results.map(|x| format!("- {}", x)).join("\n")
+            );
+            return Ok(display(&message, engine_state, stack, span));
+        }
+
         for search_group in search_space {
             if search_group.0.contains(&search.as_str()) {
                 return Ok(display(search_group.1, engine_state, stack, span));
@@ -136,7 +145,8 @@ Welcome to the Nushell tutorial!
 With the `tutor` command, you'll be able to learn a lot about how Nushell
 works along with many fun tips and tricks to speed up everyday tasks.
 
-To get started, you can use `tutor begin`.
+To get started, you can use `tutor begin`, and to see all the available
+tutorials just run `tutor list`.
 
 "#
 }
@@ -390,29 +400,6 @@ same value using:
 "#
 }
 
-fn engineq_tutor() -> &'static str {
-    r#"
-Engine-q is the upcoming engine for Nushell. Build for speed and correctness,
-it also comes with a set of changes from Nushell versions prior to 0.60. To
-get ready for engine-q look for some of these changes that might impact your
-current scripts:
-
-* Engine-q now uses a few new data structures, including a record syntax
-  that allows you to model key-value pairs similar to JSON objects.
-* Environment variables can now contain more than just strings. Structured
-  values are converted to strings for external commands using converters.
-* `if` will now use an `else` keyword before the else block.
-* We're moving from "config.toml" to "config.nu". This means startup will
-  now be a script file.
-* `config` and its subcommands are being replaced by a record that you can
-  update in the shell which contains all the settings under the variable
-  `$config`.
-* bigint/bigdecimal values are now machine i64 and f64 values
-* And more, you can read more about upcoming changes in the up-to-date list
-  at: https://github.com/nushell/engine-q/issues/522
-"#
-}
-
 fn display(help: &str, engine_state: &EngineState, stack: &mut Stack, span: Span) -> PipelineData {
     let help = help.split('`');
 
@@ -424,7 +411,7 @@ fn display(help: &str, engine_state: &EngineState, stack: &mut Stack, span: Span
             code_mode = false;
 
             //TODO: support no-color mode
-            if let Some(highlighter) = engine_state.find_decl(b"nu-highlight") {
+            if let Some(highlighter) = engine_state.find_decl(b"nu-highlight", &[]) {
                 let decl = engine_state.get_decl(highlighter);
 
                 if let Ok(output) = decl.run(

crates/nu-command/src/core_commands/use_.rs

@@ -55,20 +55,20 @@ https://www.nushell.sh/book/thinking_in_nushell.html#parsing-and-evaluation-are-
             ));
         };
 
-        if let Some(overlay_id) = import_pattern.head.id {
+        if let Some(module_id) = import_pattern.head.id {
-            let overlay = engine_state.get_overlay(overlay_id);
+            let module = engine_state.get_module(module_id);
 
             let env_vars_to_use = if import_pattern.members.is_empty() {
-                overlay.env_vars_with_head(&import_pattern.head.name)
+                module.env_vars_with_head(&import_pattern.head.name)
             } else {
                 match &import_pattern.members[0] {
-                    ImportPatternMember::Glob { .. } => overlay.env_vars(),
+                    ImportPatternMember::Glob { .. } => module.env_vars(),
                     ImportPatternMember::Name { name, span } => {
                         let mut output = vec![];
 
-                        if let Some(id) = overlay.get_env_var_id(name) {
+                        if let Some(id) = module.get_env_var_id(name) {
                             output.push((name.clone(), id));
-                        } else if !overlay.has_decl(name) && !overlay.has_alias(name) {
+                        } else if !module.has_decl(name) && !module.has_alias(name) {
                             return Err(ShellError::EnvVarNotFoundAtRuntime(
                                 String::from_utf8_lossy(name).into(),
                                 *span,
@@ -81,9 +81,9 @@ https://www.nushell.sh/book/thinking_in_nushell.html#parsing-and-evaluation-are-
                         let mut output = vec![];
 
                         for (name, span) in names {
-                            if let Some(id) = overlay.get_env_var_id(name) {
+                            if let Some(id) = module.get_env_var_id(name) {
                                 output.push((name.clone(), id));
-                            } else if !overlay.has_decl(name) && !overlay.has_alias(name) {
+                            } else if !module.has_decl(name) && !module.has_alias(name) {
                                 return Err(ShellError::EnvVarNotFoundAtRuntime(
                                     String::from_utf8_lossy(name).into(),
                                     *span,
@@ -105,8 +105,6 @@ https://www.nushell.sh/book/thinking_in_nushell.html#parsing-and-evaluation-are-
 
                 let block = engine_state.get_block(block_id);
 
-                // TODO: Add string conversions (e.g. int to string)
-                // TODO: Later expand env to take all Values
                 let val = eval_block(
                     engine_state,
                     stack,

crates/nu-command/src/core_commands/version.rs

@@ -63,21 +63,7 @@ pub fn version(
         span: call.head,
     });
 
-    cols.push("tag".to_string());
+    let commit_hash: Option<&str> = option_env!("NU_COMMIT_HASH");
-    vals.push(Value::String {
-        val: shadow_rs::tag(),
-        span: call.head,
-    });
-
-    let short_commit: Option<&str> = Some(shadow::SHORT_COMMIT).filter(|x| !x.is_empty());
-    if let Some(short_commit) = short_commit {
-        cols.push("short_commit".to_string());
-        vals.push(Value::String {
-            val: short_commit.to_string(),
-            span: call.head,
-        });
-    }
-    let commit_hash: Option<&str> = Some(shadow::COMMIT_HASH).filter(|x| !x.is_empty());
     if let Some(commit_hash) = commit_hash {
         cols.push("commit_hash".to_string());
         vals.push(Value::String {
@@ -85,14 +71,6 @@ pub fn version(
             span: call.head,
         });
     }
-    let commit_date: Option<&str> = Some(shadow::COMMIT_DATE).filter(|x| !x.is_empty());
-    if let Some(commit_date) = commit_date {
-        cols.push("commit_date".to_string());
-        vals.push(Value::String {
-            val: commit_date.to_string(),
-            span: call.head,
-        });
-    }
 
     let build_os: Option<&str> = Some(shadow::BUILD_OS).filter(|x| !x.is_empty());
     if let Some(build_os) = build_os {
@@ -105,7 +83,7 @@ pub fn version(
 
     let build_target: Option<&str> = Some(shadow::BUILD_TARGET).filter(|x| !x.is_empty());
     if let Some(build_target) = build_target {
-        cols.push("build_os".to_string());
+        cols.push("build_target".to_string());
         vals.push(Value::String {
             val: build_target.to_string(),
             span: call.head,

crates/nu-command/src/database/commands/alias.rs

@@ -8,7 +8,7 @@ use nu_protocol::{
     engine::{Command, EngineState, Stack},
     Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape,
 };
-use sqlparser::ast::{Ident, SelectItem, SetExpr, TableAlias, TableFactor};
+use sqlparser::ast::{Ident, SelectItem, SetExpr, Statement, TableAlias, TableFactor};
 
 #[derive(Clone)]
 pub struct AliasExpr;
@@ -29,26 +29,15 @@ impl Command for AliasExpr {
     }
 
     fn examples(&self) -> Vec<Example> {
-        vec![
+        vec![Example {
-            Example {
+            description: "Creates an alias for a column selection",
-                description: "Creates an alias for a column selection",
+            example: "db col name_a | db as new_a",
-                example: "db col name_a | db as new_a",
+            result: None,
-                result: None,
+        }]
-            },
-            Example {
-                description: "Creates an alias for a table",
-                example: r#"db open name 
-    | db select a 
-    | db from table_a 
-    | db as table_a_new 
-    | db describe"#,
-                result: None,
-            },
-        ]
     }
 
     fn search_terms(&self) -> Vec<&str> {
-        vec!["database", "column", "expression"]
+        vec!["database", "alias", "column"]
     }
 
     fn run(
@@ -110,44 +99,56 @@ fn alias_db(
     new_alias: String,
     call: &Call,
 ) -> Result<PipelineData, ShellError> {
-    match db.query {
+    match db.statement.as_mut() {
         None => Err(ShellError::GenericError(
             "Error creating alias".into(),
-            "there is no query defined yet".into(),
+            "there is no statement defined yet".into(),
             Some(call.head),
             None,
             Vec::new(),
         )),
-        Some(ref mut query) => match &mut query.body {
+        Some(statement) => match statement {
-            SetExpr::Select(ref mut select) => {
+            Statement::Query(query) => match &mut query.body {
-                select.as_mut().from.iter_mut().for_each(|table| {
+                SetExpr::Select(select) => {
-                    let new_alias = Some(TableAlias {
+                    select.as_mut().from.iter_mut().for_each(|table| {
-                        name: Ident {
+                        let new_alias = Some(TableAlias {
-                            value: new_alias.clone(),
+                            name: Ident {
-                            quote_style: None,
+                                value: new_alias.clone(),
-                        },
+                                quote_style: None,
-                        columns: Vec::new(),
+                            },
+                            columns: Vec::new(),
+                        });
+
+                        if let TableFactor::Table { ref mut alias, .. } = table.relation {
+                            *alias = new_alias;
+                        } else if let TableFactor::Derived { ref mut alias, .. } = table.relation {
+                            *alias = new_alias;
+                        } else if let TableFactor::TableFunction { ref mut alias, .. } =
+                            table.relation
+                        {
+                            *alias = new_alias;
+                        }
                     });
 
-                    if let TableFactor::Table { ref mut alias, .. } = table.relation {
+                    Ok(db.into_value(call.head).into_pipeline_data())
-                        *alias = new_alias;
+                }
-                    } else if let TableFactor::Derived { ref mut alias, .. } = table.relation {
+                _ => Err(ShellError::GenericError(
-                        *alias = new_alias;
+                    "Error creating alias".into(),
-                    } else if let TableFactor::TableFunction { ref mut alias, .. } = table.relation
+                    "Query has no select from defined".into(),
-                    {
+                    Some(call.head),
-                        *alias = new_alias;
+                    None,
-                    }
+                    Vec::new(),
-                });
+                )),
-
+            },
-                Ok(db.into_value(call.head).into_pipeline_data())
+            s => {
+                return Err(ShellError::GenericError(
+                    "Connection doesnt define a query".into(),
+                    format!("Expected a connection with query. Got {}", s),
+                    Some(call.head),
+                    None,
+                    Vec::new(),
+                ))
             }
-            _ => Err(ShellError::GenericError(
-                "Error creating alias".into(),
-                "Query has no select from defined".into(),
-                Some(call.head),
-                None,
-                Vec::new(),
-            )),
         },
     }
 }

crates/nu-command/src/database/commands/and.rs

@@ -8,7 +8,7 @@ use nu_protocol::{
     Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, SyntaxShape,
     Value,
 };
-use sqlparser::ast::{BinaryOperator, Expr, Query, Select, SetExpr};
+use sqlparser::ast::{BinaryOperator, Expr, Query, Select, SetExpr, Statement};
 
 #[derive(Clone)]
 pub struct AndDb;
@@ -78,12 +78,23 @@ impl Command for AndDb {
 
             Ok(expression.into_value(call.head).into_pipeline_data())
         } else if let Ok(mut db) = SQLiteDatabase::try_from_value(value.clone()) {
-            db.query = match db.query {
+            match db.statement.as_mut() {
-                Some(query) => Some(modify_query(query, expr, call.head)?),
+                Some(statement) => match statement {
+                    Statement::Query(query) => modify_query(query, expr, call.head)?,
+                    s => {
+                        return Err(ShellError::GenericError(
+                            "Connection doesnt define a query".into(),
+                            format!("Expected a connection with query. Got {}", s),
+                            Some(call.head),
+                            None,
+                            Vec::new(),
+                        ))
+                    }
+                },
                 None => {
                     return Err(ShellError::GenericError(
-                        "Connection without query".into(),
+                        "Connection without statement".into(),
-                        "Missing query in the connection".into(),
+                        "The connection needs a statement defined".into(),
                         Some(call.head),
                         None,
                         Vec::new(),
@@ -103,26 +114,24 @@ impl Command for AndDb {
     }
 }
 
-fn modify_query(mut query: Query, expression: Expr, span: Span) -> Result<Query, ShellError> {
+fn modify_query(query: &mut Box<Query>, expression: Expr, span: Span) -> Result<(), ShellError> {
-    query.body = match query.body {
+    match query.body {
-        SetExpr::Select(select) => Ok(SetExpr::Select(modify_select(select, expression, span)?)),
+        SetExpr::Select(ref mut select) => modify_select(select, expression, span)?,
-        _ => Err(ShellError::GenericError(
+        _ => {
-            "Query without a select".into(),
+            return Err(ShellError::GenericError(
-            "Missing a WHERE clause before an AND clause".into(),
+                "Query without a select".into(),
-            Some(span),
+                "Missing a WHERE clause before an AND clause".into(),
-            None,
+                Some(span),
-            Vec::new(),
+                None,
-        )),
+                Vec::new(),
-    }?;
+            ))
+        }
+    };
 
-    Ok(query)
+    Ok(())
 }
 
-fn modify_select(
+fn modify_select(select: &mut Box<Select>, expression: Expr, span: Span) -> Result<(), ShellError> {
-    mut select: Box<Select>,
-    expression: Expr,
-    span: Span,
-) -> Result<Box<Select>, ShellError> {
     let new_expression = match &select.selection {
         Some(expr) => Ok(Expr::BinaryOp {
             left: Box::new(expr.clone()),
@@ -139,5 +148,5 @@ fn modify_select(
     }?;
 
     select.as_mut().selection = Some(new_expression);
-    Ok(select)
+    Ok(())
 }

crates/nu-command/src/database/commands/col.rs

@@ -27,7 +27,7 @@ impl Command for ColExpr {
     fn examples(&self) -> Vec<Example> {
         vec![Example {
             description: "Creates a named column expression",
-            example: "col name_1",
+            example: "db col name_1",
             result: None,
         }]
     }

crates/nu-command/src/database/commands/conversions.rs

@@ -0,0 +1,66 @@
+use crate::{database::values::definitions::ConnectionDb, SQLiteDatabase};
+use nu_protocol::{ShellError, Value};
+use sqlparser::ast::{ObjectName, Statement, TableAlias, TableFactor};
+
+pub fn value_into_table_factor(
+    table: Value,
+    connection: &ConnectionDb,
+    alias: Option<TableAlias>,
+) -> Result<TableFactor, ShellError> {
+    match table {
+        Value::String { val, .. } => {
+            let ident = sqlparser::ast::Ident {
+                value: val,
+                quote_style: None,
+            };
+
+            Ok(TableFactor::Table {
+                name: ObjectName(vec![ident]),
+                alias,
+                args: Vec::new(),
+                with_hints: Vec::new(),
+            })
+        }
+        Value::CustomValue { span, .. } => {
+            let db = SQLiteDatabase::try_from_value(table)?;
+
+            if &db.connection != connection {
+                return Err(ShellError::GenericError(
+                    "Incompatible connections".into(),
+                    "trying to join on table with different connection".into(),
+                    Some(span),
+                    None,
+                    Vec::new(),
+                ));
+            }
+
+            match db.statement {
+                Some(statement) => match statement {
+                    Statement::Query(query) => Ok(TableFactor::Derived {
+                        lateral: false,
+                        subquery: query,
+                        alias,
+                    }),
+                    s => Err(ShellError::GenericError(
+                        "Connection doesnt define a query".into(),
+                        format!("Expected a connection with query. Got {}", s),
+                        Some(span),
+                        None,
+                        Vec::new(),
+                    )),
+                },
+                None => Err(ShellError::GenericError(
+                    "Error creating derived table".into(),
+                    "there is no statement defined yet".into(),
+                    Some(span),
+                    None,
+                    Vec::new(),
+                )),
+            }
+        }
+        _ => Err(ShellError::UnsupportedInput(
+            "String or connection".into(),
+            table.span()?,
+        )),
+    }
+}

crates/nu-command/src/database/commands/from.rs

@@ -1,11 +1,13 @@
-use super::super::SQLiteDatabase;
+use crate::database::values::definitions::ConnectionDb;
+
+use super::{super::SQLiteDatabase, conversions::value_into_table_factor};
 use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape,
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
 };
-use sqlparser::ast::{Ident, ObjectName, Query, Select, SetExpr, TableFactor, TableWithJoins};
+use sqlparser::ast::{Ident, Query, Select, SetExpr, Statement, TableAlias, TableWithJoins};
 
 #[derive(Clone)]
 pub struct FromDb;
@@ -23,8 +25,14 @@ impl Command for FromDb {
         Signature::build(self.name())
             .required(
                 "select",
+                SyntaxShape::Any,
+                "table of derived table to select from",
+            )
+            .named(
+                "as",
                 SyntaxShape::String,
-                "Name of table to select from",
+                "Alias for the selected table",
+                Some('a'),
             )
             .category(Category::Custom("database".into()))
     }
@@ -48,51 +56,94 @@ impl Command for FromDb {
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        let table: String = call.req(engine_state, stack, 0)?;
-
         let mut db = SQLiteDatabase::try_from_pipeline(input, call.head)?;
-        db.query = match db.query {
+        db.statement = match db.statement {
-            None => Some(create_query(table)),
+            None => Some(create_statement(&db.connection, engine_state, stack, call)?),
-            Some(query) => Some(modify_query(query, table)),
+            Some(statement) => Some(modify_statement(
+                &db.connection,
+                statement,
+                engine_state,
+                stack,
+                call,
+            )?),
         };
 
         Ok(db.into_value(call.head).into_pipeline_data())
     }
 }
 
-fn create_query(table: String) -> Query {
+fn create_statement(
-    Query {
+    connection: &ConnectionDb,
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<Statement, ShellError> {
+    let query = Query {
         with: None,
-        body: SetExpr::Select(Box::new(create_select(table))),
+        body: SetExpr::Select(Box::new(create_select(
+            connection,
+            engine_state,
+            stack,
+            call,
+        )?)),
         order_by: Vec::new(),
         limit: None,
         offset: None,
         fetch: None,
         lock: None,
+    };
+
+    Ok(Statement::Query(Box::new(query)))
+}
+
+fn modify_statement(
+    connection: &ConnectionDb,
+    mut statement: Statement,
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<Statement, ShellError> {
+    match statement {
+        Statement::Query(ref mut query) => {
+            match query.body {
+                SetExpr::Select(ref mut select) => {
+                    let table = create_table(connection, engine_state, stack, call)?;
+                    select.from.push(table);
+                }
+                _ => {
+                    query.as_mut().body = SetExpr::Select(Box::new(create_select(
+                        connection,
+                        engine_state,
+                        stack,
+                        call,
+                    )?));
+                }
+            };
+
+            Ok(statement)
+        }
+        s => Err(ShellError::GenericError(
+            "Connection doesnt define a query".into(),
+            format!("Expected a connection with query. Got {}", s),
+            Some(call.head),
+            None,
+            Vec::new(),
+        )),
     }
 }
 
-fn modify_query(mut query: Query, table: String) -> Query {
+fn create_select(
-    query.body = match query.body {
+    connection: &ConnectionDb,
-        SetExpr::Select(select) => SetExpr::Select(modify_select(select, table)),
+    engine_state: &EngineState,
-        _ => SetExpr::Select(Box::new(create_select(table))),
+    stack: &mut Stack,
-    };
+    call: &Call,
-
+) -> Result<Select, ShellError> {
-    query
+    Ok(Select {
-}
-
-fn modify_select(mut select: Box<Select>, table: String) -> Box<Select> {
-    select.as_mut().from = create_from(table);
-    select
-}
-
-fn create_select(table: String) -> Select {
-    Select {
         distinct: false,
         top: None,
         projection: Vec::new(),
         into: None,
-        from: create_from(table),
+        from: vec![create_table(connection, engine_state, stack, call)?],
         lateral_views: Vec::new(),
         selection: None,
         group_by: Vec::new(),
@@ -100,29 +151,32 @@ fn create_select(table: String) -> Select {
         distribute_by: Vec::new(),
         sort_by: Vec::new(),
         having: None,
-    }
+    })
 }
 
-// This function needs more work
+fn create_table(
-// It needs to define multi tables and joins
+    connection: &ConnectionDb,
-// I assume we will need to define expressions for the columns instead of strings
+    engine_state: &EngineState,
-fn create_from(table: String) -> Vec<TableWithJoins> {
+    stack: &mut Stack,
-    let ident = Ident {
+    call: &Call,
-        value: table,
+) -> Result<TableWithJoins, ShellError> {
-        quote_style: None,
+    let alias = call
-    };
+        .get_flag::<String>(engine_state, stack, "as")?
+        .map(|alias| TableAlias {
+            name: Ident {
+                value: alias,
+                quote_style: None,
+            },
+            columns: Vec::new(),
+        });
 
-    let table_factor = TableFactor::Table {
+    let select_table: Value = call.req(engine_state, stack, 0)?;
-        name: ObjectName(vec![ident]),
+    let table_factor = value_into_table_factor(select_table, connection, alias)?;
-        alias: None,
-        args: Vec::new(),
-        with_hints: Vec::new(),
-    };
 
     let table = TableWithJoins {
         relation: table_factor,
         joins: Vec::new(),
     };
 
-    vec![table]
+    Ok(table)
 }

crates/nu-command/src/database/commands/function.rs

@@ -0,0 +1,85 @@
+use crate::database::values::dsl::ExprDb;
+use nu_engine::CallExt;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
+};
+use sqlparser::ast::{Expr, Function, FunctionArg, FunctionArgExpr, Ident, ObjectName};
+
+#[derive(Clone)]
+pub struct FunctionExpr;
+
+impl Command for FunctionExpr {
+    fn name(&self) -> &str {
+        "db fn"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .required("name", SyntaxShape::String, "function name")
+            .switch("distinct", "distict values", Some('d'))
+            .rest("arguments", SyntaxShape::Any, "function arguments")
+            .category(Category::Custom("database".into()))
+    }
+
+    fn usage(&self) -> &str {
+        "Creates function expression for a select operation"
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Creates a function expression",
+            example: "db fn count name_1",
+            result: None,
+        }]
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["database", "function", "expression"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        _input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let name: String = call.req(engine_state, stack, 0)?;
+        let vals: Vec<Value> = call.rest(engine_state, stack, 1)?;
+        let value = Value::List {
+            vals,
+            span: call.head,
+        };
+        let expressions = ExprDb::extract_exprs(value)?;
+
+        let name: Vec<Ident> = name
+            .split('.')
+            .map(|part| Ident {
+                value: part.to_string(),
+                quote_style: None,
+            })
+            .collect();
+        let name = ObjectName(name);
+
+        let args: Vec<FunctionArg> = expressions
+            .into_iter()
+            .map(|expr| {
+                let arg = FunctionArgExpr::Expr(expr);
+
+                FunctionArg::Unnamed(arg)
+            })
+            .collect();
+
+        let expression: ExprDb = Expr::Function(Function {
+            name,
+            args,
+            over: None,
+            distinct: call.has_flag("distinct"),
+        })
+        .into();
+
+        Ok(expression.into_value(call.head).into_pipeline_data())
+    }
+}

crates/nu-command/src/database/commands/group_by.rs

@@ -0,0 +1,102 @@
+use crate::database::values::dsl::ExprDb;
+
+use super::super::SQLiteDatabase;
+use nu_engine::CallExt;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
+};
+use sqlparser::ast::{SetExpr, Statement};
+
+#[derive(Clone)]
+pub struct GroupByDb;
+
+impl Command for GroupByDb {
+    fn name(&self) -> &str {
+        "db group-by"
+    }
+
+    fn usage(&self) -> &str {
+        "Group by query"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .rest(
+                "select",
+                SyntaxShape::Any,
+                "Select expression(s) on the table",
+            )
+            .category(Category::Custom("database".into()))
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["database", "select"]
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "orders query by a column",
+            example: r#"db open db.mysql 
+    | db from table_a 
+    | db select a 
+    | db group-by a 
+    | db describe"#,
+            result: None,
+        }]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
+        let value = Value::List {
+            vals,
+            span: call.head,
+        };
+        let expressions = ExprDb::extract_exprs(value)?;
+
+        let mut db = SQLiteDatabase::try_from_pipeline(input, call.head)?;
+        match db.statement.as_mut() {
+            Some(statement) => match statement {
+                Statement::Query(ref mut query) => match &mut query.body {
+                    SetExpr::Select(ref mut select) => select.group_by = expressions,
+                    s => {
+                        return Err(ShellError::GenericError(
+                            "Connection doesnt define a select".into(),
+                            format!("Expected a connection with select query. Got {}", s),
+                            Some(call.head),
+                            None,
+                            Vec::new(),
+                        ))
+                    }
+                },
+                s => {
+                    return Err(ShellError::GenericError(
+                        "Connection doesnt define a query".into(),
+                        format!("Expected a connection with query. Got {}", s),
+                        Some(call.head),
+                        None,
+                        Vec::new(),
+                    ))
+                }
+            },
+            None => {
+                return Err(ShellError::GenericError(
+                    "Connection without statement".into(),
+                    "The connection needs a statement defined".into(),
+                    Some(call.head),
+                    None,
+                    Vec::new(),
+                ))
+            }
+        };
+
+        Ok(db.into_value(call.head).into_pipeline_data())
+    }
+}

crates/nu-command/src/database/commands/join.rs

@@ -0,0 +1,178 @@
+use super::{super::SQLiteDatabase, conversions::value_into_table_factor};
+use crate::database::values::{definitions::ConnectionDb, dsl::ExprDb};
+use nu_engine::CallExt;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
+};
+use sqlparser::ast::{
+    Ident, Join, JoinConstraint, JoinOperator, Select, SetExpr, Statement, TableAlias,
+};
+
+#[derive(Clone)]
+pub struct JoinDb;
+
+impl Command for JoinDb {
+    fn name(&self) -> &str {
+        "db join"
+    }
+
+    fn usage(&self) -> &str {
+        "Joins with another table or derived table. Default join type is inner"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .required(
+                "table",
+                SyntaxShape::Any,
+                "table or derived table to join on",
+            )
+            .required("on", SyntaxShape::Any, "expression to join tables")
+            .named(
+                "as",
+                SyntaxShape::String,
+                "Alias for the selected join",
+                Some('a'),
+            )
+            .switch("left", "left outer join", Some('l'))
+            .switch("right", "right outer join", Some('r'))
+            .switch("outer", "full outer join", Some('o'))
+            .switch("cross", "cross join", Some('c'))
+            .category(Category::Custom("database".into()))
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["database", "join"]
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "",
+            example: "",
+            result: None,
+        }]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let mut db = SQLiteDatabase::try_from_pipeline(input, call.head)?;
+
+        db.statement = match db.statement {
+            Some(statement) => Some(modify_statement(
+                &db.connection,
+                statement,
+                engine_state,
+                stack,
+                call,
+            )?),
+            None => {
+                return Err(ShellError::GenericError(
+                    "Error creating join".into(),
+                    "there is no statement defined yet".into(),
+                    Some(call.head),
+                    None,
+                    Vec::new(),
+                ))
+            }
+        };
+
+        Ok(db.into_value(call.head).into_pipeline_data())
+    }
+}
+
+fn modify_statement(
+    connection: &ConnectionDb,
+    mut statement: Statement,
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<Statement, ShellError> {
+    match statement {
+        Statement::Query(ref mut query) => {
+            match &mut query.body {
+                SetExpr::Select(ref mut select) => {
+                    modify_from(connection, select, engine_state, stack, call)?
+                }
+                s => {
+                    return Err(ShellError::GenericError(
+                        "Connection doesnt define a select".into(),
+                        format!("Expected a connection with select. Got {}", s),
+                        Some(call.head),
+                        None,
+                        Vec::new(),
+                    ))
+                }
+            };
+
+            Ok(statement)
+        }
+        s => Err(ShellError::GenericError(
+            "Connection doesnt define a query".into(),
+            format!("Expected a connection with query. Got {}", s),
+            Some(call.head),
+            None,
+            Vec::new(),
+        )),
+    }
+}
+
+fn modify_from(
+    connection: &ConnectionDb,
+    select: &mut Select,
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+) -> Result<(), ShellError> {
+    match select.from.last_mut() {
+        Some(table) => {
+            let alias = call
+                .get_flag::<String>(engine_state, stack, "as")?
+                .map(|alias| TableAlias {
+                    name: Ident {
+                        value: alias,
+                        quote_style: None,
+                    },
+                    columns: Vec::new(),
+                });
+
+            let join_table: Value = call.req(engine_state, stack, 0)?;
+            let table_factor = value_into_table_factor(join_table, connection, alias)?;
+
+            let on_expr: Value = call.req(engine_state, stack, 1)?;
+            let on_expr = ExprDb::try_from_value(&on_expr)?;
+
+            let join_on = if call.has_flag("left") {
+                JoinOperator::LeftOuter(JoinConstraint::On(on_expr.into_native()))
+            } else if call.has_flag("right") {
+                JoinOperator::RightOuter(JoinConstraint::On(on_expr.into_native()))
+            } else if call.has_flag("outer") {
+                JoinOperator::FullOuter(JoinConstraint::On(on_expr.into_native()))
+            } else {
+                JoinOperator::Inner(JoinConstraint::On(on_expr.into_native()))
+            };
+
+            let join = Join {
+                relation: table_factor,
+                join_operator: join_on,
+            };
+
+            table.joins.push(join);
+
+            Ok(())
+        }
+        None => Err(ShellError::GenericError(
+            "Connection without table defined".into(),
+            "Expected a table defined".into(),
+            Some(call.head),
+            None,
+            Vec::new(),
+        )),
+    }
+}

crates/nu-command/src/database/commands/limit.rs

@@ -6,6 +6,7 @@ use nu_protocol::{
     engine::{Command, EngineState, Stack},
     Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
 };
+use sqlparser::ast::Statement;
 
 #[derive(Clone)]
 pub struct LimitDb;
@@ -56,11 +57,19 @@ impl Command for LimitDb {
         let expr = ExprDb::try_from_value(&limit)?.into_native();
 
         let mut db = SQLiteDatabase::try_from_pipeline(input, call.head)?;
-        db.query = match db.query {
+        match db.statement {
-            Some(mut query) => {
+            Some(ref mut statement) => match statement {
-                query.limit = Some(expr);
+                Statement::Query(query) => query.as_mut().limit = Some(expr),
-                Some(query)
+                s => {
-            }
+                    return Err(ShellError::GenericError(
+                        "Connection doesnt define a statement".into(),
+                        format!("Expected a connection with query. Got {}", s),
+                        Some(call.head),
+                        None,
+                        Vec::new(),
+                    ))
+                }
+            },
             None => {
                 return Err(ShellError::GenericError(
                     "Connection without query".into(),

crates/nu-command/src/database/commands/mod.rs

@@ -1,12 +1,21 @@
+// Conversions between value and sqlparser objects
+pub mod conversions;
+
+mod alias;
 mod and;
+mod col;
 mod collect;
 mod command;
 mod describe;
 mod from;
+mod function;
+mod group_by;
+mod join;
 mod limit;
 mod open;
 mod or;
 mod order_by;
+mod over;
 mod query;
 mod schema;
 mod select;
@@ -18,21 +27,27 @@ use testing::TestingDb;
 
 use nu_protocol::engine::StateWorkingSet;
 
+use alias::AliasExpr;
 use and::AndDb;
+use col::ColExpr;
 use collect::CollectDb;
 use command::Database;
 use describe::DescribeDb;
 use from::FromDb;
+use function::FunctionExpr;
+use group_by::GroupByDb;
+use join::JoinDb;
 use limit::LimitDb;
 use open::OpenDb;
 use or::OrDb;
 use order_by::OrderByDb;
+use over::OverExpr;
 use query::QueryDb;
 use schema::SchemaDb;
 use select::ProjectionDb;
 use where_::WhereDb;
 
-pub fn add_commands_decls(working_set: &mut StateWorkingSet) {
+pub fn add_database_decls(working_set: &mut StateWorkingSet) {
     macro_rules! bind_command {
             ( $command:expr ) => {
                 working_set.add_decl(Box::new($command));
@@ -44,17 +59,23 @@ pub fn add_commands_decls(working_set: &mut StateWorkingSet) {
 
     // Series commands
     bind_command!(
+        AliasExpr,
         AndDb,
+        ColExpr,
         CollectDb,
         Database,
         DescribeDb,
         FromDb,
-        QueryDb,
+        FunctionExpr,
+        GroupByDb,
+        JoinDb,
         LimitDb,
-        ProjectionDb,
         OpenDb,
         OrderByDb,
         OrDb,
+        OverExpr,
+        QueryDb,
+        ProjectionDb,
         SchemaDb,
         TestingDb,
         WhereDb

crates/nu-command/src/database/commands/or.rs

@@ -8,7 +8,7 @@ use nu_protocol::{
     Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, SyntaxShape,
     Value,
 };
-use sqlparser::ast::{BinaryOperator, Expr, Query, Select, SetExpr};
+use sqlparser::ast::{BinaryOperator, Expr, Query, Select, SetExpr, Statement};
 
 #[derive(Clone)]
 pub struct OrDb;
@@ -78,12 +78,23 @@ impl Command for OrDb {
 
             Ok(expression.into_value(call.head).into_pipeline_data())
         } else if let Ok(mut db) = SQLiteDatabase::try_from_value(value.clone()) {
-            db.query = match db.query {
+            match db.statement {
-                Some(query) => Some(modify_query(query, expr, call.head)?),
+                Some(ref mut statement) => match statement {
+                    Statement::Query(query) => modify_query(query, expr, call.head)?,
+                    s => {
+                        return Err(ShellError::GenericError(
+                            "Connection doesnt define a query".into(),
+                            format!("Expected a connection with query. Got {}", s),
+                            Some(call.head),
+                            None,
+                            Vec::new(),
+                        ))
+                    }
+                },
                 None => {
                     return Err(ShellError::GenericError(
-                        "Connection without query".into(),
+                        "Connection without statement".into(),
-                        "Missing query in the connection".into(),
+                        "The connection needs a statement defined".into(),
                         Some(call.head),
                         None,
                         Vec::new(),
@@ -103,26 +114,24 @@ impl Command for OrDb {
     }
 }
 
-fn modify_query(mut query: Query, expression: Expr, span: Span) -> Result<Query, ShellError> {
+fn modify_query(query: &mut Box<Query>, expression: Expr, span: Span) -> Result<(), ShellError> {
-    query.body = match query.body {
+    match query.body {
-        SetExpr::Select(select) => Ok(SetExpr::Select(modify_select(select, expression, span)?)),
+        SetExpr::Select(ref mut select) => modify_select(select, expression, span)?,
-        _ => Err(ShellError::GenericError(
+        _ => {
-            "Query without a select".into(),
+            return Err(ShellError::GenericError(
-            "Missing a WHERE clause before an OR clause".into(),
+                "Query without a select".into(),
-            Some(span),
+                "Missing a WHERE clause before an OR clause".into(),
-            None,
+                Some(span),
-            Vec::new(),
+                None,
-        )),
+                Vec::new(),
-    }?;
+            ))
+        }
+    };
 
-    Ok(query)
+    Ok(())
 }
 
-fn modify_select(
+fn modify_select(select: &mut Box<Select>, expression: Expr, span: Span) -> Result<(), ShellError> {
-    mut select: Box<Select>,
-    expression: Expr,
-    span: Span,
-) -> Result<Box<Select>, ShellError> {
     let new_expression = match &select.selection {
         Some(expr) => Ok(Expr::BinaryOp {
             left: Box::new(expr.clone()),
@@ -139,5 +148,5 @@ fn modify_select(
     }?;
 
     select.as_mut().selection = Some(new_expression);
-    Ok(select)
+    Ok(())
 }

crates/nu-command/src/database/commands/order_by.rs

@@ -7,7 +7,7 @@ use nu_protocol::{
     engine::{Command, EngineState, Stack},
     Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
 };
-use sqlparser::ast::OrderByExpr;
+use sqlparser::ast::{Expr, OrderByExpr, Statement};
 
 #[derive(Clone)]
 pub struct OrderByDb;
@@ -58,40 +58,100 @@ impl Command for OrderByDb {
     ) -> Result<PipelineData, ShellError> {
         let asc = call.has_flag("ascending");
         let nulls_first = call.has_flag("nulls_first");
-
+        let expressions: Vec<Value> = call.rest(engine_state, stack, 0)?;
-        let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
+        let expressions = Value::List {
-        let value = Value::List {
+            vals: expressions,
-            vals,
             span: call.head,
         };
-        let expressions = ExprDb::extract_exprs(value)?;
+        let expressions = ExprDb::extract_exprs(expressions)?;
+        let expressions: Vec<OrderByExpr> = expressions
+            .into_iter()
+            .map(|expr| OrderByExpr {
+                expr,
+                asc: if asc { Some(asc) } else { None },
+                nulls_first: if nulls_first { Some(nulls_first) } else { None },
+            })
+            .collect();
 
-        let mut db = SQLiteDatabase::try_from_pipeline(input, call.head)?;
+        let value = input.into_value(call.head);
-        db.query = match db.query {
-            Some(mut query) => {
-                let mut order_expr: Vec<OrderByExpr> = expressions
-                    .into_iter()
-                    .map(|expr| OrderByExpr {
-                        expr,
-                        asc: if asc { Some(asc) } else { None },
-                        nulls_first: if nulls_first { Some(nulls_first) } else { None },
-                    })
-                    .collect();
 
-                query.order_by.append(&mut order_expr);
+        if let Ok(expr) = ExprDb::try_from_value(&value) {
-                Some(query)
+            update_expressions(expr, expressions, call)
-            }
+        } else if let Ok(db) = SQLiteDatabase::try_from_value(value.clone()) {
+            update_connection(db, expressions, call)
+        } else {
+            Err(ShellError::CantConvert(
+                "expression or query".into(),
+                value.get_type().to_string(),
+                value.span()?,
+                None,
+            ))
+        }
+    }
+}
+
+fn update_expressions(
+    mut expr: ExprDb,
+    mut expressions: Vec<OrderByExpr>,
+    call: &Call,
+) -> Result<PipelineData, ShellError> {
+    match expr.as_mut() {
+        Expr::Function(function) => match &mut function.over {
+            Some(over) => over.order_by.append(&mut expressions),
             None => {
                 return Err(ShellError::GenericError(
-                    "Connection without query".into(),
+                    "Expression doesnt define a partition to order".into(),
-                    "The connection needs a query defined".into(),
+                    "Expected an expression with partition".into(),
                     Some(call.head),
                     None,
                     Vec::new(),
                 ))
             }
-        };
+        },
+        s => {
+            return Err(ShellError::GenericError(
+                "Expression doesnt define a function".into(),
+                format!("Expected an expression with a function. Got {}", s),
+                Some(call.head),
+                None,
+                Vec::new(),
+            ))
+        }
+    };
 
-        Ok(db.into_value(call.head).into_pipeline_data())
+    Ok(expr.into_value(call.head).into_pipeline_data())
-    }
+}
+
+fn update_connection(
+    mut db: SQLiteDatabase,
+    mut expressions: Vec<OrderByExpr>,
+    call: &Call,
+) -> Result<PipelineData, ShellError> {
+    match db.statement.as_mut() {
+        Some(statement) => match statement {
+            Statement::Query(query) => {
+                query.order_by.append(&mut expressions);
+            }
+            s => {
+                return Err(ShellError::GenericError(
+                    "Connection doesnt define a query".into(),
+                    format!("Expected a connection with query. Got {}", s),
+                    Some(call.head),
+                    None,
+                    Vec::new(),
+                ))
+            }
+        },
+        None => {
+            return Err(ShellError::GenericError(
+                "Connection without statement".into(),
+                "The connection needs a statement defined".into(),
+                Some(call.head),
+                None,
+                Vec::new(),
+            ))
+        }
+    };
+
+    Ok(db.into_value(call.head).into_pipeline_data())
 }

crates/nu-command/src/database/commands/over.rs

@@ -0,0 +1,80 @@
+use crate::database::values::dsl::ExprDb;
+use nu_engine::CallExt;
+use nu_protocol::{
+    ast::Call,
+    engine::{Command, EngineState, Stack},
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
+};
+use sqlparser::ast::{Expr, WindowSpec};
+
+#[derive(Clone)]
+pub struct OverExpr;
+
+impl Command for OverExpr {
+    fn name(&self) -> &str {
+        "db over"
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::build(self.name())
+            .rest(
+                "partition-by",
+                SyntaxShape::Any,
+                "columns to partition the window function",
+            )
+            .category(Category::Custom("database".into()))
+    }
+
+    fn usage(&self) -> &str {
+        "Adds a partition to an expression function"
+    }
+
+    fn examples(&self) -> Vec<Example> {
+        vec![Example {
+            description: "Adds a partition to a function expresssion",
+            example: "db function avg col_a | db over col_b",
+            result: None,
+        }]
+    }
+
+    fn search_terms(&self) -> Vec<&str> {
+        vec!["database", "column", "expression"]
+    }
+
+    fn run(
+        &self,
+        engine_state: &EngineState,
+        stack: &mut Stack,
+        call: &Call,
+        input: PipelineData,
+    ) -> Result<PipelineData, ShellError> {
+        let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
+        let value = Value::List {
+            vals,
+            span: call.head,
+        };
+        let partitions = ExprDb::extract_exprs(value)?;
+
+        let mut expression = ExprDb::try_from_pipeline(input, call.head)?;
+        match expression.as_mut() {
+            Expr::Function(function) => {
+                function.over = Some(WindowSpec {
+                    partition_by: partitions,
+                    order_by: Vec::new(),
+                    window_frame: None,
+                });
+            }
+            s => {
+                return Err(ShellError::GenericError(
+                    "Expression doesnt define a function".into(),
+                    format!("Expected an expression with a function. Got {}", s),
+                    Some(call.head),
+                    None,
+                    Vec::new(),
+                ))
+            }
+        };
+
+        Ok(expression.into_value(call.head).into_pipeline_data())
+    }
+}

crates/nu-command/src/database/commands/schema.rs

@@ -1,11 +1,11 @@
 use super::super::SQLiteDatabase;
-use crate::database::values::definitions::db_row::DbRow;
+use crate::database::values::definitions::{db::Db, db_row::DbRow, db_table::DbTable};
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Category, Example, PipelineData, ShellError, Signature, Value,
+    Category, Example, PipelineData, ShellError, Signature, Span, Value,
 };
-
+use rusqlite::Connection;
 #[derive(Clone)]
 pub struct SchemaDb;
 
@@ -46,142 +46,25 @@ impl Command for SchemaDb {
         let span = call.head;
 
         let sqlite_db = SQLiteDatabase::try_from_pipeline(input, span)?;
-        let conn = sqlite_db.open_connection().map_err(|e| {
+        let conn = open_sqlite_db_connection(&sqlite_db, span)?;
-            ShellError::GenericError(
+        let dbs = get_databases_and_tables(&sqlite_db, &conn, span)?;
-                "Error opening file".into(),
-                e.to_string(),
-                Some(span),
-                None,
-                Vec::new(),
-            )
-        })?;
-
-        let dbs = sqlite_db.get_databases_and_tables(&conn).map_err(|e| {
-            ShellError::GenericError(
-                "Error getting databases and tables".into(),
-                e.to_string(),
-                Some(span),
-                None,
-                Vec::new(),
-            )
-        })?;
 
         cols.push("db_filename".into());
         vals.push(Value::String {
-            val: sqlite_db.path.to_string_lossy().to_string(),
+            val: sqlite_db.connection.to_string(),
             span,
         });
 
         for db in dbs {
-            let tables = db.tables();
+            let tables = get_database_tables(&db);
             let mut table_list: Vec<Value> = vec![];
             let mut table_names = vec![];
             let mut table_values = vec![];
             for table in tables {
-                let columns = sqlite_db.get_columns(&conn, &table).map_err(|e| {
+                let column_info = get_table_columns(&sqlite_db, &conn, &table, span)?;
-                    ShellError::GenericError(
+                let constraint_info = get_table_constraints(&sqlite_db, &conn, &table, span)?;
-                        "Error getting database columns".into(),
+                let foreign_key_info = get_table_foreign_keys(&sqlite_db, &conn, &table, span)?;
-                        e.to_string(),
+                let index_info = get_table_indexes(&sqlite_db, &conn, &table, span)?;
-                        Some(span),
-                        None,
-                        Vec::new(),
-                    )
-                })?;
-                // a record of column name = column value
-                let mut column_info = vec![];
-                for t in columns {
-                    let mut col_names = vec![];
-                    let mut col_values = vec![];
-                    let fields = t.fields();
-                    let columns = t.columns();
-                    for (k, v) in fields.iter().zip(columns.iter()) {
-                        col_names.push(k.clone());
-                        col_values.push(Value::string(v.clone(), span));
-                    }
-                    column_info.push(Value::Record {
-                        cols: col_names.clone(),
-                        vals: col_values.clone(),
-                        span,
-                    });
-                }
-
-                let constraints = sqlite_db.get_constraints(&conn, &table).map_err(|e| {
-                    ShellError::GenericError(
-                        "Error getting DB constraints".into(),
-                        e.to_string(),
-                        Some(span),
-                        None,
-                        Vec::new(),
-                    )
-                })?;
-                let mut constraint_info = vec![];
-                for constraint in constraints {
-                    let mut con_cols = vec![];
-                    let mut con_vals = vec![];
-                    let fields = constraint.fields();
-                    let columns = constraint.columns();
-                    for (k, v) in fields.iter().zip(columns.iter()) {
-                        con_cols.push(k.clone());
-                        con_vals.push(Value::string(v.clone(), span));
-                    }
-                    constraint_info.push(Value::Record {
-                        cols: con_cols.clone(),
-                        vals: con_vals.clone(),
-                        span,
-                    });
-                }
-
-                let foreign_keys = sqlite_db.get_foreign_keys(&conn, &table).map_err(|e| {
-                    ShellError::GenericError(
-                        "Error getting DB Foreign Keys".into(),
-                        e.to_string(),
-                        Some(span),
-                        None,
-                        Vec::new(),
-                    )
-                })?;
-                let mut foreign_key_info = vec![];
-                for fk in foreign_keys {
-                    let mut fk_cols = vec![];
-                    let mut fk_vals = vec![];
-                    let fields = fk.fields();
-                    let columns = fk.columns();
-                    for (k, v) in fields.iter().zip(columns.iter()) {
-                        fk_cols.push(k.clone());
-                        fk_vals.push(Value::string(v.clone(), span));
-                    }
-                    foreign_key_info.push(Value::Record {
-                        cols: fk_cols.clone(),
-                        vals: fk_vals.clone(),
-                        span,
-                    });
-                }
-
-                let indexes = sqlite_db.get_indexes(&conn, &table).map_err(|e| {
-                    ShellError::GenericError(
-                        "Error getting DB Indexes".into(),
-                        e.to_string(),
-                        Some(span),
-                        None,
-                        Vec::new(),
-                    )
-                })?;
-                let mut index_info = vec![];
-                for index in indexes {
-                    let mut idx_cols = vec![];
-                    let mut idx_vals = vec![];
-                    let fields = index.fields();
-                    let columns = index.columns();
-                    for (k, v) in fields.iter().zip(columns.iter()) {
-                        idx_cols.push(k.clone());
-                        idx_vals.push(Value::string(v.clone(), span));
-                    }
-                    index_info.push(Value::Record {
-                        cols: idx_cols.clone(),
-                        vals: idx_vals.clone(),
-                        span,
-                    });
-                }
 
                 table_names.push(table.name);
                 table_values.push(Value::Record {
@@ -219,12 +102,15 @@ impl Command for SchemaDb {
             });
 
             cols.push("databases".into());
+
             let mut rcols = vec![];
             let mut rvals = vec![];
             rcols.push("name".into());
             rvals.push(Value::string(db.name().to_string(), span));
+
             rcols.push("tables".into());
             rvals.append(&mut table_list);
+
             vals.push(Value::Record {
                 cols: rcols,
                 vals: rvals,
@@ -238,3 +124,177 @@ impl Command for SchemaDb {
         ))
     }
 }
+
+fn open_sqlite_db_connection(db: &SQLiteDatabase, span: Span) -> Result<Connection, ShellError> {
+    db.open_connection().map_err(|e| {
+        ShellError::GenericError(
+            "Error opening file".into(),
+            e.to_string(),
+            Some(span),
+            None,
+            Vec::new(),
+        )
+    })
+}
+
+fn get_databases_and_tables(
+    db: &SQLiteDatabase,
+    conn: &Connection,
+    span: Span,
+) -> Result<Vec<Db>, ShellError> {
+    db.get_databases_and_tables(conn).map_err(|e| {
+        ShellError::GenericError(
+            "Error getting databases and tables".into(),
+            e.to_string(),
+            Some(span),
+            None,
+            Vec::new(),
+        )
+    })
+}
+
+fn get_database_tables(db: &Db) -> Vec<DbTable> {
+    db.tables()
+}
+
+fn get_table_columns(
+    db: &SQLiteDatabase,
+    conn: &Connection,
+    table: &DbTable,
+    span: Span,
+) -> Result<Vec<Value>, ShellError> {
+    let columns = db.get_columns(conn, table).map_err(|e| {
+        ShellError::GenericError(
+            "Error getting database columns".into(),
+            e.to_string(),
+            Some(span),
+            None,
+            Vec::new(),
+        )
+    })?;
+
+    // a record of column name = column value
+    let mut column_info = vec![];
+    for t in columns {
+        let mut col_names = vec![];
+        let mut col_values = vec![];
+        let fields = t.fields();
+        let columns = t.columns();
+        for (k, v) in fields.iter().zip(columns.iter()) {
+            col_names.push(k.clone());
+            col_values.push(Value::string(v.clone(), span));
+        }
+        column_info.push(Value::Record {
+            cols: col_names.clone(),
+            vals: col_values.clone(),
+            span,
+        });
+    }
+
+    Ok(column_info)
+}
+
+fn get_table_constraints(
+    db: &SQLiteDatabase,
+    conn: &Connection,
+    table: &DbTable,
+    span: Span,
+) -> Result<Vec<Value>, ShellError> {
+    let constraints = db.get_constraints(conn, table).map_err(|e| {
+        ShellError::GenericError(
+            "Error getting DB constraints".into(),
+            e.to_string(),
+            Some(span),
+            None,
+            Vec::new(),
+        )
+    })?;
+    let mut constraint_info = vec![];
+    for constraint in constraints {
+        let mut con_cols = vec![];
+        let mut con_vals = vec![];
+        let fields = constraint.fields();
+        let columns = constraint.columns();
+        for (k, v) in fields.iter().zip(columns.iter()) {
+            con_cols.push(k.clone());
+            con_vals.push(Value::string(v.clone(), span));
+        }
+        constraint_info.push(Value::Record {
+            cols: con_cols.clone(),
+            vals: con_vals.clone(),
+            span,
+        });
+    }
+
+    Ok(constraint_info)
+}
+
+fn get_table_foreign_keys(
+    db: &SQLiteDatabase,
+    conn: &Connection,
+    table: &DbTable,
+    span: Span,
+) -> Result<Vec<Value>, ShellError> {
+    let foreign_keys = db.get_foreign_keys(conn, table).map_err(|e| {
+        ShellError::GenericError(
+            "Error getting DB Foreign Keys".into(),
+            e.to_string(),
+            Some(span),
+            None,
+            Vec::new(),
+        )
+    })?;
+    let mut foreign_key_info = vec![];
+    for fk in foreign_keys {
+        let mut fk_cols = vec![];
+        let mut fk_vals = vec![];
+        let fields = fk.fields();
+        let columns = fk.columns();
+        for (k, v) in fields.iter().zip(columns.iter()) {
+            fk_cols.push(k.clone());
+            fk_vals.push(Value::string(v.clone(), span));
+        }
+        foreign_key_info.push(Value::Record {
+            cols: fk_cols.clone(),
+            vals: fk_vals.clone(),
+            span,
+        });
+    }
+
+    Ok(foreign_key_info)
+}
+
+fn get_table_indexes(
+    db: &SQLiteDatabase,
+    conn: &Connection,
+    table: &DbTable,
+    span: Span,
+) -> Result<Vec<Value>, ShellError> {
+    let indexes = db.get_indexes(conn, table).map_err(|e| {
+        ShellError::GenericError(
+            "Error getting DB Indexes".into(),
+            e.to_string(),
+            Some(span),
+            None,
+            Vec::new(),
+        )
+    })?;
+    let mut index_info = vec![];
+    for index in indexes {
+        let mut idx_cols = vec![];
+        let mut idx_vals = vec![];
+        let fields = index.fields();
+        let columns = index.columns();
+        for (k, v) in fields.iter().zip(columns.iter()) {
+            idx_cols.push(k.clone());
+            idx_vals.push(Value::string(v.clone(), span));
+        }
+        index_info.push(Value::Record {
+            cols: idx_cols.clone(),
+            vals: idx_vals.clone(),
+            span,
+        });
+    }
+
+    Ok(index_info)
+}

crates/nu-command/src/database/commands/select.rs

@@ -3,9 +3,10 @@ use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
-    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
+    Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, Span, SyntaxShape,
+    Value,
 };
-use sqlparser::ast::{Query, Select, SelectItem, SetExpr};
+use sqlparser::ast::{Query, Select, SelectItem, SetExpr, Statement};
 
 #[derive(Clone)]
 pub struct ProjectionDb;
@@ -63,17 +64,17 @@ impl Command for ProjectionDb {
         let projection = SelectDb::extract_selects(value)?;
 
         let mut db = SQLiteDatabase::try_from_pipeline(input, call.head)?;
-        db.query = match db.query {
+        db.statement = match db.statement {
-            None => Some(create_query(projection)),
+            None => Some(create_statement(projection)),
-            Some(query) => Some(modify_query(query, projection)),
+            Some(statement) => Some(modify_statement(statement, projection, call.head)?),
         };
 
         Ok(db.into_value(call.head).into_pipeline_data())
     }
 }
 
-fn create_query(expressions: Vec<SelectItem>) -> Query {
+fn create_statement(expressions: Vec<SelectItem>) -> Statement {
-    Query {
+    let query = Query {
         with: None,
         body: SetExpr::Select(Box::new(create_select(expressions))),
         order_by: Vec::new(),
@@ -81,21 +82,35 @@ fn create_query(expressions: Vec<SelectItem>) -> Query {
         offset: None,
         fetch: None,
         lock: None,
-    }
-}
-
-fn modify_query(mut query: Query, expressions: Vec<SelectItem>) -> Query {
-    query.body = match query.body {
-        SetExpr::Select(select) => SetExpr::Select(modify_select(select, expressions)),
-        _ => SetExpr::Select(Box::new(create_select(expressions))),
     };
 
-    query
+    Statement::Query(Box::new(query))
 }
 
-fn modify_select(mut select: Box<Select>, projection: Vec<SelectItem>) -> Box<Select> {
+fn modify_statement(
-    select.as_mut().projection = projection;
+    mut statement: Statement,
-    select
+    expressions: Vec<SelectItem>,
+    span: Span,
+) -> Result<Statement, ShellError> {
+    match statement {
+        Statement::Query(ref mut query) => {
+            match query.body {
+                SetExpr::Select(ref mut select) => select.as_mut().projection = expressions,
+                _ => {
+                    query.as_mut().body = SetExpr::Select(Box::new(create_select(expressions)));
+                }
+            };
+
+            Ok(statement)
+        }
+        s => Err(ShellError::GenericError(
+            "Connection doesnt define a statement".into(),
+            format!("Expected a connection with query. Got {}", s),
+            Some(span),
+            None,
+            Vec::new(),
+        )),
+    }
 }
 
 fn create_select(projection: Vec<SelectItem>) -> Select {

crates/nu-command/src/database/commands/where_.rs

@@ -7,7 +7,7 @@ use nu_protocol::{
     engine::{Command, EngineState, Stack},
     Category, Example, IntoPipelineData, PipelineData, ShellError, Signature, SyntaxShape, Value,
 };
-use sqlparser::ast::{Expr, Query, Select, SetExpr};
+use sqlparser::ast::{Expr, Query, Select, SetExpr, Statement};
 
 #[derive(Clone)]
 pub struct WhereDb;
@@ -54,12 +54,23 @@ impl Command for WhereDb {
         let expr = ExprDb::try_from_value(&value)?.into_native();
 
         let mut db = SQLiteDatabase::try_from_pipeline(input, call.head)?;
-        db.query = match db.query {
+        match db.statement.as_mut() {
-            Some(query) => Some(modify_query(query, expr)),
+            Some(statement) => match statement {
+                Statement::Query(query) => modify_query(query, expr),
+                s => {
+                    return Err(ShellError::GenericError(
+                        "Connection doesnt define a query".into(),
+                        format!("Expected a connection with query. Got {}", s),
+                        Some(call.head),
+                        None,
+                        Vec::new(),
+                    ))
+                }
+            },
             None => {
                 return Err(ShellError::GenericError(
-                    "Connection without query".into(),
+                    "Connection without statement".into(),
-                    "The connection needs a query defined".into(),
+                    "The connection needs a statement defined".into(),
                     Some(call.head),
                     None,
                     Vec::new(),
@@ -71,18 +82,17 @@ impl Command for WhereDb {
     }
 }
 
-fn modify_query(mut query: Query, expression: Expr) -> Query {
+fn modify_query(query: &mut Box<Query>, expression: Expr) {
-    query.body = match query.body {
+    match query.body {
-        SetExpr::Select(select) => SetExpr::Select(modify_select(select, expression)),
+        SetExpr::Select(ref mut select) => modify_select(select, expression),
-        _ => SetExpr::Select(Box::new(create_select(expression))),
+        _ => {
+            query.as_mut().body = SetExpr::Select(Box::new(create_select(expression)));
+        }
     };
-
-    query
 }
 
-fn modify_select(mut select: Box<Select>, expression: Expr) -> Box<Select> {
+fn modify_select(select: &mut Box<Select>, expression: Expr) {
     select.as_mut().selection = Some(expression);
-    select
 }
 
 fn create_select(expression: Expr) -> Select {

crates/nu-command/src/database/expressions/mod.rs

@@ -1,21 +0,0 @@
-mod alias;
-mod col;
-
-use nu_protocol::engine::StateWorkingSet;
-
-use alias::AliasExpr;
-use col::ColExpr;
-
-pub fn add_expression_decls(working_set: &mut StateWorkingSet) {
-    macro_rules! bind_command {
-            ( $command:expr ) => {
-                working_set.add_decl(Box::new($command));
-            };
-            ( $( $command:expr ),* ) => {
-                $( working_set.add_decl(Box::new($command)); )*
-            };
-        }
-
-    // Series commands
-    bind_command!(AliasExpr, ColExpr);
-}

crates/nu-command/src/database/mod.rs

@@ -1,16 +1,8 @@
 mod commands;
 mod values;
 
-mod expressions;
+pub use commands::add_database_decls;
-pub use commands::add_commands_decls;
-pub use expressions::add_expression_decls;
-use nu_protocol::engine::StateWorkingSet;
 pub use values::{
     convert_sqlite_row_to_nu_value, convert_sqlite_value_to_nu_value, open_connection_in_memory,
     SQLiteDatabase,
 };
-
-pub fn add_database_decls(working_set: &mut StateWorkingSet) {
-    add_commands_decls(working_set);
-    add_expression_decls(working_set);
-}

crates/nu-command/src/database/values/definitions/mod.rs

@@ -1,3 +1,7 @@
+use nu_protocol::{ShellError, Span};
+use serde::{Deserialize, Serialize};
+use std::{fmt::Display, path::PathBuf};
+
 pub mod db;
 pub mod db_column;
 pub mod db_constraint;
@@ -6,3 +10,24 @@ pub mod db_index;
 pub mod db_row;
 pub mod db_schema;
 pub mod db_table;
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+pub enum ConnectionDb {
+    Path(PathBuf),
+}
+
+impl Display for ConnectionDb {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Path(path) => write!(f, "{}", path.to_str().unwrap_or("")),
+        }
+    }
+}
+
+impl ConnectionDb {
+    pub fn as_path(&self, _span: Span) -> Result<&PathBuf, ShellError> {
+        match self {
+            Self::Path(path) => Ok(path),
+        }
+    }
+}

crates/nu-command/src/database/values/dsl/expression.rs

@@ -1,6 +1,6 @@
 use nu_protocol::{
     ast::{Operator, PathMember},
-    CustomValue, ShellError, Span, Type, Value,
+    CustomValue, PipelineData, ShellError, Span, Type, Value,
 };
 use serde::{Deserialize, Serialize};
 use sqlparser::ast::{BinaryOperator, Expr, Ident};
@@ -160,6 +160,11 @@ impl ExprDb {
         }
     }
 
+    pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result<Self, ShellError> {
+        let value = input.into_value(span);
+        Self::try_from_value(&value)
+    }
+
     pub fn into_value(self, span: Span) -> Value {
         Value::CustomValue {
             val: Box::new(self),
@@ -273,6 +278,41 @@ impl ExprDb {
 
                 Value::Record { cols, vals, span }
             }
+            Expr::Function(function) => {
+                let cols = vec![
+                    "name".into(),
+                    "args".into(),
+                    "over".into(),
+                    "distinct".into(),
+                ];
+                let name = Value::String {
+                    val: function.name.to_string(),
+                    span,
+                };
+
+                let args: Vec<Value> = function
+                    .args
+                    .iter()
+                    .map(|arg| Value::String {
+                        val: arg.to_string(),
+                        span,
+                    })
+                    .collect();
+                let args = Value::List { vals: args, span };
+
+                let over = Value::String {
+                    val: format!("{:?}", function.over),
+                    span,
+                };
+
+                let distinct = Value::Bool {
+                    val: function.distinct,
+                    span,
+                };
+
+                let vals = vec![name, args, over, distinct];
+                Value::Record { cols, vals, span }
+            }
             Expr::Nested(expr) => ExprDb::expr_to_value(expr, span),
             Expr::CompoundIdentifier(_) => todo!(),
             Expr::IsNull(_) => todo!(),
@@ -292,7 +332,6 @@ impl ExprDb {
             Expr::Collate { .. } => todo!(),
             Expr::TypedString { .. } => todo!(),
             Expr::MapAccess { .. } => todo!(),
-            Expr::Function(_) => todo!(),
             Expr::Case { .. } => todo!(),
             Expr::Exists(_) => todo!(),
             Expr::Subquery(_) => todo!(),

crates/nu-command/src/database/values/sqlite.rs

@@ -1,3 +1,4 @@
+use super::definitions::ConnectionDb;
 use crate::database::values::definitions::{
     db::Db, db_column::DbColumn, db_constraint::DbConstraint, db_foreignkey::DbForeignKey,
     db_index::DbIndex, db_table::DbTable,
@@ -5,7 +6,7 @@ use crate::database::values::definitions::{
 use nu_protocol::{CustomValue, PipelineData, ShellError, Span, Spanned, Value};
 use rusqlite::{types::ValueRef, Connection, Row};
 use serde::{Deserialize, Serialize};
-use sqlparser::ast::Query;
+use sqlparser::ast::Statement;
 use std::{
     fs::File,
     io::Read,
@@ -19,15 +20,15 @@ pub struct SQLiteDatabase {
     // I considered storing a SQLite connection here, but decided against it because
     // 1) YAGNI, 2) it's not obvious how cloning a connection could work, 3) state
     // management gets tricky quick. Revisit this approach if we find a compelling use case.
-    pub path: PathBuf,
+    pub connection: ConnectionDb,
-    pub query: Option<Query>,
+    pub statement: Option<Statement>,
 }
 
 impl SQLiteDatabase {
     pub fn new(path: &Path) -> Self {
         Self {
-            path: PathBuf::from(path),
+            connection: ConnectionDb::Path(PathBuf::from(path)),
-            query: None,
+            statement: None,
         }
     }
 
@@ -51,8 +52,8 @@ impl SQLiteDatabase {
         match value {
             Value::CustomValue { val, span } => match val.as_any().downcast_ref::<Self>() {
                 Some(db) => Ok(Self {
-                    path: db.path.clone(),
+                    connection: db.connection.clone(),
-                    query: db.query.clone(),
+                    statement: db.statement.clone(),
                 }),
                 None => Err(ShellError::CantConvert(
                     "database".into(),
@@ -83,7 +84,7 @@ impl SQLiteDatabase {
     }
 
     pub fn query(&self, sql: &Spanned<String>, call_span: Span) -> Result<Value, ShellError> {
-        let db = open_sqlite_db(&self.path, call_span)?;
+        let db = open_sqlite_db(self.connection.as_path(call_span)?, call_span)?;
         run_sql_query(db, sql).map_err(|e| {
             ShellError::GenericError(
                 "Failed to query SQLite database".into(),
@@ -96,8 +97,8 @@ impl SQLiteDatabase {
     }
 
     pub fn collect(&self, call_span: Span) -> Result<Value, ShellError> {
-        let sql = match &self.query {
+        let sql = match &self.statement {
-            Some(query) => Ok(format!("{}", query)),
+            Some(statement) => Ok(format!("{}", statement)),
             None => Err(ShellError::GenericError(
                 "Error collecting from db".into(),
                 "No query found in connection".into(),
@@ -112,7 +113,7 @@ impl SQLiteDatabase {
             span: call_span,
         };
 
-        let db = open_sqlite_db(&self.path, call_span)?;
+        let db = open_sqlite_db(self.connection.as_path(call_span)?, call_span)?;
         run_sql_query(db, &sql).map_err(|e| {
             ShellError::GenericError(
                 "Failed to query SQLite database".into(),
@@ -127,12 +128,12 @@ impl SQLiteDatabase {
     pub fn describe(&self, span: Span) -> Value {
         let cols = vec!["connection".to_string(), "query".to_string()];
         let connection = Value::String {
-            val: self.path.to_str().unwrap_or("").to_string(),
+            val: self.connection.to_string(),
             span,
         };
 
-        let query = match &self.query {
+        let query = match &self.statement {
-            Some(query) => format!("{query}"),
+            Some(statement) => format!("{statement}"),
             None => "".into(),
         };
 
@@ -146,7 +147,7 @@ impl SQLiteDatabase {
     }
 
     pub fn open_connection(&self) -> Result<Connection, rusqlite::Error> {
-        let conn = match Connection::open(self.path.to_string_lossy().to_string()) {
+        let conn = match Connection::open(self.connection.to_string()) {
             Ok(conn) => conn,
             Err(err) => return Err(err),
         };
@@ -350,8 +351,8 @@ impl SQLiteDatabase {
 impl CustomValue for SQLiteDatabase {
     fn clone_value(&self, span: Span) -> Value {
         let cloned = SQLiteDatabase {
-            path: self.path.clone(),
+            connection: self.connection.clone(),
-            query: self.query.clone(),
+            statement: self.statement.clone(),
         };
 
         Value::CustomValue {
@@ -365,7 +366,7 @@ impl CustomValue for SQLiteDatabase {
     }
 
     fn to_base_value(&self, span: Span) -> Result<Value, ShellError> {
-        let db = open_sqlite_db(&self.path, span)?;
+        let db = open_sqlite_db(self.connection.as_path(span)?, span)?;
         read_entire_sqlite_db(db, span).map_err(|e| {
             ShellError::GenericError(
                 "Failed to read from SQLite database".into(),
@@ -387,7 +388,7 @@ impl CustomValue for SQLiteDatabase {
     }
 
     fn follow_path_string(&self, _column_name: String, span: Span) -> Result<Value, ShellError> {
-        let db = open_sqlite_db(&self.path, span)?;
+        let db = open_sqlite_db(self.connection.as_path(span)?, span)?;
 
         read_single_table(db, _column_name, span).map_err(|e| {
             ShellError::GenericError(

crates/nu-command/src/dataframe/eager/aggregate.rs

@@ -1,403 +0,0 @@
-use nu_engine::CallExt;
-use nu_protocol::{
-    ast::Call,
-    did_you_mean,
-    engine::{Command, EngineState, Stack},
-    Category, Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Value,
-};
-use polars::{
-    frame::groupby::GroupBy,
-    prelude::{PolarsError, QuantileInterpolOptions},
-};
-
-use crate::dataframe::values::NuGroupBy;
-
-use super::super::values::{Column, NuDataFrame};
-
-enum Operation {
-    Mean,
-    Sum,
-    Min,
-    Max,
-    First,
-    Last,
-    Nunique,
-    Quantile(f64),
-    Median,
-    Var,
-    Std,
-    Count,
-}
-
-impl Operation {
-    fn from_tagged(
-        name: &Spanned<String>,
-        quantile: Option<Spanned<f64>>,
-    ) -> Result<Operation, ShellError> {
-        match name.item.as_ref() {
-            "mean" => Ok(Operation::Mean),
-            "sum" => Ok(Operation::Sum),
-            "min" => Ok(Operation::Min),
-            "max" => Ok(Operation::Max),
-            "first" => Ok(Operation::First),
-            "last" => Ok(Operation::Last),
-            "nunique" => Ok(Operation::Nunique),
-            "quantile" => match quantile {
-                None => Err(ShellError::GenericError(
-                    "Quantile value not fount".into(),
-                    "Quantile operation requires quantile value".into(),
-                    Some(name.span),
-                    None,
-                    Vec::new(),
-                )),
-                Some(value) => {
-                    if (value.item < 0.0) | (value.item > 1.0) {
-                        Err(ShellError::GenericError(
-                            "Inappropriate quantile".into(),
-                            "Quantile value should be between 0.0 and 1.0".into(),
-                            Some(value.span),
-                            None,
-                            Vec::new(),
-                        ))
-                    } else {
-                        Ok(Operation::Quantile(value.item))
-                    }
-                }
-            },
-            "median" => Ok(Operation::Median),
-            "var" => Ok(Operation::Var),
-            "std" => Ok(Operation::Std),
-            "count" => Ok(Operation::Count),
-            selection => {
-                let possibilities = [
-                    "mean".to_string(),
-                    "sum".to_string(),
-                    "min".to_string(),
-                    "max".to_string(),
-                    "first".to_string(),
-                    "last".to_string(),
-                    "nunique".to_string(),
-                    "quantile".to_string(),
-                    "median".to_string(),
-                    "var".to_string(),
-                    "std".to_string(),
-                    "count".to_string(),
-                ];
-
-                match did_you_mean(&possibilities, selection) {
-                    Some(suggestion) => Err(ShellError::DidYouMean(suggestion, name.span)),
-                    None => Err(ShellError::GenericError(
-                        "Operation not fount".into(),
-                        "Operation does not exist".into(),
-                        Some(name.span),
-                        Some("Perhaps you want: mean, sum, min, max, first, last, nunique, quantile, median, var, std, or count".into()),
-                        Vec::new(),
-                    ))
-                }
-            }
-        }
-    }
-
-    fn to_str(&self) -> &'static str {
-        match self {
-            Self::Mean => "mean",
-            Self::Sum => "sum",
-            Self::Min => "min",
-            Self::Max => "max",
-            Self::First => "first",
-            Self::Last => "last",
-            Self::Nunique => "nunique",
-            Self::Quantile(_) => "quantile",
-            Self::Median => "median",
-            Self::Var => "var",
-            Self::Std => "std",
-            Self::Count => "count",
-        }
-    }
-}
-
-#[derive(Clone)]
-pub struct Aggregate;
-
-impl Command for Aggregate {
-    fn name(&self) -> &str {
-        "dfr aggregate"
-    }
-
-    fn usage(&self) -> &str {
-        "Performs an aggregation operation on a dataframe and groupby object"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "operation_name",
-                SyntaxShape::String,
-                "\n\tDataframes: mean, sum, min, max, quantile, median, var, std
-\tGroupBy: mean, sum, min, max, first, last, nunique, quantile, median, var, std, count",
-            )
-            .named(
-                "quantile",
-                SyntaxShape::Number,
-                "quantile value for quantile operation",
-                Some('q'),
-            )
-            .switch(
-                "explicit",
-                "returns explicit names for groupby aggregations",
-                Some('e'),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![
-            Example {
-                description: "Aggregate sum by grouping by column a and summing on col b",
-                example:
-                    "[[a b]; [one 1] [one 2]] | dfr to-df | dfr group-by a | dfr aggregate sum",
-                result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new("a".to_string(), vec![Value::test_string("one")]),
-                        Column::new("b".to_string(), vec![Value::test_int(3)]),
-                    ])
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Aggregate sum in dataframe columns",
-                example: "[[a b]; [4 1] [5 2]] | dfr to-df | dfr aggregate sum",
-                result: Some(
-                    NuDataFrame::try_from_columns(vec![
-                        Column::new("a".to_string(), vec![Value::test_int(9)]),
-                        Column::new("b".to_string(), vec![Value::test_int(3)]),
-                    ])
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-            Example {
-                description: "Aggregate sum in series",
-                example: "[4 1 5 6] | dfr to-df | dfr aggregate sum",
-                result: Some(
-                    NuDataFrame::try_from_columns(vec![Column::new(
-                        "0".to_string(),
-                        vec![Value::test_int(16)],
-                    )])
-                    .expect("simple df for test should not fail")
-                    .into_value(Span::test_data()),
-                ),
-            },
-        ]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let operation: Spanned<String> = call.req(engine_state, stack, 0)?;
-    let quantile: Option<Spanned<f64>> = call.get_flag(engine_state, stack, "quantile")?;
-    let op = Operation::from_tagged(&operation, quantile)?;
-
-    match input {
-        PipelineData::Value(Value::CustomValue { val, span }, _) => {
-            let df = val.as_any().downcast_ref::<NuDataFrame>();
-            let groupby = val.as_any().downcast_ref::<NuGroupBy>();
-
-            match (df, groupby) {
-                (Some(df), None) => {
-                    let df = df.as_ref();
-                    let res = perform_dataframe_aggregation(df, op, operation.span)?;
-
-                    Ok(PipelineData::Value(
-                        NuDataFrame::dataframe_into_value(res, span),
-                        None,
-                    ))
-                }
-                (None, Some(nu_groupby)) => {
-                    let groupby = nu_groupby.to_groupby()?;
-
-                    let res = perform_groupby_aggregation(
-                        groupby,
-                        op,
-                        operation.span,
-                        call.head,
-                        call.has_flag("explicit"),
-                    )?;
-
-                    Ok(PipelineData::Value(
-                        NuDataFrame::dataframe_into_value(res, span),
-                        None,
-                    ))
-                }
-                _ => Err(ShellError::GenericError(
-                    "Incorrect datatype".into(),
-                    "no groupby or dataframe found in input stream".into(),
-                    Some(call.head),
-                    None,
-                    Vec::new(),
-                )),
-            }
-        }
-        _ => Err(ShellError::GenericError(
-            "Incorrect datatype".into(),
-            "no groupby or dataframe found in input stream".into(),
-            Some(call.head),
-            None,
-            Vec::new(),
-        )),
-    }
-}
-
-fn perform_groupby_aggregation(
-    groupby: GroupBy,
-    operation: Operation,
-    operation_span: Span,
-    agg_span: Span,
-    explicit: bool,
-) -> Result<polars::prelude::DataFrame, ShellError> {
-    let mut res = match operation {
-        Operation::Mean => groupby.mean(),
-        Operation::Sum => groupby.sum(),
-        Operation::Min => groupby.min(),
-        Operation::Max => groupby.max(),
-        Operation::First => groupby.first(),
-        Operation::Last => groupby.last(),
-        Operation::Nunique => groupby.n_unique(),
-        Operation::Quantile(quantile) => {
-            groupby.quantile(quantile, QuantileInterpolOptions::default())
-        }
-        Operation::Median => groupby.median(),
-        Operation::Var => groupby.var(),
-        Operation::Std => groupby.std(),
-        Operation::Count => groupby.count(),
-    }
-    .map_err(|e| {
-        let span = match &e {
-            PolarsError::NotFound(_) => agg_span,
-            _ => operation_span,
-        };
-
-        ShellError::GenericError(
-            "Error calculating aggregation".into(),
-            e.to_string(),
-            Some(span),
-            None,
-            Vec::new(),
-        )
-    })?;
-
-    if !explicit {
-        let col_names = res
-            .get_column_names()
-            .iter()
-            .map(|name| name.to_string())
-            .collect::<Vec<String>>();
-
-        for col in col_names {
-            let from = match operation {
-                Operation::Mean => "_mean",
-                Operation::Sum => "_sum",
-                Operation::Min => "_min",
-                Operation::Max => "_max",
-                Operation::First => "_first",
-                Operation::Last => "_last",
-                Operation::Nunique => "_n_unique",
-                Operation::Quantile(_) => "_quantile",
-                Operation::Median => "_median",
-                Operation::Var => "_agg_var",
-                Operation::Std => "_agg_std",
-                Operation::Count => "_count",
-            };
-
-            let new_col = match col.find(from) {
-                Some(index) => &col[..index],
-                None => &col[..],
-            };
-
-            res.rename(&col, new_col)
-                .expect("Column is always there. Looping with known names");
-        }
-    }
-
-    Ok(res)
-}
-
-fn perform_dataframe_aggregation(
-    dataframe: &polars::prelude::DataFrame,
-    operation: Operation,
-    operation_span: Span,
-) -> Result<polars::prelude::DataFrame, ShellError> {
-    match operation {
-        Operation::Mean => Ok(dataframe.mean()),
-        Operation::Sum => Ok(dataframe.sum()),
-        Operation::Min => Ok(dataframe.min()),
-        Operation::Max => Ok(dataframe.max()),
-        Operation::Quantile(quantile) => dataframe
-            .quantile(quantile, QuantileInterpolOptions::default())
-            .map_err(|e| {
-                ShellError::GenericError(
-                    "Error calculating quantile".into(),
-                    e.to_string(),
-                    Some(operation_span),
-                    None,
-                    Vec::new(),
-                )
-            }),
-        Operation::Median => Ok(dataframe.median()),
-        Operation::Var => Ok(dataframe.var()),
-        Operation::Std => Ok(dataframe.std()),
-        operation => {
-            let possibilities = [
-                "mean".to_string(),
-                "sum".to_string(),
-                "min".to_string(),
-                "max".to_string(),
-                "quantile".to_string(),
-                "median".to_string(),
-                "var".to_string(),
-                "std".to_string(),
-            ];
-
-            match did_you_mean(&possibilities, operation.to_str()) {
-                Some(suggestion) => Err(ShellError::DidYouMean(suggestion, operation_span)),
-                None => Err(ShellError::GenericError(
-                    "Operation not fount".into(),
-                    "Operation does not exist".into(),
-                    Some(operation_span),
-                    Some(
-                        "Perhaps you want: mean, sum, min, max, quantile, median, var, or std"
-                            .into(),
-                    ),
-                    Vec::new(),
-                )),
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::super::CreateGroupBy;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(Aggregate {}), Box::new(CreateGroupBy {})])
-    }
-}

crates/nu-command/src/dataframe/eager/drop_duplicates.rs

@@ -4,7 +4,7 @@ use nu_protocol::{
     engine::{Command, EngineState, Stack},
     Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
 };
-use polars::prelude::DistinctKeepStrategy;
+use polars::prelude::UniqueKeepStrategy;
 
 use super::super::values::utils::convert_columns_string;
 use super::super::values::{Column, NuDataFrame};
@@ -89,13 +89,13 @@ fn command(
     let subset_slice = subset.as_ref().map(|cols| &cols[..]);
 
     let keep_strategy = if call.has_flag("last") {
-        DistinctKeepStrategy::Last
+        UniqueKeepStrategy::Last
     } else {
-        DistinctKeepStrategy::First
+        UniqueKeepStrategy::First
     };
 
     df.as_ref()
-        .distinct(subset_slice, keep_strategy)
+        .unique(subset_slice, keep_strategy)
         .map_err(|e| {
             ShellError::GenericError(
                 "Error dropping duplicates".into(),

crates/nu-command/src/dataframe/eager/dummies.rs

@@ -1,10 +1,10 @@
+use super::super::values::{Column, NuDataFrame};
 use nu_protocol::{
     ast::Call,
     engine::{Command, EngineState, Stack},
     Category, Example, PipelineData, ShellError, Signature, Span, Value,
 };
-
+use polars::prelude::DataFrameOps;
-use super::super::values::{Column, NuDataFrame};
 
 #[derive(Clone)]
 pub struct Dummies;

crates/nu-command/src/dataframe/eager/filter_with.rs

@@ -4,6 +4,9 @@ use nu_protocol::{
     engine::{Command, EngineState, Stack},
     Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
 };
+use polars::prelude::LazyFrame;
+
+use crate::dataframe::values::{NuExpression, NuLazyFrame};
 
 use super::super::values::{Column, NuDataFrame};
 
@@ -16,12 +19,16 @@ impl Command for FilterWith {
     }
 
     fn usage(&self) -> &str {
-        "Filters dataframe using a mask as reference"
+        "Filters dataframe using a mask or expression as reference"
     }
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
-            .required("mask", SyntaxShape::Any, "boolean mask used to filter data")
+            .required(
+                "mask or expression",
+                SyntaxShape::Any,
+                "boolean mask used to filter data",
+            )
             .category(Category::Custom("dataframe".into()))
     }
 
@@ -48,15 +55,30 @@ impl Command for FilterWith {
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
+        let value = input.into_value(call.head);
+
+        if NuLazyFrame::can_downcast(&value) {
+            let df = NuLazyFrame::try_from_value(value)?;
+            command_lazy(engine_state, stack, call, df)
+        } else if NuDataFrame::can_downcast(&value) {
+            let df = NuDataFrame::try_from_value(value)?;
+            command_eager(engine_state, stack, call, df)
+        } else {
+            Err(ShellError::CantConvert(
+                "expression or query".into(),
+                value.get_type().to_string(),
+                value.span()?,
+                None,
+            ))
+        }
     }
 }
 
-fn command(
+fn command_eager(
     engine_state: &EngineState,
     stack: &mut Stack,
     call: &Call,
-    input: PipelineData,
+    df: NuDataFrame,
 ) -> Result<PipelineData, ShellError> {
     let mask_value: Value = call.req(engine_state, stack, 0)?;
 
@@ -72,8 +94,6 @@ fn command(
         )
     })?;
 
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
     df.as_ref()
         .filter(mask)
         .map_err(|e| {
@@ -88,6 +108,23 @@ fn command(
         .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
 }
 
+fn command_lazy(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+    lazy: NuLazyFrame,
+) -> Result<PipelineData, ShellError> {
+    let expr: Value = call.req(engine_state, stack, 0)?;
+    let expr = NuExpression::try_from_value(expr)?;
+
+    let lazy = lazy.apply_with_expr(expr, LazyFrame::filter);
+
+    Ok(PipelineData::Value(
+        NuLazyFrame::into_value(lazy, call.head),
+        None,
+    ))
+}
+
 #[cfg(test)]
 mod test {
     use super::super::super::test_dataframe::test_dataframe;

crates/nu-command/src/dataframe/eager/first.rs

@@ -1,3 +1,5 @@
+use super::super::values::{utils::DEFAULT_ROWS, Column, NuDataFrame};
+use crate::dataframe::values::NuExpression;
 use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
@@ -5,8 +7,6 @@ use nu_protocol::{
     Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
 };
 
-use super::super::values::{utils::DEFAULT_ROWS, Column, NuDataFrame};
-
 #[derive(Clone)]
 pub struct FirstDF;
 
@@ -16,7 +16,7 @@ impl Command for FirstDF {
     }
 
     fn usage(&self) -> &str {
-        "Creates new dataframe with first rows"
+        "Creates new dataframe with first rows or creates a first expression"
     }
 
     fn signature(&self) -> Signature {
@@ -26,18 +26,25 @@ impl Command for FirstDF {
     }
 
     fn examples(&self) -> Vec<Example> {
-        vec![Example {
+        vec![
-            description: "Create new dataframe with head rows",
+            Example {
-            example: "[[a b]; [1 2] [3 4]] | dfr to-df | dfr first 1",
+                description: "Create new dataframe with head rows",
-            result: Some(
+                example: "[[a b]; [1 2] [3 4]] | dfr to-df | dfr first 1",
-                NuDataFrame::try_from_columns(vec![
+                result: Some(
-                    Column::new("a".to_string(), vec![Value::test_int(1)]),
+                    NuDataFrame::try_from_columns(vec![
-                    Column::new("b".to_string(), vec![Value::test_int(2)]),
+                        Column::new("a".to_string(), vec![Value::test_int(1)]),
-                ])
+                        Column::new("b".to_string(), vec![Value::test_int(2)]),
-                .expect("simple df for test should not fail")
+                    ])
-                .into_value(Span::test_data()),
+                    .expect("simple df for test should not fail")
-            ),
+                    .into_value(Span::test_data()),
-        }]
+                ),
+            },
+            Example {
+                description: "Creates a first expression from a column",
+                example: "dfr col a | dfr first",
+                result: None,
+            },
+        ]
     }
 
     fn run(
@@ -47,7 +54,27 @@ impl Command for FirstDF {
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
+        let value = input.into_value(call.head);
+
+        if NuExpression::can_downcast(&value) {
+            let expr = NuExpression::try_from_value(value)?;
+            let expr: NuExpression = expr.into_polars().is_null().into();
+
+            Ok(PipelineData::Value(
+                NuExpression::into_value(expr, call.head),
+                None,
+            ))
+        } else if NuDataFrame::can_downcast(&value) {
+            let df = NuDataFrame::try_from_value(value)?;
+            command(engine_state, stack, call, df)
+        } else {
+            Err(ShellError::CantConvert(
+                "expression or query".into(),
+                value.get_type().to_string(),
+                value.span()?,
+                None,
+            ))
+        }
     }
 }
 
@@ -55,12 +82,11 @@ fn command(
     engine_state: &EngineState,
     stack: &mut Stack,
     call: &Call,
-    input: PipelineData,
+    df: NuDataFrame,
 ) -> Result<PipelineData, ShellError> {
     let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
     let rows = rows.unwrap_or(DEFAULT_ROWS);
 
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
     let res = df.as_ref().head(Some(rows));
     Ok(PipelineData::Value(
         NuDataFrame::dataframe_into_value(res, call.head),

crates/nu-command/src/dataframe/eager/groupby.rs

@@ -1,77 +0,0 @@
-use nu_engine::CallExt;
-use nu_protocol::{
-    ast::Call,
-    engine::{Command, EngineState, Stack},
-    Category, Example, PipelineData, ShellError, Signature, SyntaxShape, Value,
-};
-
-use super::super::values::{utils::convert_columns_string, NuDataFrame, NuGroupBy};
-
-#[derive(Clone)]
-pub struct CreateGroupBy;
-
-impl Command for CreateGroupBy {
-    fn name(&self) -> &str {
-        "dfr group-by"
-    }
-
-    fn usage(&self) -> &str {
-        "Creates a groupby object that can be used for other aggregations"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .rest("rest", SyntaxShape::Any, "groupby columns")
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Grouping by column a",
-            example: "[[a b]; [one 1] [one 2]] | dfr to-df | dfr group-by a",
-            result: None,
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    // Extracting the names of the columns to perform the groupby
-    let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
-    let (col_string, col_span) = convert_columns_string(columns, call.head)?;
-
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-
-    // This is the expensive part of the groupby; to create the
-    // groups that will be used for grouping the data in the
-    // dataframe. Once it has been done these values can be stored
-    // in a NuGroupBy
-    let groupby = df.as_ref().groupby(&col_string).map_err(|e| {
-        ShellError::GenericError(
-            "Error creating groupby".into(),
-            e.to_string(),
-            Some(col_span),
-            None,
-            Vec::new(),
-        )
-    })?;
-
-    let groups = groupby.get_groups();
-    let groupby = NuGroupBy::new(df.as_ref().clone(), col_string, groups);
-
-    Ok(PipelineData::Value(groupby.into_value(call.head), None))
-}

crates/nu-command/src/dataframe/eager/join.rs

@@ -1,235 +0,0 @@
-use nu_engine::CallExt;
-use nu_protocol::{
-    ast::Call,
-    engine::{Command, EngineState, Stack},
-    Category, Example, PipelineData, ShellError, Signature, Span, Spanned, SyntaxShape, Value,
-};
-use polars::prelude::JoinType;
-
-use crate::dataframe::values::utils::convert_columns_string;
-
-use super::super::values::{Column, NuDataFrame};
-
-#[derive(Clone)]
-pub struct JoinDF;
-
-impl Command for JoinDF {
-    fn name(&self) -> &str {
-        "dfr join"
-    }
-
-    fn usage(&self) -> &str {
-        "Joins a dataframe using columns as reference"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required("dataframe", SyntaxShape::Any, "right dataframe to join")
-            .required_named(
-                "left",
-                SyntaxShape::Table,
-                "left column names to perform join",
-                Some('l'),
-            )
-            .required_named(
-                "right",
-                SyntaxShape::Table,
-                "right column names to perform join",
-                Some('r'),
-            )
-            .named(
-                "type",
-                SyntaxShape::String,
-                "type of join. Inner by default",
-                Some('t'),
-            )
-            .named(
-                "suffix",
-                SyntaxShape::String,
-                "suffix for the columns of the right dataframe",
-                Some('s'),
-            )
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "inner join dataframe",
-            example: r#"let right = ([[a b c]; [1 2 5] [3 4 5] [5 6 6]] | dfr to-df);
-    $right | dfr join $right -l [a b] -r [a b]"#,
-            result: Some(
-                NuDataFrame::try_from_columns(vec![
-                    Column::new(
-                        "a".to_string(),
-                        vec![Value::test_int(1), Value::test_int(3), Value::test_int(5)],
-                    ),
-                    Column::new(
-                        "b".to_string(),
-                        vec![Value::test_int(2), Value::test_int(4), Value::test_int(6)],
-                    ),
-                    Column::new(
-                        "c".to_string(),
-                        vec![Value::test_int(5), Value::test_int(5), Value::test_int(6)],
-                    ),
-                    Column::new(
-                        "c_right".to_string(),
-                        vec![Value::test_int(5), Value::test_int(5), Value::test_int(6)],
-                    ),
-                ])
-                .expect("simple df for test should not fail")
-                .into_value(Span::test_data()),
-            ),
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let r_df: Value = call.req(engine_state, stack, 0)?;
-    let l_col: Vec<Value> = call
-        .get_flag(engine_state, stack, "left")?
-        .expect("required value in syntax");
-    let r_col: Vec<Value> = call
-        .get_flag(engine_state, stack, "right")?
-        .expect("required value in syntax");
-    let suffix: Option<String> = call.get_flag(engine_state, stack, "suffix")?;
-    let join_type_op: Option<Spanned<String>> = call.get_flag(engine_state, stack, "type")?;
-
-    let join_type = match join_type_op {
-        None => JoinType::Inner,
-        Some(val) => match val.item.as_ref() {
-            "inner" => JoinType::Inner,
-            "outer" => JoinType::Outer,
-            "left" => JoinType::Left,
-            _ => {
-                return Err(ShellError::GenericError(
-                    "Incorrect join type".into(),
-                    "Invalid join type".into(),
-                    Some(val.span),
-                    Some("Options: inner, outer or left".into()),
-                    Vec::new(),
-                ))
-            }
-        },
-    };
-
-    let (l_col_string, l_col_span) = convert_columns_string(l_col, call.head)?;
-    let (r_col_string, r_col_span) = convert_columns_string(r_col, call.head)?;
-
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
-    let r_df = NuDataFrame::try_from_value(r_df)?;
-
-    check_column_datatypes(
-        df.as_ref(),
-        r_df.as_ref(),
-        &l_col_string,
-        l_col_span,
-        &r_col_string,
-        r_col_span,
-    )?;
-
-    df.as_ref()
-        .join(
-            r_df.as_ref(),
-            &l_col_string,
-            &r_col_string,
-            join_type,
-            suffix,
-        )
-        .map_err(|e| {
-            ShellError::GenericError(
-                "Error joining dataframes".into(),
-                e.to_string(),
-                Some(l_col_span),
-                None,
-                Vec::new(),
-            )
-        })
-        .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-}
-
-fn check_column_datatypes<T: AsRef<str>>(
-    df_l: &polars::prelude::DataFrame,
-    df_r: &polars::prelude::DataFrame,
-    l_cols: &[T],
-    l_col_span: Span,
-    r_cols: &[T],
-    r_col_span: Span,
-) -> Result<(), ShellError> {
-    if l_cols.len() != r_cols.len() {
-        return Err(ShellError::GenericError(
-            "Mismatched number of column names".into(),
-            format!(
-                "found {} left names vs {} right names",
-                l_cols.len(),
-                r_cols.len()
-            ),
-            Some(l_col_span),
-            Some("perhaps you need to change the number of columns to join".into()),
-            Vec::new(),
-        ));
-    }
-
-    for (l, r) in l_cols.iter().zip(r_cols) {
-        let l_series = df_l.column(l.as_ref()).map_err(|e| {
-            ShellError::GenericError(
-                "Error selecting the columns".into(),
-                e.to_string(),
-                Some(l_col_span),
-                None,
-                Vec::new(),
-            )
-        })?;
-
-        let r_series = df_r.column(r.as_ref()).map_err(|e| {
-            ShellError::GenericError(
-                "Error selecting the columns".into(),
-                e.to_string(),
-                Some(r_col_span),
-                None,
-                Vec::new(),
-            )
-        })?;
-
-        if l_series.dtype() != r_series.dtype() {
-            return Err(ShellError::GenericError(
-                "Mismatched datatypes".into(),
-                format!(
-                    "left column type '{}' doesn't match '{}' right column match",
-                    l_series.dtype(),
-                    r_series.dtype()
-                ),
-                Some(l_col_span),
-                Some("perhaps you need to select other column to match".into()),
-                Vec::new(),
-            ));
-        }
-    }
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod test {
-    use super::super::super::test_dataframe::test_dataframe;
-    use super::*;
-
-    #[test]
-    fn test_examples() {
-        test_dataframe(vec![Box::new(JoinDF {})])
-    }
-}

crates/nu-command/src/dataframe/eager/last.rs

@@ -1,3 +1,5 @@
+use super::super::values::{utils::DEFAULT_ROWS, Column, NuDataFrame};
+use crate::dataframe::values::NuExpression;
 use nu_engine::CallExt;
 use nu_protocol::{
     ast::Call,
@@ -5,8 +7,6 @@ use nu_protocol::{
     Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
 };
 
-use super::super::values::{utils::DEFAULT_ROWS, Column, NuDataFrame};
-
 #[derive(Clone)]
 pub struct LastDF;
 
@@ -16,7 +16,7 @@ impl Command for LastDF {
     }
 
     fn usage(&self) -> &str {
-        "Creates new dataframe with tail rows"
+        "Creates new dataframe with tail rows or creates a last expression"
     }
 
     fn signature(&self) -> Signature {
@@ -26,18 +26,25 @@ impl Command for LastDF {
     }
 
     fn examples(&self) -> Vec<Example> {
-        vec![Example {
+        vec![
-            description: "Create new dataframe with last rows",
+            Example {
-            example: "[[a b]; [1 2] [3 4]] | dfr to-df | dfr last 1",
+                description: "Create new dataframe with last rows",
-            result: Some(
+                example: "[[a b]; [1 2] [3 4]] | dfr to-df | dfr last 1",
-                NuDataFrame::try_from_columns(vec![
+                result: Some(
-                    Column::new("a".to_string(), vec![Value::test_int(3)]),
+                    NuDataFrame::try_from_columns(vec![
-                    Column::new("b".to_string(), vec![Value::test_int(4)]),
+                        Column::new("a".to_string(), vec![Value::test_int(3)]),
-                ])
+                        Column::new("b".to_string(), vec![Value::test_int(4)]),
-                .expect("simple df for test should not fail")
+                    ])
-                .into_value(Span::test_data()),
+                    .expect("simple df for test should not fail")
-            ),
+                    .into_value(Span::test_data()),
-        }]
+                ),
+            },
+            Example {
+                description: "Creates a last expression from a column",
+                example: "dfr col a | dfr last",
+                result: None,
+            },
+        ]
     }
 
     fn run(
@@ -47,7 +54,27 @@ impl Command for LastDF {
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
+        let value = input.into_value(call.head);
+
+        if NuExpression::can_downcast(&value) {
+            let expr = NuExpression::try_from_value(value)?;
+            let expr: NuExpression = expr.into_polars().is_null().into();
+
+            Ok(PipelineData::Value(
+                NuExpression::into_value(expr, call.head),
+                None,
+            ))
+        } else if NuDataFrame::can_downcast(&value) {
+            let df = NuDataFrame::try_from_value(value)?;
+            command(engine_state, stack, call, df)
+        } else {
+            Err(ShellError::CantConvert(
+                "expression or query".into(),
+                value.get_type().to_string(),
+                value.span()?,
+                None,
+            ))
+        }
     }
 }
 
@@ -55,12 +82,11 @@ fn command(
     engine_state: &EngineState,
     stack: &mut Stack,
     call: &Call,
-    input: PipelineData,
+    df: NuDataFrame,
 ) -> Result<PipelineData, ShellError> {
     let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
     let rows = rows.unwrap_or(DEFAULT_ROWS);
 
-    let df = NuDataFrame::try_from_pipeline(input, call.head)?;
     let res = df.as_ref().tail(Some(rows));
     Ok(PipelineData::Value(
         NuDataFrame::dataframe_into_value(res, call.head),

crates/nu-command/src/dataframe/eager/list.rs

@@ -11,7 +11,7 @@ pub struct ListDF;
 
 impl Command for ListDF {
     fn name(&self) -> &str {
-        "dfr list"
+        "dfr ls"
     }
 
     fn usage(&self) -> &str {
@@ -26,7 +26,7 @@ impl Command for ListDF {
         vec![Example {
             description: "Creates a new dataframe and shows it in the dataframe list",
             example: r#"let test = ([[a b];[1 2] [3 4]] | dfr to-df);
-    dfr list"#,
+    dfr ls"#,
             result: None,
         }]
     }
@@ -38,25 +38,19 @@ impl Command for ListDF {
         call: &Call,
         _input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        let vals = engine_state
+        let mut vals: Vec<(String, Value)> = vec![];
-            .scope
+
-            .iter()
+        for overlay_frame in engine_state.active_overlays(&[]) {
-            .flat_map(|frame| {
+            for var in &overlay_frame.vars {
-                frame
+                if let Ok(value) = stack.get_var(*var.1, call.head) {
-                    .vars
+                    let name = String::from_utf8_lossy(var.0).to_string();
-                    .iter()
+                    vals.push((name, value));
-                    .filter_map(|var| {
+                }
-                        let value = stack.get_var(*var.1, call.head);
+            }
-                        match value {
+        }
-                            Ok(value) => {
+
-                                let name = String::from_utf8_lossy(var.0).to_string();
+        let vals = vals
-                                Some((name, value))
+            .into_iter()
-                            }
-                            Err(_) => None,
-                        }
-                    })
-                    .collect::<Vec<(String, Value)>>()
-            })
             .filter_map(|(name, value)| match NuDataFrame::try_from_value(value) {
                 Ok(df) => Some((name, df)),
                 Err(_) => None,

crates/nu-command/src/dataframe/eager/mod.rs

@@ -1,4 +1,3 @@
-mod aggregate;
 mod append;
 mod column;
 mod command;
@@ -11,13 +10,10 @@ mod dummies;
 mod filter_with;
 mod first;
 mod get;
-mod groupby;
-mod join;
 mod last;
 mod list;
 mod melt;
 mod open;
-mod pivot;
 mod rename;
 mod sample;
 mod shape;
@@ -32,7 +28,6 @@ mod with_column;
 
 use nu_protocol::engine::StateWorkingSet;
 
-pub use aggregate::Aggregate;
 pub use append::AppendDF;
 pub use column::ColumnDF;
 pub use command::Dataframe;
@@ -45,13 +40,10 @@ pub use dummies::Dummies;
 pub use filter_with::FilterWith;
 pub use first::FirstDF;
 pub use get::GetDF;
-pub use groupby::CreateGroupBy;
-pub use join::JoinDF;
 pub use last::LastDF;
 pub use list::ListDF;
 pub use melt::MeltDF;
 pub use open::OpenDataFrame;
-pub use pivot::PivotDF;
 pub use rename::RenameDF;
 pub use sample::SampleDF;
 pub use shape::ShapeDF;
@@ -76,10 +68,8 @@ pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
 
     // Dataframe commands
     bind_command!(
-        Aggregate,
         AppendDF,
         ColumnDF,
-        CreateGroupBy,
         Dataframe,
         DataTypes,
         DescribeDF,
@@ -90,12 +80,10 @@ pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
         FilterWith,
         FirstDF,
         GetDF,
-        JoinDF,
         LastDF,
         ListDF,
         MeltDF,
         OpenDataFrame,
-        PivotDF,
         RenameDF,
         SampleDF,
         ShapeDF,

crates/nu-command/src/dataframe/eager/pivot.rs

@@ -1,198 +0,0 @@
-use nu_engine::CallExt;
-use nu_protocol::{
-    ast::Call,
-    engine::{Command, EngineState, Stack},
-    Category, Example, PipelineData, ShellError, Signature, Spanned, SyntaxShape,
-};
-use polars::prelude::DataType;
-
-use crate::dataframe::values::NuGroupBy;
-
-use super::super::values::NuDataFrame;
-
-enum Operation {
-    First,
-    Sum,
-    Min,
-    Max,
-    Mean,
-    Median,
-}
-
-impl Operation {
-    fn from_tagged(name: Spanned<String>) -> Result<Operation, ShellError> {
-        match name.item.as_ref() {
-            "first" => Ok(Operation::First),
-            "sum" => Ok(Operation::Sum),
-            "min" => Ok(Operation::Min),
-            "max" => Ok(Operation::Max),
-            "mean" => Ok(Operation::Mean),
-            "median" => Ok(Operation::Median),
-            _ => Err(ShellError::GenericError(
-                "Operation not fount".into(),
-                "Operation does not exist for pivot".into(),
-                Some(name.span),
-                Some("Options: first, sum, min, max, mean, median".into()),
-                Vec::new(),
-            )),
-        }
-    }
-}
-
-#[derive(Clone)]
-pub struct PivotDF;
-
-impl Command for PivotDF {
-    fn name(&self) -> &str {
-        "dfr pivot"
-    }
-
-    fn usage(&self) -> &str {
-        "Performs a pivot operation on a groupby object"
-    }
-
-    fn signature(&self) -> Signature {
-        Signature::build(self.name())
-            .required(
-                "pivot_column",
-                SyntaxShape::String,
-                "pivot column to perform pivot",
-            )
-            .required(
-                "value_column",
-                SyntaxShape::String,
-                "value column to perform pivot",
-            )
-            .required("operation", SyntaxShape::String, "aggregate operation")
-            .category(Category::Custom("dataframe".into()))
-    }
-
-    fn examples(&self) -> Vec<Example> {
-        vec![Example {
-            description: "Pivot a dataframe on b and aggregation on col c",
-            example:
-                "[[a b c]; [one x 1] [two y 2]] | dfr to-df | dfr group-by a | dfr pivot b c sum",
-            result: None,
-        }]
-    }
-
-    fn run(
-        &self,
-        engine_state: &EngineState,
-        stack: &mut Stack,
-        call: &Call,
-        input: PipelineData,
-    ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
-    }
-}
-
-fn command(
-    engine_state: &EngineState,
-    stack: &mut Stack,
-    call: &Call,
-    input: PipelineData,
-) -> Result<PipelineData, ShellError> {
-    let pivot_col: Spanned<String> = call.req(engine_state, stack, 0)?;
-    let value_col: Spanned<String> = call.req(engine_state, stack, 1)?;
-    let operation: Spanned<String> = call.req(engine_state, stack, 2)?;
-    let op = Operation::from_tagged(operation)?;
-
-    let nu_groupby = NuGroupBy::try_from_pipeline(input, call.head)?;
-    let df_ref = nu_groupby.as_ref();
-
-    check_pivot_column(df_ref, &pivot_col)?;
-    check_value_column(df_ref, &value_col)?;
-
-    let mut groupby = nu_groupby.to_groupby()?;
-
-    let pivot = groupby.pivot(vec![&pivot_col.item], vec![&value_col.item]);
-
-    match op {
-        Operation::Mean => pivot.mean(),
-        Operation::Sum => pivot.sum(),
-        Operation::Min => pivot.min(),
-        Operation::Max => pivot.max(),
-        Operation::First => pivot.first(),
-        Operation::Median => pivot.median(),
-    }
-    .map_err(|e| {
-        ShellError::GenericError(
-            "Error creating pivot".into(),
-            e.to_string(),
-            Some(call.head),
-            None,
-            Vec::new(),
-        )
-    })
-    .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
-}
-
-fn check_pivot_column(
-    df: &polars::prelude::DataFrame,
-    col: &Spanned<String>,
-) -> Result<(), ShellError> {
-    let series = df.column(&col.item).map_err(|e| {
-        ShellError::GenericError(
-            "Column not found".into(),
-            e.to_string(),
-            Some(col.span),
-            None,
-            Vec::new(),
-        )
-    })?;
-
-    match series.dtype() {
-        DataType::UInt8
-        | DataType::UInt16
-        | DataType::UInt32
-        | DataType::UInt64
-        | DataType::Int8
-        | DataType::Int16
-        | DataType::Int32
-        | DataType::Int64
-        | DataType::Utf8 => Ok(()),
-        _ => Err(ShellError::GenericError(
-            "Pivot error".into(),
-            format!("Unsupported datatype {}", series.dtype()),
-            Some(col.span),
-            None,
-            Vec::new(),
-        )),
-    }
-}
-
-fn check_value_column(
-    df: &polars::prelude::DataFrame,
-    col: &Spanned<String>,
-) -> Result<(), ShellError> {
-    let series = df.column(&col.item).map_err(|e| {
-        ShellError::GenericError(
-            "Column not found".into(),
-            e.to_string(),
-            Some(col.span),
-            None,
-            Vec::new(),
-        )
-    })?;
-
-    match series.dtype() {
-        DataType::UInt8
-        | DataType::UInt16
-        | DataType::UInt32
-        | DataType::UInt64
-        | DataType::Int8
-        | DataType::Int16
-        | DataType::Int32
-        | DataType::Int64
-        | DataType::Float32
-        | DataType::Float64 => Ok(()),
-        _ => Err(ShellError::GenericError(
-            "Pivot error".into(),
-            format!("Unsupported datatype {}", series.dtype()),
-            Some(col.span),
-            None,
-            Vec::new(),
-        )),
-    }
-}

crates/nu-command/src/dataframe/eager/rename.rs

@@ -5,6 +5,8 @@ use nu_protocol::{
     Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value,
 };
 
+use crate::dataframe::{utils::extract_strings, values::NuLazyFrame};
+
 use super::super::values::{Column, NuDataFrame};
 
 #[derive(Clone)]
@@ -21,8 +23,16 @@ impl Command for RenameDF {
 
     fn signature(&self) -> Signature {
         Signature::build(self.name())
-            .required("from", SyntaxShape::String, "column name to be renamed")
+            .required(
-            .required("to", SyntaxShape::String, "new column name")
+                "columns",
+                SyntaxShape::Any,
+                "Column(s) to be renamed. A string or list of strings",
+            )
+            .required(
+                "new names",
+                SyntaxShape::Any,
+                "New names for the selected column(s). A string or list of strings",
+            )
             .category(Category::Custom("dataframe".into()))
     }
 
@@ -54,24 +64,39 @@ impl Command for RenameDF {
         call: &Call,
         input: PipelineData,
     ) -> Result<PipelineData, ShellError> {
-        command(engine_state, stack, call, input)
+        let value = input.into_value(call.head);
+
+        if NuLazyFrame::can_downcast(&value) {
+            let df = NuLazyFrame::try_from_value(value)?;
+            command_lazy(engine_state, stack, call, df)
+        } else if NuDataFrame::can_downcast(&value) {
+            let df = NuDataFrame::try_from_value(value)?;
+            command_eager(engine_state, stack, call, df)
+        } else {
+            Err(ShellError::CantConvert(
+                "expression or query".into(),
+                value.get_type().to_string(),
+                value.span()?,
+                None,
+            ))
+        }
     }
 }
 
-fn command(
+fn command_eager(
     engine_state: &EngineState,
     stack: &mut Stack,
     call: &Call,
-    input: PipelineData,
+    mut df: NuDataFrame,
 ) -> Result<PipelineData, ShellError> {
-    let from: String = call.req(engine_state, stack, 0)?;
+    let columns: Value = call.req(engine_state, stack, 0)?;
-    let to: String = call.req(engine_state, stack, 1)?;
+    let columns = extract_strings(columns)?;
 
-    let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
+    let new_names: Value = call.req(engine_state, stack, 1)?;
+    let new_names = extract_strings(new_names)?;
 
-    df.as_mut()
+    for (from, to) in columns.iter().zip(new_names.iter()) {
-        .rename(&from, &to)
+        df.as_mut().rename(from, to).map_err(|e| {
-        .map_err(|e| {
             ShellError::GenericError(
                 "Error renaming".into(),
                 e.to_string(),
@@ -79,13 +104,36 @@ fn command(
                 None,
                 Vec::new(),
             )
-        })
+        })?;
-        .map(|df| {
+    }
-            PipelineData::Value(
+
-                NuDataFrame::dataframe_into_value(df.clone(), call.head),
+    Ok(PipelineData::Value(df.into_value(call.head), None))
-                None,
+}
-            )
+
-        })
+fn command_lazy(
+    engine_state: &EngineState,
+    stack: &mut Stack,
+    call: &Call,
+    lazy: NuLazyFrame,
+) -> Result<PipelineData, ShellError> {
+    let columns: Value = call.req(engine_state, stack, 0)?;
+    let columns = extract_strings(columns)?;
+
+    let new_names: Value = call.req(engine_state, stack, 1)?;
+    let new_names = extract_strings(new_names)?;
+
+    if columns.len() != new_names.len() {
+        let value: Value = call.req(engine_state, stack, 1)?;
+        return Err(ShellError::IncompatibleParametersSingle(
+            "New name list has different size to column list".into(),
+            value.span()?,
+        ));
+    }
+
+    let lazy = lazy.into_polars();
+    let lazy: NuLazyFrame = lazy.rename(&columns, &new_names).into();
+
+    Ok(PipelineData::Value(lazy.into_value(call.head), None))
 }
 
 #[cfg(test)]

crates/nu-command/src/dataframe/eager/sample.rs

@@ -33,6 +33,12 @@ impl Command for SampleDF {
                 "fraction of dataframe to be taken",
                 Some('f'),
             )
+            .named(
+                "seed",
+                SyntaxShape::Number,
+                "seed for the selection",
+                Some('s'),
+            )
             .switch("replace", "sample with replace", Some('e'))
             .category(Category::Custom("dataframe".into()))
     }
@@ -71,12 +77,15 @@ fn command(
 ) -> Result<PipelineData, ShellError> {
     let rows: Option<Spanned<usize>> = call.get_flag(engine_state, stack, "n-rows")?;
     let fraction: Option<Spanned<f64>> = call.get_flag(engine_state, stack, "fraction")?;
+    let seed: Option<u64> = call
+        .get_flag::<i64>(engine_state, stack, "seed")?
+        .map(|val| val as u64);
     let replace: bool = call.has_flag("replace");
 
     let df = NuDataFrame::try_from_pipeline(input, call.head)?;
 
     match (rows, fraction) {
-        (Some(rows), None) => df.as_ref().sample_n(rows.item, replace, 0).map_err(|e| {
+        (Some(rows), None) => df.as_ref().sample_n(rows.item, replace, seed).map_err(|e| {
             ShellError::GenericError(
                 "Error creating sample".into(),
                 e.to_string(),
@@ -85,15 +94,18 @@ fn command(
                 Vec::new(),
             )
         }),
-        (None, Some(frac)) => df.as_ref().sample_frac(frac.item, replace, 0).map_err(|e| {
+        (None, Some(frac)) => df
-            ShellError::GenericError(
+            .as_ref()
-                "Error creating sample".into(),
+            .sample_frac(frac.item, replace, seed)
-                e.to_string(),
+            .map_err(|e| {
-                Some(frac.span),
+                ShellError::GenericError(
-                None,
+                    "Error creating sample".into(),
-                Vec::new(),
+                    e.to_string(),
-            )
+                    Some(frac.span),
-        }),
+                    None,
+                    Vec::new(),
+                )
+            }),
         (Some(_), Some(_)) => Err(ShellError::GenericError(
             "Incompatible flags".into(),
             "Only one selection criterion allowed".into(),