extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2025-04-22 12:18:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into ecow-record

Browse Source
This commit is contained in:
Ian Manske 2025-02-22 16:19:38 -08:00
commit 5556b5416c
1593 changed files with 92154 additions and 60759 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View File

@ -13,7 +13,7 @@ body:
id: repro
attributes:
label: How to reproduce
description: Steps to reproduce the behavior
description: Steps to reproduce the behavior (including succinct code examples or screenshots of the observed behavior)
placeholder: |
1.
2.
@ -28,13 +28,6 @@ body:
placeholder: I expected nu to...
validations:
required: true
- type: textarea
id: screenshots
attributes:
label: Screenshots
description: Please add any relevant screenshots here, if any
validations:
required: false
- type: textarea
id: config
attributes:
@ -55,10 +48,3 @@ body:
| installed_plugins | binaryview, chart bar, chart line, fetch, from bson, from sqlite, inc, match, post, ps, query json, s3, selector, start, sys, textview, to bson, to sqlite, tree, xpath |
validations:
required: true
- type: textarea
id: context
attributes:
label: Additional context
description: Add any other context about the problem here.
validations:
required: false

15
.github/dependabot.yml vendored
View File

@ -18,6 +18,21 @@ updates:
ignore:
- dependency-name: "*"
update-types: ["version-update:semver-patch"]
groups:
# Only update polars as a whole as there are many subcrates that need to
# be updated at once. We explicitly depend on some of them, so batch their
# updates to not take up dependabot PR slots with dysfunctional PRs
polars:
patterns:
- "polars"
- "polars-*"
# uutils/coreutils also versions all their workspace crates the same at the moment
# Most of them have bleeding edge version requirements (some not)
# see: https://github.com/uutils/coreutils/blob/main/Cargo.toml
uutils:
patterns:
- "uucore"
- "uu_*"
- package-ecosystem: "github-actions"
directory: "/"
schedule:

4
.github/workflows/audit.yml vendored
View File

@ -19,7 +19,7 @@ jobs:
# Prevent sudden announcement of a new advisory from failing ci:
continue-on-error: true
steps:
- uses: actions/checkout@v4.1.5
- uses: rustsec/audit-check@v1.4.1
- uses: actions/checkout@v4.1.7
- uses: rustsec/audit-check@v2.0.0
with:
token: ${{ secrets.GITHUB_TOKEN }}

121
.github/workflows/ci.yml vendored
View File

@ -3,6 +3,7 @@ on:
push:
branches:
- main
- 'patch-release-*'
name: continuous-integration
@ -21,84 +22,53 @@ jobs:
strategy:
fail-fast: true
matrix:
# Pinning to Ubuntu 20.04 because building on newer Ubuntu versions causes linux-gnu
# Pinning to Ubuntu 22.04 because building on newer Ubuntu versions causes linux-gnu
# builds to link against a too-new-for-many-Linux-installs glibc version. Consider
# revisiting this when 20.04 is closer to EOL (April 2025)
# revisiting this when 22.04 is closer to EOL (June 2027)
#
# Using macOS 13 runner because 14 is based on the M1 and has half as much RAM (7 GB,
# instead of 14 GB) which is too little for us right now. Revisit when `dfr` commands are
# removed and we're only building the `polars` plugin instead
platform: [windows-latest, macos-13, ubuntu-20.04]
feature: [default, dataframe]
include:
- feature: default
flags: ""
- feature: dataframe
flags: "--features=dataframe"
exclude:
- platform: windows-latest
feature: dataframe
- platform: macos-13
feature: dataframe
platform: [windows-latest, macos-13, ubuntu-22.04]
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4.1.5
- uses: actions/checkout@v4.1.7
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
- name: cargo fmt
run: cargo fmt --all -- --check
# If changing these settings also change toolkit.nu
- name: Clippy
run: cargo clippy --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- $CLIPPY_OPTIONS
run: cargo clippy --workspace --exclude nu_plugin_* -- $CLIPPY_OPTIONS
# In tests we don't have to deny unwrap
- name: Clippy of tests
run: cargo clippy --tests --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
run: cargo clippy --tests --workspace --exclude nu_plugin_* -- -D warnings
- name: Clippy of benchmarks
run: cargo clippy --benches --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
run: cargo clippy --benches --workspace --exclude nu_plugin_* -- -D warnings
tests:
strategy:
fail-fast: true
matrix:
platform: [windows-latest, macos-latest, ubuntu-20.04]
feature: [default, dataframe]
include:
# linux CI cannot handle clipboard feature
- default-flags: ""
- platform: ubuntu-20.04
default-flags: "--no-default-features --features=default-no-clipboard"
- feature: default
flags: ""
- feature: dataframe
flags: "--features=dataframe"
exclude:
- platform: windows-latest
feature: dataframe
- platform: macos-latest
feature: dataframe
platform: [windows-latest, macos-latest, ubuntu-22.04]
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4.1.5
- uses: actions/checkout@v4.1.7
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
- name: Tests
run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} ${{ matrix.flags }}
run: cargo test --workspace --profile ci --exclude nu_plugin_*
- name: Check for clean repo
shell: bash
run: |
@ -114,22 +84,20 @@ jobs:
strategy:
fail-fast: true
matrix:
platform: [ubuntu-20.04, macos-latest, windows-latest]
platform: [ubuntu-22.04, macos-latest, windows-latest]
py:
- py
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4.1.5
- uses: actions/checkout@v4.1.7
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
- name: Install Nushell
run: cargo install --path . --locked --no-default-features
run: cargo install --path . --locked --force
- name: Standard library tests
run: nu -c 'use crates/nu-std/testing.nu; testing run-tests --path crates/nu-std'
@ -169,17 +137,15 @@ jobs:
# instead of 14 GB) which is too little for us right now.
#
# Failure occurring with clippy for rust 1.77.2
platform: [windows-latest, macos-13, ubuntu-20.04]
platform: [windows-latest, macos-13, ubuntu-22.04]
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4.1.5
- uses: actions/checkout@v4.1.7
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
- name: Clippy
run: cargo clippy --package nu_plugin_* -- $CLIPPY_OPTIONS
@ -197,3 +163,50 @@ jobs:
else
echo "no changes in working directory";
fi
wasm:
env:
WASM_OPTIONS: --no-default-features --target wasm32-unknown-unknown
CLIPPY_CONF_DIR: ${{ github.workspace }}/clippy/wasm/
strategy:
matrix:
job:
- name: Build WASM
command: cargo build
args:
- name: Clippy WASM
command: cargo clippy
args: -- $CLIPPY_OPTIONS
name: ${{ matrix.job.name }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.7
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
- name: Add wasm32-unknown-unknown target
run: rustup target add wasm32-unknown-unknown
- run: ${{ matrix.job.command }} -p nu-cmd-base $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-cmd-extra $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-cmd-lang $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-color-config $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-command $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-derive-value $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-engine $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-glob $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-json $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-parser $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-path $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-pretty-hex $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-protocol $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-std $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-system $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-table $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-term-grid $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nu-utils $WASM_OPTIONS ${{ matrix.job.args }}
- run: ${{ matrix.job.command }} -p nuon $WASM_OPTIONS ${{ matrix.job.args }}

30
.github/workflows/milestone.yml vendored Normal file
View File

@ -0,0 +1,30 @@
# Description:
# - Add milestone to a merged PR automatically
# - Add milestone to a closed issue that has a merged PR fix (if any)
name: Milestone Action
on:
issues:
types: [closed]
pull_request_target:
types: [closed]
jobs:
update-milestone:
runs-on: ubuntu-latest
name: Milestone Update
steps:
- name: Set Milestone for PR
uses: hustcer/milestone-action@main
if: github.event.pull_request.merged == true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Bind milestone to closed issue that has a merged PR fix
- name: Set Milestone for Issue
uses: hustcer/milestone-action@v2
if: github.event.issue.state == 'closed'
with:
action: bind-issue
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

174
.github/workflows/nightly-build.yml vendored
View File

@ -27,7 +27,7 @@ jobs:
# if: github.repository == 'nushell/nightly'
steps:
- name: Checkout
uses: actions/checkout@v4.1.5
uses: actions/checkout@v4
if: github.repository == 'nushell/nightly'
with:
ref: main
@ -36,10 +36,10 @@ jobs:
token: ${{ secrets.WORKFLOW_TOKEN }}
- name: Setup Nushell
uses: hustcer/setup-nu@v3.10
uses: hustcer/setup-nu@v3
if: github.repository == 'nushell/nightly'
with:
version: 0.93.0
version: 0.101.0
# Synchronize the main branch of nightly repo with the main branch of Nushell official repo
- name: Prepare for Nightly Release
@ -65,7 +65,7 @@ jobs:
}
standard:
name: Std
name: Nu
needs: prepare
strategy:
fail-fast: false
@ -78,52 +78,50 @@ jobs:
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- aarch64-unknown-linux-gnu
- aarch64-unknown-linux-musl
- armv7-unknown-linux-gnueabihf
- armv7-unknown-linux-musleabihf
- riscv64gc-unknown-linux-gnu
- loongarch64-unknown-linux-gnu
extra: ['bin']
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: x86_64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: ''
os: ubuntu-22.04
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
target_rustflags: ''
os: ubuntu-22.04
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: ''
os: ubuntu-22.04
- target: aarch64-unknown-linux-musl
os: ubuntu-22.04
- target: armv7-unknown-linux-gnueabihf
os: ubuntu-20.04
target_rustflags: ''
os: ubuntu-22.04
- target: armv7-unknown-linux-musleabihf
os: ubuntu-22.04
- target: riscv64gc-unknown-linux-gnu
os: ubuntu-latest
target_rustflags: ''
os: ubuntu-22.04
- target: loongarch64-unknown-linux-gnu
os: ubuntu-22.04
runs-on: ${{matrix.os}}
steps:
- uses: actions/checkout@v4.1.5
- uses: actions/checkout@v4
with:
ref: main
fetch-depth: 0
@ -133,26 +131,24 @@ jobs:
echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
with:
rustflags: ''
- name: Setup Nushell
uses: hustcer/setup-nu@v3.10
uses: hustcer/setup-nu@v3
with:
version: 0.93.0
version: 0.101.0
- name: Release Nu Binary
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: standard
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
- name: Create an Issue for Release Failure
if: ${{ failure() }}
@ -174,7 +170,7 @@ jobs:
# REF: https://github.com/marketplace/actions/gh-release
# Create a release only in nushell/nightly repo
- name: Publish Archive
uses: softprops/action-gh-release@v2.0.5
uses: softprops/action-gh-release@v2.0.9
if: ${{ startsWith(github.repository, 'nushell/nightly') }}
with:
prerelease: true
@ -184,122 +180,6 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
full:
name: Full
needs: prepare
strategy:
fail-fast: false
matrix:
target:
- aarch64-apple-darwin
- x86_64-apple-darwin
- x86_64-pc-windows-msvc
- aarch64-pc-windows-msvc
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- aarch64-unknown-linux-gnu
extra: ['bin']
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
runs-on: ${{matrix.os}}
steps:
- uses: actions/checkout@v4.1.5
with:
ref: main
fetch-depth: 0
- name: Update Rust Toolchain Target
run: |
echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
with:
rustflags: ''
- name: Setup Nushell
uses: hustcer/setup-nu@v3.10
with:
version: 0.93.0
- name: Release Nu Binary
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: full
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
- name: Create an Issue for Release Failure
if: ${{ failure() }}
uses: JasonEtco/create-an-issue@v2.9.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
update_existing: true
search_existing: open
filename: .github/AUTO_ISSUE_TEMPLATE/nightly-build-fail.md
- name: Set Outputs of Short SHA
id: vars
run: |
echo "date=$(date -u +'%Y-%m-%d')" >> $GITHUB_OUTPUT
sha_short=$(git rev-parse --short HEAD)
echo "sha_short=${sha_short:0:7}" >> $GITHUB_OUTPUT
# REF: https://github.com/marketplace/actions/gh-release
# Create a release only in nushell/nightly repo
- name: Publish Archive
uses: softprops/action-gh-release@v2.0.5
if: ${{ startsWith(github.repository, 'nushell/nightly') }}
with:
draft: false
prerelease: true
name: Nu-nightly-${{ steps.vars.outputs.date }}-${{ steps.vars.outputs.sha_short }}
tag_name: nightly-${{ steps.vars.outputs.sha_short }}
body: |
This is a NIGHTLY build of Nushell.
It is NOT recommended for production use.
files: ${{ steps.nu.outputs.archive }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
cleanup:
name: Cleanup
# Should only run in nushell/nightly repo
@ -310,14 +190,14 @@ jobs:
- name: Waiting for Release
run: sleep 1800
- uses: actions/checkout@v4.1.5
- uses: actions/checkout@v4
with:
ref: main
- name: Setup Nushell
uses: hustcer/setup-nu@v3.10
uses: hustcer/setup-nu@v3
with:
version: 0.93.0
version: 0.101.0
# Keep the last a few releases
- name: Delete Older Releases

102
.github/workflows/release-pkg.nu vendored
View File

@ -9,7 +9,6 @@
# Instructions for manually creating an MSI for Winget Releases when they fail
# Added 2022-11-29 when Windows packaging wouldn't work
# Updated again on 2023-02-23 because msis are still failing validation
# Update on 2023-10-18 to use RELEASE_TYPE env var to determine if full or not
# To run this manual for windows here are the steps I take
# checkout the release you want to publish
# 1. git checkout 0.86.0
@ -17,28 +16,26 @@
# 2. $env:CARGO_TARGET_DIR = ""
# 2. hide-env CARGO_TARGET_DIR
# 3. $env.TARGET = 'x86_64-pc-windows-msvc'
# 4. $env.TARGET_RUSTFLAGS = ''
# 5. $env.GITHUB_WORKSPACE = 'D:\nushell'
# 6. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
# 7. $env.OS = 'windows-latest'
# 8. $env.RELEASE_TYPE = '' # There is full and '' for normal releases
# 4. $env.GITHUB_WORKSPACE = 'D:\nushell'
# 5. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
# 6. $env.OS = 'windows-latest'
# make sure 7z.exe is in your path https://www.7-zip.org/download.html
# 9. $env.Path = ($env.Path | append 'c:\apps\7-zip')
# 7. $env.Path = ($env.Path | append 'c:\apps\7-zip')
# make sure aria2c.exe is in your path https://github.com/aria2/aria2
# 10. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
# 8. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
# make sure you have the wixtools installed https://wixtoolset.org/
# 11. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
# 9. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
# You need to run the release-pkg twice. The first pass, with _EXTRA_ as 'bin', makes the output
# folder and builds everything. The second pass, that generates the msi file, with _EXTRA_ as 'msi'
# 12. $env._EXTRA_ = 'bin'
# 13. source .github\workflows\release-pkg.nu
# 14. cd ..
# 15. $env._EXTRA_ = 'msi'
# 16. source .github\workflows\release-pkg.nu
# 10. $env._EXTRA_ = 'bin'
# 11. source .github\workflows\release-pkg.nu
# 12. cd ..
# 13. $env._EXTRA_ = 'msi'
# 14. source .github\workflows\release-pkg.nu
# After msi is generated, you have to update winget-pkgs repo, you'll need to patch the release
# by deleting the existing msi and uploading this new msi. Then you'll need to update the hash
# on the winget-pkgs PR. To generate the hash, run this command
# 17. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
# 15. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
# Then, just take the output and put it in the winget-pkgs PR for the hash on the msi
@ -48,31 +45,15 @@ let os = $env.OS
let target = $env.TARGET
# Repo source dir like `/home/runner/work/nushell/nushell`
let src = $env.GITHUB_WORKSPACE
let flags = $env.TARGET_RUSTFLAGS
let dist = $'($env.GITHUB_WORKSPACE)/output'
let version = (open Cargo.toml | get package.version)
print $'Debugging info:'
print { version: $version, bin: $bin, os: $os, releaseType: $env.RELEASE_TYPE, target: $target, src: $src, flags: $flags, dist: $dist }; hr-line -b
# Rename the full release name so that we won't break the existing scripts for standard release downloading, such as:
# curl -s https://api.github.com/repos/chmln/sd/releases/latest | grep browser_download_url | cut -d '"' -f 4 | grep x86_64-unknown-linux-musl
const FULL_RLS_NAMING = {
x86_64-apple-darwin: 'x86_64-darwin-full',
aarch64-apple-darwin: 'aarch64-darwin-full',
x86_64-unknown-linux-gnu: 'x86_64-linux-gnu-full',
x86_64-pc-windows-msvc: 'x86_64-windows-msvc-full',
x86_64-unknown-linux-musl: 'x86_64-linux-musl-full',
aarch64-unknown-linux-gnu: 'aarch64-linux-gnu-full',
aarch64-pc-windows-msvc: 'aarch64-windows-msvc-full',
riscv64gc-unknown-linux-gnu: 'riscv64-linux-gnu-full',
armv7-unknown-linux-gnueabihf: 'armv7-linux-gnueabihf-full',
}
print { version: $version, bin: $bin, os: $os, target: $target, src: $src, dist: $dist }; hr-line -b
# $env
let USE_UBUNTU = $os starts-with ubuntu
let FULL_NAME = $FULL_RLS_NAMING | get -i $target | default 'unknown-target-full'
print $'(char nl)Packaging ($bin) v($version) for ($target) in ($src)...'; hr-line -b
if not ('Cargo.lock' | path exists) { cargo generate-lockfile }
@ -91,23 +72,44 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
'aarch64-unknown-linux-gnu' => {
sudo apt-get install gcc-aarch64-linux-gnu -y
$env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = 'aarch64-linux-gnu-gcc'
cargo-build-nu $flags
cargo-build-nu
}
'riscv64gc-unknown-linux-gnu' => {
sudo apt-get install gcc-riscv64-linux-gnu -y
$env.CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER = 'riscv64-linux-gnu-gcc'
cargo-build-nu $flags
cargo-build-nu
}
'armv7-unknown-linux-gnueabihf' => {
sudo apt-get install pkg-config gcc-arm-linux-gnueabihf -y
$env.CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER = 'arm-linux-gnueabihf-gcc'
cargo-build-nu $flags
cargo-build-nu
}
'aarch64-unknown-linux-musl' => {
aria2c https://musl.cc/aarch64-linux-musl-cross.tgz
tar -xf aarch64-linux-musl-cross.tgz -C $env.HOME
$env.PATH = ($env.PATH | split row (char esep) | prepend $'($env.HOME)/aarch64-linux-musl-cross/bin')
$env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER = 'aarch64-linux-musl-gcc'
cargo-build-nu
}
'armv7-unknown-linux-musleabihf' => {
aria2c https://musl.cc/armv7r-linux-musleabihf-cross.tgz
tar -xf armv7r-linux-musleabihf-cross.tgz -C $env.HOME
$env.PATH = ($env.PATH | split row (char esep) | prepend $'($env.HOME)/armv7r-linux-musleabihf-cross/bin')
$env.CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABIHF_LINKER = 'armv7r-linux-musleabihf-gcc'
cargo-build-nu
}
'loongarch64-unknown-linux-gnu' => {
aria2c https://github.com/loongson/build-tools/releases/download/2024.08.08/x86_64-cross-tools-loongarch64-binutils_2.43-gcc_14.2.0-glibc_2.40.tar.xz
tar xf x86_64-cross-tools-loongarch64-*.tar.xz
$env.PATH = ($env.PATH | split row (char esep) | prepend $'($env.PWD)/cross-tools/bin')
$env.CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_LINKER = 'loongarch64-unknown-linux-gnu-gcc'
cargo-build-nu
}
_ => {
# musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?'
# Actually just for x86_64-unknown-linux-musl target
if $USE_UBUNTU { sudo apt install musl-tools -y }
cargo-build-nu $flags
cargo-build-nu
}
}
}
@ -116,7 +118,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
# Build for Windows without static-link-openssl feature
# ----------------------------------------------------------------------------
if $os in ['windows-latest'] {
cargo-build-nu $flags
cargo-build-nu
}
# ----------------------------------------------------------------------------
@ -162,7 +164,7 @@ cd $dist; print $'(char nl)Creating release archive...'; hr-line
if $os in ['macos-latest'] or $USE_UBUNTU {
let files = (ls | get name)
let dest = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
let dest = $'($bin)-($version)-($target)'
let archive = $'($dist)/($dest).tar.gz'
mkdir $dest
@ -177,11 +179,15 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
} else if $os == 'windows-latest' {
let releaseStem = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
let releaseStem = $'($bin)-($version)-($target)'
print $'(char nl)Download less related stuffs...'; hr-line
# todo: less-v661 is out but is released as a zip file. maybe we should switch to that and extract it?
aria2c https://github.com/jftuga/less-Windows/releases/download/less-v608/less.exe -o less.exe
aria2c https://raw.githubusercontent.com/jftuga/less-Windows/master/LICENSE -o LICENSE-for-less.txt
# the below was renamed because it was failing to download for darren. it should work but it wasn't
# todo: maybe we should get rid of this aria2c dependency and just use http get?
#aria2c https://raw.githubusercontent.com/jftuga/less-Windows/master/LICENSE -o LICENSE-for-less.txt
aria2c https://github.com/jftuga/less-Windows/blob/master/LICENSE -o LICENSE-for-less.txt
# Create Windows msi release package
if (get-env _EXTRA_) == 'msi' {
@ -214,19 +220,11 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
}
}
def 'cargo-build-nu' [ options: string ] {
if ($options | str trim | is-empty) {
if $os == 'windows-latest' {
cargo build --release --all --target $target
} else {
cargo build --release --all --target $target --features=static-link-openssl
}
def 'cargo-build-nu' [] {
if $os == 'windows-latest' {
cargo build --release --all --target $target
} else {
if $os == 'windows-latest' {
cargo build --release --all --target $target $options
} else {
cargo build --release --all --target $target --features=static-link-openssl $options
}
cargo build --release --all --target $target --features=static-link-openssl
}
}

150
.github/workflows/release.yml vendored
View File

@ -7,15 +7,17 @@ name: Create Release Draft
on:
workflow_dispatch:
push:
tags: ["[0-9]+.[0-9]+.[0-9]+*"]
tags:
- '[0-9]+.[0-9]+.[0-9]+*'
- '!*nightly*' # Don't trigger release for nightly tags
defaults:
run:
shell: bash
jobs:
standard:
name: Std
release:
name: Nu
strategy:
fail-fast: false
@ -28,81 +30,78 @@ jobs:
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- aarch64-unknown-linux-gnu
- aarch64-unknown-linux-musl
- armv7-unknown-linux-gnueabihf
- armv7-unknown-linux-musleabihf
- riscv64gc-unknown-linux-gnu
- loongarch64-unknown-linux-gnu
extra: ['bin']
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: x86_64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: ''
os: ubuntu-22.04
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
target_rustflags: ''
os: ubuntu-22.04
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: ''
os: ubuntu-22.04
- target: aarch64-unknown-linux-musl
os: ubuntu-22.04
- target: armv7-unknown-linux-gnueabihf
os: ubuntu-20.04
target_rustflags: ''
os: ubuntu-22.04
- target: armv7-unknown-linux-musleabihf
os: ubuntu-22.04
- target: riscv64gc-unknown-linux-gnu
os: ubuntu-latest
target_rustflags: ''
os: ubuntu-22.04
- target: loongarch64-unknown-linux-gnu
os: ubuntu-22.04
runs-on: ${{matrix.os}}
steps:
- uses: actions/checkout@v4.1.5
- uses: actions/checkout@v4
- name: Update Rust Toolchain Target
run: |
echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
- name: Setup Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
with:
cache: false
rustflags: ''
- name: Setup Nushell
uses: hustcer/setup-nu@v3.10
uses: hustcer/setup-nu@v3
with:
version: 0.93.0
version: 0.101.0
- name: Release Nu Binary
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: standard
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
# REF: https://github.com/marketplace/actions/gh-release
# WARN: Don't upgrade this action due to the release per asset issue.
# See: https://github.com/softprops/action-gh-release/issues/445
- name: Publish Archive
uses: softprops/action-gh-release@v2.0.5
if: ${{ startsWith(github.ref, 'refs/tags/') }}
@ -112,92 +111,25 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
full:
name: Full
strategy:
fail-fast: false
matrix:
target:
- aarch64-apple-darwin
- x86_64-apple-darwin
- x86_64-pc-windows-msvc
- aarch64-pc-windows-msvc
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- aarch64-unknown-linux-gnu
extra: ['bin']
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
runs-on: ${{matrix.os}}
sha256sum:
needs: release
name: Create Sha256sum
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.5
- name: Update Rust Toolchain Target
run: |
echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
- name: Setup Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
with:
cache: false
rustflags: ''
- name: Setup Nushell
uses: hustcer/setup-nu@v3.10
with:
version: 0.93.0
- name: Release Nu Binary
id: nu
run: nu .github/workflows/release-pkg.nu
- name: Download Release Archives
env:
RELEASE_TYPE: full
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
# REF: https://github.com/marketplace/actions/gh-release
- name: Publish Archive
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: >-
gh release download ${{ github.ref_name }}
--repo ${{ github.repository }}
--pattern '*'
--dir release
- name: Create Checksums
run: cd release && shasum -a 256 * > ../SHA256SUMS
- name: Publish Checksums
uses: softprops/action-gh-release@v2.0.5
if: ${{ startsWith(github.ref, 'refs/tags/') }}
with:
draft: true
files: ${{ steps.nu.outputs.archive }}
files: SHA256SUMS
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

4
.github/workflows/typos.yml vendored
View File

@ -7,7 +7,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout Actions Repository
uses: actions/checkout@v4.1.5
uses: actions/checkout@v4.1.7
- name: Check spelling
uses: crate-ci/typos@v1.21.0
uses: crate-ci/typos@v1.29.5

26
CITATION.cff Normal file
View File

@ -0,0 +1,26 @@
cff-version: 1.2.0
title: 'Nushell'
message: >-
If you use this software and wish to cite it,
you can use the metadata from this file.
type: software
authors:
- name: "The Nushell Project Team"
identifiers:
- type: url
value: 'https://github.com/nushell/nushell'
description: Repository
repository-code: 'https://github.com/nushell/nushell'
url: 'https://www.nushell.sh/'
abstract: >-
The goal of the Nushell project is to take the Unix
philosophy of shells, where pipes connect simple commands
together, and bring it to the modern style of development.
Thus, rather than being either a shell, or a programming
language, Nushell connects both by bringing a rich
programming language and a full-featured shell together
into one package.
keywords:
- nushell
- shell
license: MIT

10
CONTRIBUTING.md
View File

@ -71,11 +71,6 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref
cargo run
```
- Build and run with dataframe support.
```nushell
cargo run --features=dataframe
```
- Run Clippy on Nushell:
```nushell
@ -93,11 +88,6 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref
cargo test --workspace
```
along with dataframe tests
```nushell
cargo test --workspace --features=dataframe
```
or via the `toolkit.nu` command:
```nushell
use toolkit.nu test

4829
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

215
Cargo.toml
View File

@ -10,8 +10,8 @@ homepage = "https://www.nushell.sh"
license = "MIT"
name = "nu"
repository = "https://github.com/nushell/nushell"
rust-version = "1.77.2"
version = "0.93.1"
rust-version = "1.83.0"
version = "0.102.1"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@ -32,7 +32,6 @@ members = [
"crates/nu-cmd-extra",
"crates/nu-cmd-lang",
"crates/nu-cmd-plugin",
"crates/nu-cmd-dataframe",
"crates/nu-command",
"crates/nu-color-config",
"crates/nu-explore",
@ -40,6 +39,7 @@ members = [
"crates/nu-lsp",
"crates/nu-pretty-hex",
"crates/nu-protocol",
"crates/nu-derive-value",
"crates/nu-plugin",
"crates/nu-plugin-core",
"crates/nu-plugin-engine",
@ -66,142 +66,160 @@ alphanumeric-sort = "1.5"
ansi-str = "0.8"
anyhow = "1.0.82"
base64 = "0.22.1"
bracoxide = "0.1.2"
brotli = "5.0"
bracoxide = "0.1.5"
brotli = "7.0"
byteorder = "1.5"
bytesize = "1.3"
calamine = "0.24.0"
bytes = "1"
bytesize = "1.3.1"
calamine = "0.26.1"
chardetng = "0.1.17"
chrono = { default-features = false, version = "0.4.34" }
chrono-humanize = "0.2.3"
chrono-tz = "0.8"
chrono-tz = "0.10"
crossbeam-channel = "0.5.8"
crossterm = "0.27"
crossterm = "0.28.1"
csv = "1.3"
ctrlc = "3.4"
devicons = "0.6.12"
dialoguer = { default-features = false, version = "0.11" }
digest = { default-features = false, version = "0.10" }
dirs-next = "2.0"
dirs = "5.0"
dirs-sys = "0.4"
dtparse = "2.0"
encoding_rs = "0.8"
fancy-regex = "0.13"
fancy-regex = "0.14"
filesize = "0.2"
filetime = "0.2"
fs_extra = "1.3"
fuzzy-matcher = "0.3"
hamcrest2 = "0.3"
heck = "0.5.0"
human-date-parser = "0.1.1"
indexmap = "2.2"
human-date-parser = "0.2.0"
indexmap = "2.7"
indicatif = "0.17"
interprocess = "2.1.0"
interprocess = "2.2.0"
is_executable = "1.0"
itertools = "0.12"
itertools = "0.13"
libc = "0.2"
libproc = "0.14"
log = "0.4"
lru = "0.12"
lscolors = { version = "0.17", default-features = false }
lsp-server = "0.7.5"
lsp-types = "0.95.0"
lsp-server = "0.7.8"
lsp-types = { version = "0.97.0", features = ["proposed"] }
lsp-textdocument = "0.4.1"
mach2 = "0.4"
md5 = { version = "0.10", package = "md-5" }
miette = "7.2"
mime = "0.3"
miette = "7.5"
mime = "0.3.17"
mime_guess = "2.0"
mockito = { version = "1.4", default-features = false }
mockito = { version = "1.6", default-features = false }
multipart-rs = "0.1.13"
native-tls = "0.2"
nix = { version = "0.28", default-features = false }
nix = { version = "0.29", default-features = false }
notify-debouncer-full = { version = "0.3", default-features = false }
nu-ansi-term = "0.50.0"
nu-ansi-term = "0.50.1"
nucleo-matcher = "0.3"
num-format = "0.4"
num-traits = "0.2"
oem_cp = "2.0.0"
omnipath = "0.1"
once_cell = "1.18"
open = "5.1"
os_pipe = { version = "1.1", features = ["io_safety"] }
open = "5.3"
os_pipe = { version = "1.2", features = ["io_safety"] }
pathdiff = "0.2"
percent-encoding = "2"
pretty_assertions = "1.4"
print-positions = "0.6"
procfs = "0.16.0"
proc-macro-error2 = "2.0"
proc-macro2 = "1.0"
procfs = "0.17.0"
pwd = "1.3"
quick-xml = "0.31.0"
quick-xml = "0.37.0"
quickcheck = "1.0"
quickcheck_macros = "1.0"
quote = "1.0"
rand = "0.8"
getrandom = "0.2" # pick same version that rand requires
rand_chacha = "0.3.1"
ratatui = "0.26"
rayon = "1.10"
reedline = "0.32.0"
regex = "1.9.5"
reedline = "0.38.0"
rmp = "0.8"
rmp-serde = "1.3"
ropey = "1.6.1"
roxmltree = "0.19"
rstest = { version = "0.18", default-features = false }
roxmltree = "0.20"
rstest = { version = "0.23", default-features = false }
rstest_reuse = "0.7"
rusqlite = "0.31"
rust-embed = "8.4.0"
same-file = "1.0"
serde = { version = "1.0", default-features = false }
rust-embed = "8.5.0"
scopeguard = { version = "1.2.0" }
serde = { version = "1.0" }
serde_json = "1.0"
serde_urlencoded = "0.7.1"
serde_yaml = "0.9"
serde_yaml = "0.9.33"
sha2 = "0.10"
strip-ansi-escapes = "0.2.0"
sysinfo = "0.30"
tabled = { version = "0.14.0", default-features = false }
tempfile = "3.10"
terminal_size = "0.3"
titlecase = "2.0"
syn = "2.0"
sysinfo = "0.33"
tabled = { version = "0.17.0", default-features = false }
tempfile = "3.15"
titlecase = "3.0"
toml = "0.8"
trash = "3.3"
trash = "5.2"
update-informer = { version = "1.2.0", default-features = false, features = ["github", "native-tls", "ureq"] }
umask = "2.1"
unicode-segmentation = "1.11"
unicode-width = "0.1"
ureq = { version = "2.9", default-features = false }
unicode-segmentation = "1.12"
unicode-width = "0.2"
ureq = { version = "2.12", default-features = false }
url = "2.2"
uu_cp = "0.0.25"
uu_mkdir = "0.0.25"
uu_mktemp = "0.0.25"
uu_mv = "0.0.25"
uu_whoami = "0.0.25"
uu_uname = "0.0.25"
uucore = "0.0.25"
uuid = "1.8.0"
uu_cp = "0.0.29"
uu_mkdir = "0.0.29"
uu_mktemp = "0.0.29"
uu_mv = "0.0.29"
uu_touch = "0.0.29"
uu_whoami = "0.0.29"
uu_uname = "0.0.29"
uucore = "0.0.29"
uuid = "1.12.0"
v_htmlescape = "0.15.0"
wax = "0.6"
which = "6.0.0"
windows = "0.54"
web-time = "1.1.0"
which = "7.0.0"
windows = "0.56"
windows-sys = "0.48"
winreg = "0.52"
memchr = "2.7.4"
[workspace.lints.clippy]
# Warning: workspace lints affect library code as well as tests, so don't enable lints that would be too noisy in tests like that.
# todo = "warn"
unchecked_duration_subtraction = "warn"
[lints]
workspace = true
[dependencies]
nu-cli = { path = "./crates/nu-cli", version = "0.93.1" }
nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.93.1" }
nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.93.1" }
nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.93.1", optional = true }
nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.93.1", features = [
"dataframe",
], optional = true }
nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.93.1" }
nu-command = { path = "./crates/nu-command", version = "0.93.1" }
nu-engine = { path = "./crates/nu-engine", version = "0.93.1" }
nu-explore = { path = "./crates/nu-explore", version = "0.93.1" }
nu-lsp = { path = "./crates/nu-lsp/", version = "0.93.1" }
nu-parser = { path = "./crates/nu-parser", version = "0.93.1" }
nu-path = { path = "./crates/nu-path", version = "0.93.1" }
nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.93.1" }
nu-protocol = { path = "./crates/nu-protocol", version = "0.93.1" }
nu-std = { path = "./crates/nu-std", version = "0.93.1" }
nu-system = { path = "./crates/nu-system", version = "0.93.1" }
nu-utils = { path = "./crates/nu-utils", version = "0.93.1" }
nu-cli = { path = "./crates/nu-cli", version = "0.102.1" }
nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.102.1" }
nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.102.1" }
nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.102.1", optional = true }
nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.102.1" }
nu-command = { path = "./crates/nu-command", version = "0.102.1" }
nu-engine = { path = "./crates/nu-engine", version = "0.102.1" }
nu-explore = { path = "./crates/nu-explore", version = "0.102.1" }
nu-lsp = { path = "./crates/nu-lsp/", version = "0.102.1" }
nu-parser = { path = "./crates/nu-parser", version = "0.102.1" }
nu-path = { path = "./crates/nu-path", version = "0.102.1" }
nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.102.1" }
nu-protocol = { path = "./crates/nu-protocol", version = "0.102.1" }
nu-std = { path = "./crates/nu-std", version = "0.102.1" }
nu-system = { path = "./crates/nu-system", version = "0.102.1" }
nu-utils = { path = "./crates/nu-utils", version = "0.102.1" }
reedline = { workspace = true, features = ["bashisms", "sqlite"] }
crossterm = { workspace = true }
ctrlc = { workspace = true }
dirs = { workspace = true }
log = { workspace = true }
miette = { workspace = true, features = ["fancy-no-backtrace", "fancy"] }
mimalloc = { version = "0.1.37", default-features = false, optional = true }
mimalloc = { version = "0.1.42", default-features = false, optional = true }
multipart-rs = { workspace = true }
serde_json = { workspace = true }
simplelog = "0.12"
time = "0.3"
@ -222,33 +240,36 @@ nix = { workspace = true, default-features = false, features = [
] }
[dev-dependencies]
nu-test-support = { path = "./crates/nu-test-support", version = "0.93.1" }
nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.93.1" }
nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.93.1" }
nu-test-support = { path = "./crates/nu-test-support", version = "0.102.1" }
nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.102.1" }
nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.102.1" }
assert_cmd = "2.0"
dirs-next = { workspace = true }
tango-bench = "0.5"
dirs = { workspace = true }
tango-bench = "0.6"
pretty_assertions = { workspace = true }
fancy-regex = { workspace = true }
rstest = { workspace = true, default-features = false }
serial_test = "3.1"
serial_test = "3.2"
tempfile = { workspace = true }
[features]
plugin = [
"nu-plugin-engine",
# crates
"nu-cmd-plugin",
"nu-plugin-engine",
# features
"nu-cli/plugin",
"nu-parser/plugin",
"nu-cmd-lang/plugin",
"nu-command/plugin",
"nu-protocol/plugin",
"nu-engine/plugin",
"nu-engine/plugin",
"nu-parser/plugin",
"nu-protocol/plugin",
]
default = ["default-no-clipboard", "system-clipboard"]
# Enables convenient omitting of the system-clipboard feature, as it leads to problems in ci on linux
# See https://github.com/nushell/nushell/pull/11535
default-no-clipboard = [
default = [
"plugin",
"which-support",
"trash-support",
"sqlite",
"mimalloc",
@ -261,6 +282,8 @@ stable = ["default"]
static-link-openssl = ["dep:openssl", "nu-cmd-lang/static-link-openssl"]
mimalloc = ["nu-cmd-lang/mimalloc", "dep:mimalloc"]
# Optional system clipboard support in `reedline`, this behavior has problematic compatibility with some systems.
# Missing X server/ Wayland can cause issues
system-clipboard = [
"reedline/system_clipboard",
"nu-cli/system-clipboard",
@ -268,14 +291,10 @@ system-clipboard = [
]
# Stable (Default)
which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"]
trash-support = ["nu-command/trash-support", "nu-cmd-lang/trash-support"]
# Dataframe feature for nushell
dataframe = ["dep:nu-cmd-dataframe", "nu-cmd-lang/dataframe"]
# SQLite commands for nushell
sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite"]
sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite", "nu-std/sqlite"]
[profile.release]
opt-level = "s" # Optimize for size

2
LICENSE
View File

@ -1,6 +1,6 @@
MIT License
Copyright (c) 2019 - 2023 The Nushell Project Developers
Copyright (c) 2019 - 2025 The Nushell Project Developers
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

122
README.md
View File

@ -4,7 +4,6 @@
[![Nightly Build](https://github.com/nushell/nushell/actions/workflows/nightly-build.yml/badge.svg)](https://github.com/nushell/nushell/actions/workflows/nightly-build.yml)
[![Discord](https://img.shields.io/discord/601130461678272522.svg?logo=discord)](https://discord.gg/NtAbbGn)
[![The Changelog #363](https://img.shields.io/badge/The%20Changelog-%23363-61c192.svg)](https://changelog.com/podcast/363)
[![@nu_shell](https://img.shields.io/badge/twitter-@nu_shell-1DA1F3?style=flat-square)](https://twitter.com/nu_shell)
[![GitHub commit activity](https://img.shields.io/github/commit-activity/m/nushell/nushell)](https://github.com/nushell/nushell/graphs/commit-activity)
[![GitHub contributors](https://img.shields.io/github/contributors/nushell/nushell)](https://github.com/nushell/nushell/graphs/contributors)
@ -35,7 +34,7 @@ This project has reached a minimum-viable-product level of quality. Many people
The [Nushell book](https://www.nushell.sh/book/) is the primary source of Nushell documentation. You can find [a full list of Nu commands in the book](https://www.nushell.sh/commands/), and we have many examples of using Nu in our [cookbook](https://www.nushell.sh/cookbook/).
We're also active on [Discord](https://discord.gg/NtAbbGn) and [Twitter](https://twitter.com/nu_shell); come and chat with us!
We're also active on [Discord](https://discord.gg/NtAbbGn); come and chat with us!
## Installation
@ -52,13 +51,13 @@ To use `Nu` in GitHub Action, check [setup-nu](https://github.com/marketplace/ac
Detailed installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/installation.html). Nu is available via many package managers:
[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg)](https://repology.org/project/nushell/versions)
[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg?columns=3)](https://repology.org/project/nushell/versions)
For details about which platforms the Nushell team actively supports, see [our platform support policy](devdocs/PLATFORM_SUPPORT.md).
## Configuration
The default configurations can be found at [sample_config](crates/nu-utils/src/sample_config)
The default configurations can be found at [sample_config](crates/nu-utils/src/default_files)
which are the configuration files one gets when they startup Nushell for the first time.
It sets all of the default configuration to run Nushell. From here one can
@ -95,44 +94,44 @@ Commands that work in the pipeline fit into one of three categories:
Commands are separated by the pipe symbol (`|`) to denote a pipeline flowing left to right.
```shell
> ls | where type == "dir" | table
╭────┬──────────┬──────┬─────────┬───────────────╮
│ # │ name │ type │ size │ modified │
├────┼──────────┼──────┼─────────┼───────────────┤
│ 0 │ .cargo │ dir │ 0 B │ 9 minutes ago │
│ 1 │ assets │ dir │ 0 B │ 2 weeks ago │
│ 2 │ crates │ dir │ 4.0 KiB │ 2 weeks ago │
│ 3 │ docker │ dir │ 0 B │ 2 weeks ago │
│ 4 │ docs │ dir │ 0 B │ 2 weeks ago │
│ 5 │ images │ dir │ 0 B │ 2 weeks ago │
│ 6 │ pkg_mgrs │ dir │ 0 B │ 2 weeks ago │
│ 7 │ samples │ dir │ 0 B │ 2 weeks ago │
│ 8 │ src │ dir │ 4.0 KiB │ 2 weeks ago │
│ 9 │ target │ dir │ 0 B │ a day ago │
│ 10 │ tests │ dir │ 4.0 KiB │ 2 weeks ago │
│ 11 │ wix │ dir │ 0 B │ 2 weeks ago │
╰────┴──────────┴──────┴─────────┴───────────────╯
ls | where type == "dir" | table
# => ╭────┬──────────┬──────┬─────────┬───────────────╮
# => │ # │ name │ type │ size │ modified │
# => ├────┼──────────┼──────┼─────────┼───────────────┤
# => │ 0 │ .cargo │ dir │ 0 B │ 9 minutes ago │
# => │ 1 │ assets │ dir │ 0 B │ 2 weeks ago │
# => │ 2 │ crates │ dir │ 4.0 KiB │ 2 weeks ago │
# => │ 3 │ docker │ dir │ 0 B │ 2 weeks ago │
# => │ 4 │ docs │ dir │ 0 B │ 2 weeks ago │
# => │ 5 │ images │ dir │ 0 B │ 2 weeks ago │
# => │ 6 │ pkg_mgrs │ dir │ 0 B │ 2 weeks ago │
# => │ 7 │ samples │ dir │ 0 B │ 2 weeks ago │
# => │ 8 │ src │ dir │ 4.0 KiB │ 2 weeks ago │
# => │ 9 │ target │ dir │ 0 B │ a day ago │
# => │ 10 │ tests │ dir │ 4.0 KiB │ 2 weeks ago │
# => │ 11 │ wix │ dir │ 0 B │ 2 weeks ago │
# => ╰────┴──────────┴──────┴─────────┴───────────────╯
```
Because most of the time you'll want to see the output of a pipeline, `table` is assumed.
We could have also written the above:
```shell
> ls | where type == "dir"
ls | where type == "dir"
```
Being able to use the same commands and compose them differently is an important philosophy in Nu.
For example, we could use the built-in `ps` command to get a list of the running processes, using the same `where` as above.
```shell
> ps | where cpu > 0
╭───┬───────┬───────────┬───────┬───────────┬───────────╮
│ # │ pid │ name │ cpu │ mem │ virtual │
├───┼───────┼───────────┼───────┼───────────┼───────────┤
│ 0 │ 2240 │ Slack.exe │ 16.40 │ 178.3 MiB │ 232.6 MiB │
│ 1 │ 16948 │ Slack.exe │ 16.32 │ 205.0 MiB │ 197.9 MiB │
│ 2 │ 17700 │ nu.exe │ 3.77 │ 26.1 MiB │ 8.8 MiB │
╰───┴───────┴───────────┴───────┴───────────┴───────────╯
ps | where cpu > 0
# => ╭───┬───────┬───────────┬───────┬───────────┬───────────╮
# => │ # │ pid │ name │ cpu │ mem │ virtual │
# => ├───┼───────┼───────────┼───────┼───────────┼───────────┤
# => │ 0 │ 2240 │ Slack.exe │ 16.40 │ 178.3 MiB │ 232.6 MiB │
# => │ 1 │ 16948 │ Slack.exe │ 16.32 │ 205.0 MiB │ 197.9 MiB │
# => │ 2 │ 17700 │ nu.exe │ 3.77 │ 26.1 MiB │ 8.8 MiB │
# => ╰───┴───────┴───────────┴───────┴───────────┴───────────╯
```
### Opening files
@ -141,46 +140,46 @@ Nu can load file and URL contents as raw text or structured data (if it recogniz
For example, you can load a .toml file as structured data and explore it:
```shell
> open Cargo.toml
╭──────────────────┬────────────────────╮
│ bin │ [table 1 row] │
│ dependencies │ {record 25 fields} │
│ dev-dependencies │ {record 8 fields} │
│ features │ {record 10 fields} │
│ package │ {record 13 fields} │
│ patch │ {record 1 field} │
│ profile │ {record 3 fields} │
│ target │ {record 3 fields} │
│ workspace │ {record 1 field} │
╰──────────────────┴────────────────────╯
open Cargo.toml
# => ╭──────────────────┬────────────────────╮
# => │ bin │ [table 1 row] │
# => │ dependencies │ {record 25 fields} │
# => │ dev-dependencies │ {record 8 fields} │
# => │ features │ {record 10 fields} │
# => │ package │ {record 13 fields} │
# => │ patch │ {record 1 field} │
# => │ profile │ {record 3 fields} │
# => │ target │ {record 3 fields} │
# => │ workspace │ {record 1 field} │
# => ╰──────────────────┴────────────────────╯
```
We can pipe this into a command that gets the contents of one of the columns:
```shell
> open Cargo.toml | get package
╭───────────────┬────────────────────────────────────╮
│ authors │ [list 1 item] │
│ default-run │ nu │
│ description │ A new type of shell │
│ documentation │ https://www.nushell.sh/book/ │
│ edition │ 2018 │
│ exclude │ [list 1 item] │
│ homepage │ https://www.nushell.sh │
│ license │ MIT │
│ metadata │ {record 1 field} │
│ name │ nu │
│ repository │ https://github.com/nushell/nushell │
│ rust-version │ 1.60 │
│ version │ 0.72.0 │
╰───────────────┴────────────────────────────────────╯
open Cargo.toml | get package
# => ╭───────────────┬────────────────────────────────────╮
# => │ authors │ [list 1 item] │
# => │ default-run │ nu │
# => │ description │ A new type of shell │
# => │ documentation │ https://www.nushell.sh/book/ │
# => │ edition │ 2018 │
# => │ exclude │ [list 1 item] │
# => │ homepage │ https://www.nushell.sh │
# => │ license │ MIT │
# => │ metadata │ {record 1 field} │
# => │ name │ nu │
# => │ repository │ https://github.com/nushell/nushell │
# => │ rust-version │ 1.60 │
# => │ version │ 0.72.0 │
# => ╰───────────────┴────────────────────────────────────╯
```
And if needed we can drill down further:
```shell
> open Cargo.toml | get package.version
0.72.0
open Cargo.toml | get package.version
# => 0.72.0
```
### Plugins
@ -222,13 +221,14 @@ Please submit an issue or PR to be added to this list.
- [clap](https://github.com/clap-rs/clap/tree/master/clap_complete_nushell)
- [Dorothy](http://github.com/bevry/dorothy)
- [Direnv](https://github.com/direnv/direnv/blob/master/docs/hook.md#nushell)
- [x-cmd](https://x-cmd.com/mod/nu)
## Contributing
See [Contributing](CONTRIBUTING.md) for details. Thanks to all the people who already contributed!
<a href="https://github.com/nushell/nushell/graphs/contributors">
<img src="https://contributors-img.web.app/image?repo=nushell/nushell&max=750" />
<img src="https://contributors-img.web.app/image?repo=nushell/nushell&max=750&columns=20" />
</a>
## License

29
SECURITY.md Normal file
View File

@ -0,0 +1,29 @@
# Security Policy
As a shell and programming language Nushell provides you with great powers and the potential to do dangerous things to your computer and data. Whenever there is a risk that a malicious actor can abuse a bug or a violation of documented behavior/assumptions in Nushell to harm you this is a *security* risk.
We want to fix those issues without exposing our users to unnecessary risk. Thus we want to explain our security policy.
Additional issues may be part of *safety* where the behavior of Nushell as designed and implemented can cause unintended harm or a bug causes damage without the involvement of a third party.
## Supported Versions
As Nushell is still under very active pre-stable development, the only version the core team prioritizes for security and safety fixes is the [most recent version as published on GitHub](https://github.com/nushell/nushell/releases/latest).
Only if you provide a strong reasoning and the necessary resources, will we consider blessing a backported fix with an official patch release for a previous version.
## Reporting a Vulnerability
If you suspect that a bug or behavior of Nushell can affect security or may be potentially exploitable, please report the issue to us in private.
Either reach out to the core team on [our Discord server](https://discord.gg/NtAbbGn) to arrange a private channel or use the [GitHub vulnerability reporting form](https://github.com/nushell/nushell/security/advisories/new).
Please try to answer the following questions:
- How can we reach you for further questions?
- What is the bug? Which system of Nushell may be affected?
- Do you have proof-of-concept for a potential exploit or have you observed an exploit in the wild?
- What is your assessment of the severity based on what could be impacted should the bug be exploited?
- Are additional people aware of the issue or deserve credit for identifying the issue?
We will try to get back to you within a week with:
- acknowledging the receipt of the report
- an initial plan of how we want to address this including the primary points of contact for further communication
- our preliminary assessment of how severe we judge the issue
- a proposal for how we can coordinate responsible disclosure (e.g. how we ship the bugfix, if we need to coordinate with distribution maintainers, when you can release a blog post if you want to etc.)
For purely *safety* related issues where the impact is severe by direct user action instead of malicious input or third parties, feel free to open a regular issue. If we deem that there may be an additional *security* risk on a *safety* issue we may continue discussions in a restricted forum.

28
benches/benchmarks.rs
View File

@ -4,11 +4,14 @@ use nu_plugin_protocol::{PluginCallResponse, PluginOutput};
use nu_protocol::{
engine::{EngineState, Stack},
PipelineData, Span, Spanned, Value,
PipelineData, Signals, Span, Spanned, Value,
};
use nu_std::load_standard_library;
use nu_utils::{get_default_config, get_default_env};
use std::rc::Rc;
use std::{
rc::Rc,
sync::{atomic::AtomicBool, Arc},
};
use std::hint::black_box;
@ -42,13 +45,13 @@ fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) {
};
let mut stack = Stack::new();
evaluate_commands(
&commands,
&mut engine,
&mut stack,
PipelineData::empty(),
None,
false,
Default::default(),
)
.unwrap();
@ -90,8 +93,7 @@ fn bench_command(
&mut engine,
&mut stack,
PipelineData::empty(),
None,
false,
Default::default(),
)
.unwrap(),
);
@ -250,14 +252,12 @@ fn bench_eval_interleave(n: i32) -> impl IntoBenchmarks {
)
}
fn bench_eval_interleave_with_ctrlc(n: i32) -> impl IntoBenchmarks {
fn bench_eval_interleave_with_interrupt(n: i32) -> impl IntoBenchmarks {
let mut engine = setup_engine();
engine.ctrlc = Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new(
false,
)));
engine.set_signals(Signals::new(Arc::new(AtomicBool::new(false))));
let stack = Stack::new();
bench_command(
&format!("eval_interleave_with_ctrlc_{n}"),
&format!("eval_interleave_with_interrupt_{n}"),
&format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
stack,
engine,
@ -445,9 +445,9 @@ tango_benchmarks!(
bench_eval_interleave(100),
bench_eval_interleave(1_000),
bench_eval_interleave(10_000),
bench_eval_interleave_with_ctrlc(100),
bench_eval_interleave_with_ctrlc(1_000),
bench_eval_interleave_with_ctrlc(10_000),
bench_eval_interleave_with_interrupt(100),
bench_eval_interleave_with_interrupt(1_000),
bench_eval_interleave_with_interrupt(10_000),
// For
bench_eval_for(1),
bench_eval_for(10),

3
clippy/wasm/clippy.toml Normal file
View File

@ -0,0 +1,3 @@
[[disallowed-types]]
path = "std::time::Instant"
reason = "WASM panics if used, use `web_time::Instant` instead"

34
crates/nu-cli/Cargo.toml
View File

@ -5,41 +5,40 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli"
edition = "2021"
license = "MIT"
name = "nu-cli"
version = "0.93.1"
version = "0.102.1"
[lib]
bench = false
[dev-dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" }
nu-command = { path = "../nu-command", version = "0.93.1" }
nu-test-support = { path = "../nu-test-support", version = "0.93.1" }
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.102.1" }
nu-command = { path = "../nu-command", version = "0.102.1" }
nu-test-support = { path = "../nu-test-support", version = "0.102.1" }
rstest = { workspace = true, default-features = false }
tempfile = { workspace = true }
[dependencies]
nu-cmd-base = { path = "../nu-cmd-base", version = "0.93.1" }
nu-engine = { path = "../nu-engine", version = "0.93.1" }
nu-path = { path = "../nu-path", version = "0.93.1" }
nu-parser = { path = "../nu-parser", version = "0.93.1" }
nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.93.1", optional = true }
nu-protocol = { path = "../nu-protocol", version = "0.93.1" }
nu-utils = { path = "../nu-utils", version = "0.93.1" }
nu-color-config = { path = "../nu-color-config", version = "0.93.1" }
nu-cmd-base = { path = "../nu-cmd-base", version = "0.102.1" }
nu-engine = { path = "../nu-engine", version = "0.102.1", features = ["os"] }
nu-glob = { path = "../nu-glob", version = "0.102.1" }
nu-path = { path = "../nu-path", version = "0.102.1" }
nu-parser = { path = "../nu-parser", version = "0.102.1" }
nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.102.1", optional = true }
nu-protocol = { path = "../nu-protocol", version = "0.102.1", features = ["os"] }
nu-utils = { path = "../nu-utils", version = "0.102.1" }
nu-color-config = { path = "../nu-color-config", version = "0.102.1" }
nu-ansi-term = { workspace = true }
reedline = { workspace = true, features = ["bashisms", "sqlite"] }
chrono = { default-features = false, features = ["std"], workspace = true }
crossterm = { workspace = true }
fancy-regex = { workspace = true }
fuzzy-matcher = { workspace = true }
is_executable = { workspace = true }
log = { workspace = true }
miette = { workspace = true, features = ["fancy-no-backtrace"] }
lscolors = { workspace = true, default-features = false, features = ["nu-ansi-term"] }
once_cell = { workspace = true }
miette = { workspace = true, features = ["fancy-no-backtrace"] }
nucleo-matcher = { workspace = true }
percent-encoding = { workspace = true }
pathdiff = { workspace = true }
sysinfo = { workspace = true }
unicode-segmentation = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
@ -48,3 +47,6 @@ which = { workspace = true }
[features]
plugin = ["nu-plugin-engine"]
system-clipboard = ["reedline/system_clipboard"]
[lints]
workspace = true

7
crates/nu-cli/README.md Normal file
View File

@ -0,0 +1,7 @@
This crate implements the core functionality of the interactive Nushell REPL and interfaces with `reedline`.
Currently implements the syntax highlighting and completions logic.
Furthermore includes a few commands that are specific to `reedline`
## Internal Nushell crate
This crate implements components of Nushell and is not designed to support plugin authors or other users directly.

2
crates/nu-cli/src/commands/commandline/commandline_.rs
View File

@ -14,7 +14,7 @@ impl Command for Commandline {
.category(Category::Core)
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"View the current command line input buffer."
}

4
crates/nu-cli/src/commands/commandline/edit.rs
View File

@ -29,12 +29,12 @@ impl Command for SubCommand {
.required(
"str",
SyntaxShape::String,
"the string to perform the operation with",
"The string to perform the operation with.",
)
.category(Category::Core)
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"Modify the current command line input buffer."
}

2
crates/nu-cli/src/commands/commandline/get_cursor.rs
View File

@ -16,7 +16,7 @@ impl Command for SubCommand {
.category(Category::Core)
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"Get the current cursor position."
}

4
crates/nu-cli/src/commands/commandline/set_cursor.rs
View File

@ -18,11 +18,11 @@ impl Command for SubCommand {
"set the current cursor position to the end of the buffer",
Some('e'),
)
.optional("pos", SyntaxShape::Int, "Cursor position to be set")
.optional("pos", SyntaxShape::Int, "Cursor position to be set.")
.category(Category::Core)
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"Set the current cursor position."
}

1
crates/nu-cli/src/commands/default_context.rs
View File

@ -17,6 +17,7 @@ pub fn add_cli_context(mut engine_state: EngineState) -> EngineState {
CommandlineGetCursor,
CommandlineSetCursor,
History,
HistoryImport,
HistorySession,
Keybindings,
KeybindingsDefault,

9
crates/nu-cli/src/commands/history/fields.rs Normal file
View File

@ -0,0 +1,9 @@
// Each const is named after a HistoryItem field, and the value is the field name to be displayed to
// the user (or accept during import).
pub const COMMAND_LINE: &str = "command";
pub const START_TIMESTAMP: &str = "start_timestamp";
pub const HOSTNAME: &str = "hostname";
pub const CWD: &str = "cwd";
pub const EXIT_STATUS: &str = "exit_status";
pub const DURATION: &str = "duration";
pub const SESSION_ID: &str = "session_id";

231
crates/nu-cli/src/commands/history/history_.rs
View File

@ -1,10 +1,12 @@
use nu_engine::command_prelude::*;
use nu_protocol::HistoryFileFormat;
use nu_protocol::{shell_error::io::IoError, HistoryFileFormat};
use reedline::{
FileBackedHistory, History as ReedlineHistory, HistoryItem, SearchDirection, SearchQuery,
SqliteBackedHistory,
};
use super::fields;
#[derive(Clone)]
pub struct History;
@ -13,7 +15,7 @@ impl Command for History {
"history"
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"Get the command history."
}
@ -42,91 +44,78 @@ impl Command for History {
let Some(history) = engine_state.history_config() else {
return Ok(PipelineData::empty());
};
// todo for sqlite history this command should be an alias to `open ~/.config/nushell/history.sqlite3 | get history`
if let Some(config_path) = nu_path::config_dir() {
let clear = call.has_flag(engine_state, stack, "clear")?;
let long = call.has_flag(engine_state, stack, "long")?;
let ctrlc = engine_state.ctrlc.clone();
let Some(history_path) = history.file_path() else {
return Err(ShellError::ConfigDirNotFound { span: Some(head) });
};
let mut history_path = config_path;
history_path.push("nushell");
match history.file_format {
HistoryFileFormat::Sqlite => {
history_path.push("history.sqlite3");
}
HistoryFileFormat::PlainText => {
history_path.push("history.txt");
}
}
if call.has_flag(engine_state, stack, "clear")? {
let _ = std::fs::remove_file(history_path);
// TODO: FIXME also clear the auxiliary files when using sqlite
return Ok(PipelineData::empty());
}
if clear {
let _ = std::fs::remove_file(history_path);
// TODO: FIXME also clear the auxiliary files when using sqlite
Ok(PipelineData::empty())
} else {
let history_reader: Option<Box<dyn ReedlineHistory>> = match history.file_format {
HistoryFileFormat::Sqlite => {
SqliteBackedHistory::with_file(history_path.clone(), None, None)
.map(|inner| {
let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
boxed
})
.ok()
}
HistoryFileFormat::PlainText => FileBackedHistory::with_file(
history.max_size as usize,
history_path.clone(),
)
let long = call.has_flag(engine_state, stack, "long")?;
let signals = engine_state.signals().clone();
let history_reader: Option<Box<dyn ReedlineHistory>> = match history.file_format {
HistoryFileFormat::Sqlite => {
SqliteBackedHistory::with_file(history_path.clone(), None, None)
.map(|inner| {
let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
boxed
})
.ok(),
};
match history.file_format {
HistoryFileFormat::PlainText => Ok(history_reader
.and_then(|h| {
h.search(SearchQuery::everything(SearchDirection::Forward, None))
.ok()
})
.map(move |entries| {
entries.into_iter().enumerate().map(move |(idx, entry)| {
Value::record(
record! {
"command" => Value::string(entry.command_line, head),
"index" => Value::int(idx as i64, head),
},
head,
)
})
})
.ok_or(ShellError::FileNotFound {
file: history_path.display().to_string(),
span: head,
})?
.into_pipeline_data(head, ctrlc)),
HistoryFileFormat::Sqlite => Ok(history_reader
.and_then(|h| {
h.search(SearchQuery::everything(SearchDirection::Forward, None))
.ok()
})
.map(move |entries| {
entries.into_iter().enumerate().map(move |(idx, entry)| {
create_history_record(idx, entry, long, head)
})
})
.ok_or(ShellError::FileNotFound {
file: history_path.display().to_string(),
span: head,
})?
.into_pipeline_data(head, ctrlc)),
}
.ok()
}
} else {
Err(ShellError::ConfigDirNotFound { span: Some(head) })
HistoryFileFormat::Plaintext => {
FileBackedHistory::with_file(history.max_size as usize, history_path.clone())
.map(|inner| {
let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
boxed
})
.ok()
}
};
match history.file_format {
HistoryFileFormat::Plaintext => Ok(history_reader
.and_then(|h| {
h.search(SearchQuery::everything(SearchDirection::Forward, None))
.ok()
})
.map(move |entries| {
entries.into_iter().enumerate().map(move |(idx, entry)| {
Value::record(
record! {
fields::COMMAND_LINE => Value::string(entry.command_line, head),
// TODO: This name is inconsistent with create_history_record.
"index" => Value::int(idx as i64, head),
},
head,
)
})
})
.ok_or(IoError::new(
std::io::ErrorKind::NotFound,
head,
history_path,
))?
.into_pipeline_data(head, signals)),
HistoryFileFormat::Sqlite => Ok(history_reader
.and_then(|h| {
h.search(SearchQuery::everything(SearchDirection::Forward, None))
.ok()
})
.map(move |entries| {
entries
.into_iter()
.enumerate()
.map(move |(idx, entry)| create_history_record(idx, entry, long, head))
})
.ok_or(IoError::new(
std::io::ErrorKind::NotFound,
head,
history_path,
))?
.into_pipeline_data(head, signals)),
}
}
@ -156,58 +145,34 @@ fn create_history_record(idx: usize, entry: HistoryItem, long: bool, head: Span)
//2. Create a record of either short or long columns and values
let item_id_value = Value::int(
match entry.id {
Some(id) => {
let ids = id.to_string();
match ids.parse::<i64>() {
Ok(i) => i,
_ => 0i64,
}
}
None => 0i64,
},
entry
.id
.and_then(|id| id.to_string().parse::<i64>().ok())
.unwrap_or_default(),
head,
);
let start_timestamp_value = Value::string(
match entry.start_timestamp {
Some(time) => time.to_string(),
None => "".into(),
},
entry
.start_timestamp
.map(|time| time.to_string())
.unwrap_or_default(),
head,
);
let command_value = Value::string(entry.command_line, head);
let session_id_value = Value::int(
match entry.session_id {
Some(sid) => {
let sids = sid.to_string();
match sids.parse::<i64>() {
Ok(i) => i,
_ => 0i64,
}
}
None => 0i64,
},
head,
);
let hostname_value = Value::string(
match entry.hostname {
Some(host) => host,
None => "".into(),
},
head,
);
let cwd_value = Value::string(
match entry.cwd {
Some(cwd) => cwd,
None => "".into(),
},
entry
.session_id
.and_then(|id| id.to_string().parse::<i64>().ok())
.unwrap_or_default(),
head,
);
let hostname_value = Value::string(entry.hostname.unwrap_or_default(), head);
let cwd_value = Value::string(entry.cwd.unwrap_or_default(), head);
let duration_value = Value::duration(
match entry.duration {
Some(d) => d.as_nanos().try_into().unwrap_or(0),
None => 0,
},
entry
.duration
.and_then(|d| d.as_nanos().try_into().ok())
.unwrap_or(0),
head,
);
let exit_status_value = Value::int(entry.exit_status.unwrap_or(0), head);
@ -216,13 +181,13 @@ fn create_history_record(idx: usize, entry: HistoryItem, long: bool, head: Span)
Value::record(
record! {
"item_id" => item_id_value,
"start_timestamp" => start_timestamp_value,
"command" => command_value,
"session_id" => session_id_value,
"hostname" => hostname_value,
"cwd" => cwd_value,
"duration" => duration_value,
"exit_status" => exit_status_value,
fields::START_TIMESTAMP => start_timestamp_value,
fields::COMMAND_LINE => command_value,
fields::SESSION_ID => session_id_value,
fields::HOSTNAME => hostname_value,
fields::CWD => cwd_value,
fields::DURATION => duration_value,
fields::EXIT_STATUS => exit_status_value,
"idx" => index_value,
},
head,
@ -230,11 +195,11 @@ fn create_history_record(idx: usize, entry: HistoryItem, long: bool, head: Span)
} else {
Value::record(
record! {
"start_timestamp" => start_timestamp_value,
"command" => command_value,
"cwd" => cwd_value,
"duration" => duration_value,
"exit_status" => exit_status_value,
fields::START_TIMESTAMP => start_timestamp_value,
fields::COMMAND_LINE => command_value,
fields::CWD => cwd_value,
fields::DURATION => duration_value,
fields::EXIT_STATUS => exit_status_value,
},
head,
)

441
crates/nu-cli/src/commands/history/history_import.rs Normal file
View File

@ -0,0 +1,441 @@
use std::path::{Path, PathBuf};
use nu_engine::command_prelude::*;
use nu_protocol::{
shell_error::{self, io::IoError},
HistoryFileFormat,
};
use reedline::{
FileBackedHistory, History, HistoryItem, ReedlineError, SearchQuery, SqliteBackedHistory,
};
use super::fields;
#[derive(Clone)]
pub struct HistoryImport;
impl Command for HistoryImport {
fn name(&self) -> &str {
"history import"
}
fn description(&self) -> &str {
"Import command line history."
}
fn extra_description(&self) -> &str {
r#"Can import history from input, either successive command lines or more detailed records. If providing records, available fields are:
command_line, id, start_timestamp, hostname, cwd, duration, exit_status.
If no input is provided, will import all history items from existing history in the other format: if current history is stored in sqlite, it will store it in plain text and vice versa.
Note that history item IDs are ignored when importing from file."#
}
fn signature(&self) -> nu_protocol::Signature {
Signature::build("history import")
.category(Category::History)
.input_output_types(vec![
(Type::Nothing, Type::Nothing),
(Type::String, Type::Nothing),
(Type::List(Box::new(Type::String)), Type::Nothing),
(Type::table(), Type::Nothing),
])
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
example: "history import",
description:
"Append all items from history in the other format to the current history",
result: None,
},
Example {
example: "echo foo | history import",
description: "Append `foo` to the current history",
result: None,
},
Example {
example: "[[ command_line cwd ]; [ foo /home ]] | history import",
description: "Append `foo` ran from `/home` to the current history",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let span = call.head;
let ok = Ok(Value::nothing(call.head).into_pipeline_data());
let Some(history) = engine_state.history_config() else {
return ok;
};
let Some(current_history_path) = history.file_path() else {
return Err(ShellError::ConfigDirNotFound { span: span.into() });
};
if let Some(bak_path) = backup(&current_history_path, span)? {
println!("Backed history to {}", bak_path.display());
}
match input {
PipelineData::Empty => {
let other_format = match history.file_format {
HistoryFileFormat::Sqlite => HistoryFileFormat::Plaintext,
HistoryFileFormat::Plaintext => HistoryFileFormat::Sqlite,
};
let src = new_backend(other_format, None)?;
let mut dst = new_backend(history.file_format, Some(current_history_path))?;
let items = src
.search(SearchQuery::everything(
reedline::SearchDirection::Forward,
None,
))
.map_err(error_from_reedline)?
.into_iter()
.map(Ok);
import(dst.as_mut(), items)
}
_ => {
let input = input.into_iter().map(item_from_value);
import(
new_backend(history.file_format, Some(current_history_path))?.as_mut(),
input,
)
}
}?;
ok
}
}
fn new_backend(
format: HistoryFileFormat,
path: Option<PathBuf>,
) -> Result<Box<dyn History>, ShellError> {
let path = match path {
Some(path) => path,
None => {
let Some(mut path) = nu_path::nu_config_dir() else {
return Err(ShellError::ConfigDirNotFound { span: None });
};
path.push(format.default_file_name());
path.into_std_path_buf()
}
};
fn map(
result: Result<impl History + 'static, ReedlineError>,
) -> Result<Box<dyn History>, ShellError> {
result
.map(|x| Box::new(x) as Box<dyn History>)
.map_err(error_from_reedline)
}
match format {
// Use a reasonably large value for maximum capacity.
HistoryFileFormat::Plaintext => map(FileBackedHistory::with_file(0xfffffff, path)),
HistoryFileFormat::Sqlite => map(SqliteBackedHistory::with_file(path, None, None)),
}
}
fn import(
dst: &mut dyn History,
src: impl Iterator<Item = Result<HistoryItem, ShellError>>,
) -> Result<(), ShellError> {
for item in src {
let mut item = item?;
item.id = None;
dst.save(item).map_err(error_from_reedline)?;
}
Ok(())
}
fn error_from_reedline(e: ReedlineError) -> ShellError {
// TODO: Should we add a new ShellError variant?
ShellError::GenericError {
error: "Reedline error".to_owned(),
msg: format!("{e}"),
span: None,
help: None,
inner: Vec::new(),
}
}
fn item_from_value(v: Value) -> Result<HistoryItem, ShellError> {
let span = v.span();
match v {
Value::Record { val, .. } => item_from_record(val.into_owned(), span),
Value::String { val, .. } => Ok(HistoryItem {
command_line: val,
id: None,
start_timestamp: None,
session_id: None,
hostname: None,
cwd: None,
duration: None,
exit_status: None,
more_info: None,
}),
_ => Err(ShellError::UnsupportedInput {
msg: "Only list and record inputs are supported".to_owned(),
input: v.get_type().to_string(),
msg_span: span,
input_span: span,
}),
}
}
fn item_from_record(mut rec: Record, span: Span) -> Result<HistoryItem, ShellError> {
let cmd = match rec.remove(fields::COMMAND_LINE) {
Some(v) => v.as_str()?.to_owned(),
None => {
return Err(ShellError::TypeMismatch {
err_message: format!("missing column: {}", fields::COMMAND_LINE),
span,
})
}
};
fn get<T>(
rec: &mut Record,
field: &'static str,
f: impl FnOnce(Value) -> Result<T, ShellError>,
) -> Result<Option<T>, ShellError> {
rec.remove(field).map(f).transpose()
}
let rec = &mut rec;
let item = HistoryItem {
command_line: cmd,
id: None,
start_timestamp: get(rec, fields::START_TIMESTAMP, |v| Ok(v.as_date()?.to_utc()))?,
hostname: get(rec, fields::HOSTNAME, |v| Ok(v.as_str()?.to_owned()))?,
cwd: get(rec, fields::CWD, |v| Ok(v.as_str()?.to_owned()))?,
exit_status: get(rec, fields::EXIT_STATUS, |v| v.as_int())?,
duration: get(rec, fields::DURATION, |v| duration_from_value(v, span))?,
more_info: None,
// TODO: Currently reedline doesn't let you create session IDs.
session_id: None,
};
if !rec.is_empty() {
let cols = rec.columns().map(|s| s.as_str()).collect::<Vec<_>>();
return Err(ShellError::TypeMismatch {
err_message: format!("unsupported column names: {}", cols.join(", ")),
span,
});
}
Ok(item)
}
fn duration_from_value(v: Value, span: Span) -> Result<std::time::Duration, ShellError> {
chrono::Duration::nanoseconds(v.as_duration()?)
.to_std()
.map_err(|_| ShellError::NeedsPositiveValue { span })
}
fn find_backup_path(path: &Path, span: Span) -> Result<PathBuf, ShellError> {
let Ok(mut bak_path) = path.to_path_buf().into_os_string().into_string() else {
// This isn't fundamentally problem, but trying to work with OsString is a nightmare.
return Err(ShellError::GenericError {
error: "History path not UTF-8".to_string(),
msg: "History path must be representable as UTF-8".to_string(),
span: Some(span),
help: None,
inner: vec![],
});
};
bak_path.push_str(".bak");
if !Path::new(&bak_path).exists() {
return Ok(bak_path.into());
}
let base_len = bak_path.len();
for i in 1..100 {
use std::fmt::Write;
bak_path.truncate(base_len);
write!(&mut bak_path, ".{i}").unwrap();
if !Path::new(&bak_path).exists() {
return Ok(PathBuf::from(bak_path));
}
}
Err(ShellError::GenericError {
error: "Too many backup files".to_string(),
msg: "Found too many existing backup files".to_string(),
span: Some(span),
help: None,
inner: vec![],
})
}
fn backup(path: &Path, span: Span) -> Result<Option<PathBuf>, ShellError> {
match path.metadata() {
Ok(md) if md.is_file() => (),
Ok(_) => {
return Err(IoError::new_with_additional_context(
shell_error::io::ErrorKind::NotAFile,
span,
PathBuf::from(path),
"history path exists but is not a file",
)
.into())
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => {
return Err(IoError::new_internal(
e.kind(),
"Could not get metadata",
nu_protocol::location!(),
)
.into())
}
}
let bak_path = find_backup_path(path, span)?;
std::fs::copy(path, &bak_path).map_err(|err| {
IoError::new_internal(
err.kind(),
"Could not copy backup",
nu_protocol::location!(),
)
})?;
Ok(Some(bak_path))
}
#[cfg(test)]
mod tests {
use chrono::DateTime;
use rstest::rstest;
use super::*;
#[test]
fn test_item_from_value_string() -> Result<(), ShellError> {
let item = item_from_value(Value::string("foo", Span::unknown()))?;
assert_eq!(
item,
HistoryItem {
command_line: "foo".to_string(),
id: None,
start_timestamp: None,
session_id: None,
hostname: None,
cwd: None,
duration: None,
exit_status: None,
more_info: None
}
);
Ok(())
}
#[test]
fn test_item_from_value_record() {
let span = Span::unknown();
let rec = new_record(&[
("command", Value::string("foo", span)),
(
"start_timestamp",
Value::date(
DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00").unwrap(),
span,
),
),
("hostname", Value::string("localhost", span)),
("cwd", Value::string("/home/test", span)),
("duration", Value::duration(100_000_000, span)),
("exit_status", Value::int(42, span)),
]);
let item = item_from_value(rec).unwrap();
assert_eq!(
item,
HistoryItem {
command_line: "foo".to_string(),
id: None,
start_timestamp: Some(
DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")
.unwrap()
.to_utc()
),
hostname: Some("localhost".to_string()),
cwd: Some("/home/test".to_string()),
duration: Some(std::time::Duration::from_nanos(100_000_000)),
exit_status: Some(42),
session_id: None,
more_info: None
}
);
}
#[test]
fn test_item_from_value_record_extra_field() {
let span = Span::unknown();
let rec = new_record(&[
("command_line", Value::string("foo", span)),
("id_nonexistent", Value::int(1, span)),
]);
assert!(item_from_value(rec).is_err());
}
#[test]
fn test_item_from_value_record_bad_type() {
let span = Span::unknown();
let rec = new_record(&[
("command_line", Value::string("foo", span)),
("id", Value::string("one".to_string(), span)),
]);
assert!(item_from_value(rec).is_err());
}
fn new_record(rec: &[(&'static str, Value)]) -> Value {
let span = Span::unknown();
let rec = Record::from_raw_cols_vals(
rec.iter().map(|(col, _)| col.to_string()).collect(),
rec.iter().map(|(_, val)| val.clone()).collect(),
span,
span,
)
.unwrap();
Value::record(rec, span)
}
#[rstest]
#[case::no_backup(&["history.dat"], "history.dat.bak")]
#[case::backup_exists(&["history.dat", "history.dat.bak"], "history.dat.bak.1")]
#[case::multiple_backups_exists( &["history.dat", "history.dat.bak", "history.dat.bak.1"], "history.dat.bak.2")]
fn test_find_backup_path(#[case] existing: &[&str], #[case] want: &str) {
let dir = tempfile::tempdir().unwrap();
for name in existing {
std::fs::File::create_new(dir.path().join(name)).unwrap();
}
let got = find_backup_path(&dir.path().join("history.dat"), Span::test_data()).unwrap();
assert_eq!(got, dir.path().join(want))
}
#[test]
fn test_backup() {
let dir = tempfile::tempdir().unwrap();
let mut history = std::fs::File::create_new(dir.path().join("history.dat")).unwrap();
use std::io::Write;
write!(&mut history, "123").unwrap();
let want_bak_path = dir.path().join("history.dat.bak");
assert_eq!(
backup(&dir.path().join("history.dat"), Span::test_data()),
Ok(Some(want_bak_path.clone()))
);
let got_data = String::from_utf8(std::fs::read(want_bak_path).unwrap()).unwrap();
assert_eq!(got_data, "123");
}
#[test]
fn test_backup_no_file() {
let dir = tempfile::tempdir().unwrap();
let bak_path = backup(&dir.path().join("history.dat"), Span::test_data()).unwrap();
assert!(bak_path.is_none());
}
}

2
crates/nu-cli/src/commands/history/history_session.rs
View File

@ -8,7 +8,7 @@ impl Command for HistorySession {
"history session"
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"Get the command history session."
}

3
crates/nu-cli/src/commands/history/mod.rs
View File

@ -1,5 +1,8 @@
mod fields;
mod history_;
mod history_import;
mod history_session;
pub use history_::History;
pub use history_import::HistoryImport;
pub use history_session::HistorySession;

16
crates/nu-cli/src/commands/keybindings.rs
View File

@ -14,11 +14,11 @@ impl Command for Keybindings {
.input_output_types(vec![(Type::Nothing, Type::String)])
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"Keybindings related commands."
}
fn extra_usage(&self) -> &str {
fn extra_description(&self) -> &str {
r#"You must use one of the following subcommands. Using this command as-is will only produce this help message.
For more information on input and keybindings, check:
@ -36,16 +36,6 @@ For more information on input and keybindings, check:
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
Ok(Value::string(
get_full_help(
&Keybindings.signature(),
&Keybindings.examples(),
engine_state,
stack,
self.is_parser_keyword(),
),
call.head,
)
.into_pipeline_data())
Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data())
}
}

2
crates/nu-cli/src/commands/keybindings_default.rs
View File

@ -15,7 +15,7 @@ impl Command for KeybindingsDefault {
.input_output_types(vec![(Type::Nothing, Type::table())])
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"List default keybindings."
}

32
crates/nu-cli/src/commands/keybindings_list.rs
View File

@ -23,7 +23,7 @@ impl Command for KeybindingsList {
.category(Category::Platform)
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"List available options that can be used to create keybindings."
}
@ -49,22 +49,26 @@ impl Command for KeybindingsList {
fn run(
&self,
_engine_state: &EngineState,
_stack: &mut Stack,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let records = if call.named_len() == 0 {
let all_options = ["modifiers", "keycodes", "edits", "modes", "events"];
all_options
.iter()
.flat_map(|argument| get_records(argument, call.head))
.collect()
} else {
call.named_iter()
.flat_map(|(argument, _, _)| get_records(argument.item.as_str(), call.head))
.collect()
};
let all_options = ["modifiers", "keycodes", "edits", "modes", "events"];
let presence = all_options
.iter()
.map(|option| call.has_flag(engine_state, stack, option))
.collect::<Result<Vec<_>, ShellError>>()?;
let no_option_specified = presence.iter().all(|present| !*present);
let records = all_options
.iter()
.zip(presence)
.filter(|(_, present)| no_option_specified || *present)
.flat_map(|(option, _)| get_records(option, call.head))
.collect();
Ok(Value::list(records, call.head).into_pipeline_data())
}

63
crates/nu-cli/src/commands/keybindings_listen.rs
View File

@ -2,6 +2,7 @@ use crossterm::{
event::Event, event::KeyCode, event::KeyEvent, execute, terminal, QueueableCommand,
};
use nu_engine::command_prelude::*;
use nu_protocol::shell_error::io::IoError;
use std::io::{stdout, Write};
#[derive(Clone)]
@ -12,11 +13,11 @@ impl Command for KeybindingsListen {
"keybindings listen"
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"Get input from the user."
}
fn extra_usage(&self) -> &str {
fn extra_description(&self) -> &str {
"This is an internal debugging tool. For better output, try `input listen --types [key]`"
}
@ -39,7 +40,13 @@ impl Command for KeybindingsListen {
match print_events(engine_state) {
Ok(v) => Ok(v.into_pipeline_data()),
Err(e) => {
terminal::disable_raw_mode()?;
terminal::disable_raw_mode().map_err(|err| {
IoError::new_internal(
err.kind(),
"Could not disable raw mode",
nu_protocol::location!(),
)
})?;
Err(ShellError::GenericError {
error: "Error with input".into(),
msg: "".into(),
@ -63,8 +70,20 @@ impl Command for KeybindingsListen {
pub fn print_events(engine_state: &EngineState) -> Result<Value, ShellError> {
let config = engine_state.get_config();
stdout().flush()?;
terminal::enable_raw_mode()?;
stdout().flush().map_err(|err| {
IoError::new_internal(
err.kind(),
"Could not flush stdout",
nu_protocol::location!(),
)
})?;
terminal::enable_raw_mode().map_err(|err| {
IoError::new_internal(
err.kind(),
"Could not enable raw mode",
nu_protocol::location!(),
)
})?;
if config.use_kitty_protocol {
if let Ok(false) = crossterm::terminal::supports_keyboard_enhancement() {
@ -94,7 +113,9 @@ pub fn print_events(engine_state: &EngineState) -> Result<Value, ShellError> {
let mut stdout = std::io::BufWriter::new(std::io::stderr());
loop {
let event = crossterm::event::read()?;
let event = crossterm::event::read().map_err(|err| {
IoError::new_internal(err.kind(), "Could not read event", nu_protocol::location!())
})?;
if event == Event::Key(KeyCode::Esc.into()) {
break;
}
@ -113,9 +134,25 @@ pub fn print_events(engine_state: &EngineState) -> Result<Value, ShellError> {
_ => "".to_string(),
};
stdout.queue(crossterm::style::Print(o))?;
stdout.queue(crossterm::style::Print("\r\n"))?;
stdout.flush()?;
stdout.queue(crossterm::style::Print(o)).map_err(|err| {
IoError::new_internal(
err.kind(),
"Could not print output record",
nu_protocol::location!(),
)
})?;
stdout
.queue(crossterm::style::Print("\r\n"))
.map_err(|err| {
IoError::new_internal(
err.kind(),
"Could not print linebreak",
nu_protocol::location!(),
)
})?;
stdout.flush().map_err(|err| {
IoError::new_internal(err.kind(), "Could not flush", nu_protocol::location!())
})?;
}
if config.use_kitty_protocol {
@ -125,7 +162,13 @@ pub fn print_events(engine_state: &EngineState) -> Result<Value, ShellError> {
);
}
terminal::disable_raw_mode()?;
terminal::disable_raw_mode().map_err(|err| {
IoError::new_internal(
err.kind(),
"Could not disable raw mode",
nu_protocol::location!(),
)
})?;
Ok(Value::nothing(Span::unknown()))
}

2
crates/nu-cli/src/commands/mod.rs
View File

@ -7,7 +7,7 @@ mod keybindings_list;
mod keybindings_listen;
pub use commandline::{Commandline, CommandlineEdit, CommandlineGetCursor, CommandlineSetCursor};
pub use history::{History, HistorySession};
pub use history::{History, HistoryImport, HistorySession};
pub use keybindings::Keybindings;
pub use keybindings_default::KeybindingsDefault;
pub use keybindings_list::KeybindingsList;

97
crates/nu-cli/src/completions/attribute_completions.rs Normal file
View File

@ -0,0 +1,97 @@
use super::{completion_options::NuMatcher, SemanticSuggestion};
use crate::{
completions::{Completer, CompletionOptions},
SuggestionKind,
};
use nu_protocol::{
engine::{Stack, StateWorkingSet},
Span,
};
use reedline::Suggestion;
pub struct AttributeCompletion;
pub struct AttributableCompletion;
impl Completer for AttributeCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
_stack: &Stack,
_prefix: &[u8],
span: Span,
offset: usize,
_pos: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
let partial = working_set.get_span_contents(span);
let mut matcher = NuMatcher::new(String::from_utf8_lossy(partial), options.clone());
let attr_commands = working_set.find_commands_by_predicate(
|s| {
s.strip_prefix(b"attr ")
.map(String::from_utf8_lossy)
.is_some_and(|name| matcher.matches(&name))
},
true,
);
for (name, desc, ty) in attr_commands {
let name = name.strip_prefix(b"attr ").unwrap_or(&name);
matcher.add_semantic_suggestion(SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(name).into_owned(),
description: desc,
style: None,
extra: None,
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
append_whitespace: false,
},
kind: Some(SuggestionKind::Command(ty)),
});
}
matcher.results()
}
}
impl Completer for AttributableCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
_stack: &Stack,
_prefix: &[u8],
span: Span,
offset: usize,
_pos: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
let partial = working_set.get_span_contents(span);
let mut matcher = NuMatcher::new(String::from_utf8_lossy(partial), options.clone());
for s in ["def", "extern", "export def", "export extern"] {
let decl_id = working_set
.find_decl(s.as_bytes())
.expect("internal error, builtin declaration not found");
let cmd = working_set.get_decl(decl_id);
matcher.add_semantic_suggestion(SemanticSuggestion {
suggestion: Suggestion {
value: cmd.name().into(),
description: Some(cmd.description().into()),
style: None,
extra: None,
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
append_whitespace: false,
},
kind: Some(SuggestionKind::Command(cmd.command_type())),
});
}
matcher.results()
}
}

36
crates/nu-cli/src/completions/base.rs
View File

@ -1,50 +1,23 @@
use crate::completions::{CompletionOptions, SortBy};
use crate::completions::CompletionOptions;
use nu_protocol::{
engine::{Stack, StateWorkingSet},
levenshtein_distance, Span,
Span,
};
use reedline::Suggestion;
// Completer trait represents the three stages of the completion
// fetch, filter and sort
pub trait Completer {
/// Fetch, filter, and sort completions
#[allow(clippy::too_many_arguments)]
fn fetch(
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
prefix: Vec<u8>,
prefix: &[u8],
span: Span,
offset: usize,
pos: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion>;
fn get_sort_by(&self) -> SortBy {
SortBy::Ascending
}
fn sort(&self, items: Vec<SemanticSuggestion>, prefix: Vec<u8>) -> Vec<SemanticSuggestion> {
let prefix_str = String::from_utf8_lossy(&prefix).to_string();
let mut filtered_items = items;
// Sort items
match self.get_sort_by() {
SortBy::LevenshteinDistance => {
filtered_items.sort_by(|a, b| {
let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value);
let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value);
a_distance.cmp(&b_distance)
});
}
SortBy::Ascending => {
filtered_items.sort_by(|a, b| a.suggestion.value.cmp(&b.suggestion.value));
}
SortBy::None => {}
};
filtered_items
}
}
#[derive(Debug, Default, PartialEq)]
@ -58,6 +31,7 @@ pub struct SemanticSuggestion {
pub enum SuggestionKind {
Command(nu_protocol::engine::CommandType),
Type(nu_protocol::Type),
Module,
}
impl From<Suggestion> for SemanticSuggestion {

97
crates/nu-cli/src/completions/cell_path_completions.rs Normal file
View File

@ -0,0 +1,97 @@
use crate::completions::{Completer, CompletionOptions, SemanticSuggestion, SuggestionKind};
use nu_engine::{column::get_columns, eval_variable};
use nu_protocol::{
ast::{Expr, FullCellPath, PathMember},
engine::{Stack, StateWorkingSet},
eval_const::eval_constant,
Span, Value,
};
use reedline::Suggestion;
use super::completion_options::NuMatcher;
pub struct CellPathCompletion<'a> {
pub full_cell_path: &'a FullCellPath,
}
impl Completer for CellPathCompletion<'_> {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
_prefix: &[u8],
_span: Span,
offset: usize,
_pos: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
// empty tail is already handled as variable names completion
let Some((prefix_member, path_members)) = self.full_cell_path.tail.split_last() else {
return vec![];
};
let (mut prefix_str, span) = match prefix_member {
PathMember::String { val, span, .. } => (val.clone(), span),
PathMember::Int { val, span, .. } => (val.to_string(), span),
};
// strip the placeholder
prefix_str.pop();
let true_end = std::cmp::max(span.start, span.end - 1);
let span = Span::new(span.start, true_end);
let current_span = reedline::Span {
start: span.start - offset,
end: true_end - offset,
};
let mut matcher = NuMatcher::new(prefix_str, options.clone());
// evaluate the head expression to get its value
let value = if let Expr::Var(var_id) = self.full_cell_path.head.expr {
working_set
.get_variable(var_id)
.const_val
.to_owned()
.or_else(|| eval_variable(working_set.permanent_state, stack, var_id, span).ok())
} else {
eval_constant(working_set, &self.full_cell_path.head).ok()
}
.unwrap_or_default();
for suggestion in nested_suggestions(&value, path_members, current_span) {
matcher.add_semantic_suggestion(suggestion);
}
matcher.results()
}
}
// Find recursively the values for cell_path
fn nested_suggestions(
val: &Value,
path_members: &[PathMember],
current_span: reedline::Span,
) -> Vec<SemanticSuggestion> {
let value = val
.clone()
.follow_cell_path(path_members, false)
.unwrap_or_default();
let kind = SuggestionKind::Type(value.get_type());
let str_to_suggestion = |s: String| SemanticSuggestion {
suggestion: Suggestion {
value: s,
span: current_span,
..Suggestion::default()
},
kind: Some(kind.to_owned()),
};
match value {
Value::Record { val, .. } => val
.columns()
.map(|s| str_to_suggestion(s.to_string()))
.collect(),
Value::List { vals, .. } => get_columns(vals.as_slice())
.into_iter()
.map(str_to_suggestion)
.collect(),
_ => vec![],
}
}

196
crates/nu-cli/src/completions/command_completions.rs
View File

@ -1,5 +1,7 @@
use std::collections::HashMap;
use crate::{
completions::{Completer, CompletionOptions, MatchAlgorithm, SortBy},
completions::{Completer, CompletionOptions},
SuggestionKind,
};
use nu_parser::FlatShape;
@ -9,7 +11,7 @@ use nu_protocol::{
};
use reedline::Suggestion;
use super::SemanticSuggestion;
use super::{completion_options::NuMatcher, SemanticSuggestion};
pub struct CommandCompletion {
flattened: Vec<(Span, FlatShape)>,
@ -33,15 +35,15 @@ impl CommandCompletion {
fn external_command_completion(
&self,
working_set: &StateWorkingSet,
prefix: &str,
match_algorithm: MatchAlgorithm,
) -> Vec<String> {
let mut executables = vec![];
sugg_span: reedline::Span,
matched_internal: impl Fn(&str) -> bool,
matcher: &mut NuMatcher<String>,
) -> HashMap<String, SemanticSuggestion> {
let mut suggs = HashMap::new();
// os agnostic way to get the PATH env var
let paths = working_set.permanent_state.get_path_env_var();
let paths = working_set.permanent_state.get_env_var_insensitive("path");
if let Some(paths) = paths {
if let Some((_, paths)) = paths {
if let Ok(paths) = paths.as_list() {
for path in paths {
let path = path.coerce_str().unwrap_or_default();
@ -51,25 +53,41 @@ impl CommandCompletion {
if working_set
.permanent_state
.config
.max_external_completion_results
> executables.len() as i64
&& !executables.contains(
&item
.path()
.file_name()
.map(|x| x.to_string_lossy().to_string())
.unwrap_or_default(),
)
&& matches!(
item.path().file_name().map(|x| match_algorithm
.matches_str(&x.to_string_lossy(), prefix)),
Some(true)
)
&& is_executable::is_executable(item.path())
.completions
.external
.max_results
<= suggs.len() as i64
{
if let Ok(name) = item.file_name().into_string() {
executables.push(name);
}
break;
}
let Ok(name) = item.file_name().into_string() else {
continue;
};
let value = if matched_internal(&name) {
format!("^{}", name)
} else {
name.clone()
};
if suggs.contains_key(&value) {
continue;
}
if matcher.matches(&name) && is_executable::is_executable(item.path()) {
// If there's an internal command with the same name, adds ^cmd to the
// matcher so that both the internal and external command are included
matcher.add(&name, value.clone());
suggs.insert(
value.clone(),
SemanticSuggestion {
suggestion: Suggestion {
value,
span: sugg_span,
append_whitespace: true,
..Default::default()
},
// TODO: is there a way to create a test?
kind: None,
},
);
}
}
}
@ -77,7 +95,7 @@ impl CommandCompletion {
}
}
executables
suggs
}
fn complete_commands(
@ -86,73 +104,59 @@ impl CommandCompletion {
span: Span,
offset: usize,
find_externals: bool,
match_algorithm: MatchAlgorithm,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
let partial = working_set.get_span_contents(span);
let mut matcher = NuMatcher::new(String::from_utf8_lossy(partial), options.clone());
let filter_predicate = |command: &[u8]| match_algorithm.matches_u8(command, partial);
let sugg_span = reedline::Span::new(span.start - offset, span.end - offset);
let mut results = working_set
.find_commands_by_predicate(filter_predicate, true)
.into_iter()
.map(move |x| SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(&x.0).to_string(),
description: x.1,
style: None,
extra: None,
span: reedline::Span::new(span.start - offset, span.end - offset),
append_whitespace: true,
},
kind: Some(SuggestionKind::Command(x.2)),
})
.collect::<Vec<_>>();
let partial = working_set.get_span_contents(span);
let partial = String::from_utf8_lossy(partial).to_string();
if find_externals {
let results_external = self
.external_command_completion(working_set, &partial, match_algorithm)
.into_iter()
.map(move |x| SemanticSuggestion {
let mut internal_suggs = HashMap::new();
let filtered_commands = working_set.find_commands_by_predicate(
|name| {
let name = String::from_utf8_lossy(name);
matcher.add(&name, name.to_string())
},
true,
);
for (name, description, typ) in filtered_commands {
let name = String::from_utf8_lossy(&name);
internal_suggs.insert(
name.to_string(),
SemanticSuggestion {
suggestion: Suggestion {
value: x,
description: None,
style: None,
extra: None,
span: reedline::Span::new(span.start - offset, span.end - offset),
value: name.to_string(),
description,
span: sugg_span,
append_whitespace: true,
..Suggestion::default()
},
// TODO: is there a way to create a test?
kind: None,
});
let results_strings: Vec<String> =
results.iter().map(|x| x.suggestion.value.clone()).collect();
for external in results_external {
if results_strings.contains(&external.suggestion.value) {
results.push(SemanticSuggestion {
suggestion: Suggestion {
value: format!("^{}", external.suggestion.value),
description: None,
style: None,
extra: None,
span: external.suggestion.span,
append_whitespace: true,
},
kind: external.kind,
})
} else {
results.push(external)
}
}
results
} else {
results
kind: Some(SuggestionKind::Command(typ)),
},
);
}
let mut external_suggs = if find_externals {
self.external_command_completion(
working_set,
sugg_span,
|name| internal_suggs.contains_key(name),
&mut matcher,
)
} else {
HashMap::new()
};
let mut res = Vec::new();
for cmd_name in matcher.results() {
if let Some(sugg) = internal_suggs
.remove(&cmd_name)
.or_else(|| external_suggs.remove(&cmd_name))
{
res.push(sugg);
}
}
res
}
}
@ -161,7 +165,7 @@ impl Completer for CommandCompletion {
&mut self,
working_set: &StateWorkingSet,
_stack: &Stack,
_prefix: Vec<u8>,
_prefix: &[u8],
span: Span,
offset: usize,
pos: usize,
@ -191,7 +195,7 @@ impl Completer for CommandCompletion {
Span::new(last.0.start, pos),
offset,
false,
options.match_algorithm,
options,
)
} else {
vec![]
@ -202,7 +206,7 @@ impl Completer for CommandCompletion {
}
let config = working_set.get_config();
let commands = if matches!(self.flat_shape, nu_parser::FlatShape::External)
if matches!(self.flat_shape, nu_parser::FlatShape::External)
|| matches!(self.flat_shape, nu_parser::FlatShape::InternalCall(_))
|| ((span.end - span.start) == 0)
|| is_passthrough_command(working_set.delta.get_file_contents())
@ -216,18 +220,12 @@ impl Completer for CommandCompletion {
working_set,
span,
offset,
config.enable_external_completion,
options.match_algorithm,
config.completions.external.enable,
options,
)
} else {
vec![]
};
subcommands.into_iter().chain(commands).collect::<Vec<_>>()
}
fn get_sort_by(&self) -> SortBy {
SortBy::LevenshteinDistance
}
}
}

733
crates/nu-cli/src/completions/completer.rs
View File

@ -1,11 +1,14 @@
use crate::completions::{
CommandCompletion, Completer, CompletionOptions, CustomCompletion, DirectoryCompletion,
DotNuCompletion, FileCompletion, FlagCompletion, VariableCompletion,
AttributableCompletion, AttributeCompletion, CellPathCompletion, CommandCompletion, Completer,
CompletionOptions, CustomCompletion, DirectoryCompletion, DotNuCompletion, FileCompletion,
FlagCompletion, OperatorCompletion, VariableCompletion,
};
use log::debug;
use nu_color_config::{color_record_to_nustyle, lookup_ansi_color_style};
use nu_engine::eval_block;
use nu_parser::{flatten_pipeline_element, parse, FlatShape};
use nu_parser::{flatten_expression, parse, FlatShape};
use nu_protocol::{
ast::{Expr, Expression, FindMapResult, Traverse},
debugger::WithoutDebug,
engine::{Closure, EngineState, Stack, StateWorkingSet},
PipelineData, Span, Value,
@ -15,6 +18,73 @@ use std::{str, sync::Arc};
use super::base::{SemanticSuggestion, SuggestionKind};
/// Used as the function `f` in find_map Traverse
///
/// returns the inner-most pipeline_element of interest
/// i.e. the one that contains given position and needs completion
fn find_pipeline_element_by_position<'a>(
expr: &'a Expression,
working_set: &'a StateWorkingSet,
pos: usize,
) -> FindMapResult<&'a Expression> {
// skip the entire expression if the position is not in it
if !expr.span.contains(pos) {
return FindMapResult::Stop;
}
let closure = |expr: &'a Expression| find_pipeline_element_by_position(expr, working_set, pos);
match &expr.expr {
Expr::Call(call) => call
.arguments
.iter()
.find_map(|arg| arg.expr().and_then(|e| e.find_map(working_set, &closure)))
// if no inner call/external_call found, then this is the inner-most one
.or(Some(expr))
.map(FindMapResult::Found)
.unwrap_or_default(),
// TODO: clear separation of internal/external completion logic
Expr::ExternalCall(head, arguments) => arguments
.iter()
.find_map(|arg| arg.expr().find_map(working_set, &closure))
.or(head.as_ref().find_map(working_set, &closure))
.or(Some(expr))
.map(FindMapResult::Found)
.unwrap_or_default(),
// complete the operator
Expr::BinaryOp(lhs, _, rhs) => lhs
.find_map(working_set, &closure)
.or(rhs.find_map(working_set, &closure))
.or(Some(expr))
.map(FindMapResult::Found)
.unwrap_or_default(),
Expr::FullCellPath(fcp) => fcp
.head
.find_map(working_set, &closure)
.or(Some(expr))
.map(FindMapResult::Found)
.unwrap_or_default(),
Expr::Var(_) => FindMapResult::Found(expr),
Expr::AttributeBlock(ab) => ab
.attributes
.iter()
.map(|attr| &attr.expr)
.chain(Some(ab.item.as_ref()))
.find_map(|expr| expr.find_map(working_set, &closure))
.or(Some(expr))
.map(FindMapResult::Found)
.unwrap_or_default(),
_ => FindMapResult::Continue,
}
}
/// Before completion, an additional character `a` is added to the source as a placeholder for correct parsing results.
/// This function helps to strip it
fn strip_placeholder<'a>(working_set: &'a StateWorkingSet, span: &Span) -> (Span, &'a [u8]) {
let new_end = std::cmp::max(span.end - 1, span.start);
let new_span = Span::new(span.start, new_end);
let prefix = working_set.get_span_contents(new_span);
(new_span, prefix)
}
#[derive(Clone)]
pub struct NuCompleter {
engine_state: Arc<EngineState>,
@ -25,7 +95,7 @@ impl NuCompleter {
pub fn new(engine_state: Arc<EngineState>, stack: Arc<Stack>) -> Self {
Self {
engine_state,
stack: Stack::with_parent(stack).reset_out_dest().capture(),
stack: Stack::with_parent(stack).reset_out_dest().collect_value(),
}
}
@ -33,12 +103,34 @@ impl NuCompleter {
self.completion_helper(line, pos)
}
fn variable_names_completion_helper(
&self,
working_set: &StateWorkingSet,
span: Span,
offset: usize,
) -> Vec<SemanticSuggestion> {
let (new_span, prefix) = strip_placeholder(working_set, &span);
if !prefix.starts_with(b"$") {
return vec![];
}
let mut variable_names_completer = VariableCompletion {};
self.process_completion(
&mut variable_names_completer,
working_set,
prefix,
new_span,
offset,
// pos is not required
0,
)
}
// Process the completion for a given completer
fn process_completion<T: Completer>(
&self,
completer: &mut T,
working_set: &StateWorkingSet,
prefix: Vec<u8>,
prefix: &[u8],
new_span: Span,
offset: usize,
pos: usize,
@ -46,26 +138,26 @@ impl NuCompleter {
let config = self.engine_state.get_config();
let options = CompletionOptions {
case_sensitive: config.case_sensitive_completions,
match_algorithm: config.completion_algorithm.into(),
case_sensitive: config.completions.case_sensitive,
match_algorithm: config.completions.algorithm.into(),
sort: config.completions.sort,
..Default::default()
};
// Fetch
let mut suggestions = completer.fetch(
debug!(
"process_completion: prefix: {}, new_span: {new_span:?}, offset: {offset}, pos: {pos}",
String::from_utf8_lossy(prefix)
);
completer.fetch(
working_set,
&self.stack,
prefix.clone(),
prefix,
new_span,
offset,
pos,
&options,
);
// Sort
suggestions = completer.sort(suggestions, prefix);
suggestions
)
}
fn external_completion(
@ -104,18 +196,24 @@ impl NuCompleter {
);
match result.and_then(|data| data.into_value(span)) {
Ok(value) => {
if let Value::List { vals, .. } = value {
let result =
map_value_completions(vals.iter(), Span::new(span.start, span.end), offset);
return Some(result);
}
Ok(Value::List { vals, .. }) => {
let result =
map_value_completions(vals.iter(), Span::new(span.start, span.end), offset);
Some(result)
}
Ok(Value::Nothing { .. }) => None,
Ok(value) => {
log::error!(
"External completer returned invalid value of type {}",
value.get_type().to_string()
);
Some(vec![])
}
Err(err) => {
log::error!("failed to eval completer block: {err}");
Some(vec![])
}
Err(err) => println!("failed to eval completer block: {err}"),
}
None
}
fn completion_helper(&mut self, line: &str, pos: usize) -> Vec<SemanticSuggestion> {
@ -133,245 +231,299 @@ impl NuCompleter {
let config = self.engine_state.get_config();
let output = parse(&mut working_set, Some("completer"), line.as_bytes(), false);
let block = parse(&mut working_set, Some("completer"), line.as_bytes(), false);
let Some(element_expression) = block.find_map(&working_set, &|expr: &Expression| {
find_pipeline_element_by_position(expr, &working_set, pos)
}) else {
return vec![];
};
for pipeline in &output.pipelines {
for pipeline_element in &pipeline.elements {
let flattened = flatten_pipeline_element(&working_set, pipeline_element);
let mut spans: Vec<String> = vec![];
match &element_expression.expr {
Expr::Var(_) => {
return self.variable_names_completion_helper(
&working_set,
element_expression.span,
fake_offset,
);
}
Expr::FullCellPath(full_cell_path) => {
// e.g. `$e<tab>` parsed as FullCellPath
if full_cell_path.tail.is_empty() {
return self.variable_names_completion_helper(
&working_set,
element_expression.span,
fake_offset,
);
} else {
let mut cell_path_completer = CellPathCompletion { full_cell_path };
return self.process_completion(
&mut cell_path_completer,
&working_set,
&[],
element_expression.span,
fake_offset,
pos,
);
}
}
_ => (),
}
for (flat_idx, flat) in flattened.iter().enumerate() {
let is_passthrough_command = spans
.first()
.filter(|content| content.as_str() == "sudo" || content.as_str() == "doas")
.is_some();
// Read the current spam to string
let current_span = working_set.get_span_contents(flat.0).to_vec();
let current_span_str = String::from_utf8_lossy(&current_span);
let flattened = flatten_expression(&working_set, element_expression);
let mut spans: Vec<String> = vec![];
let is_last_span = pos >= flat.0.start && pos < flat.0.end;
for (flat_idx, (span, shape)) in flattened.iter().enumerate() {
let is_passthrough_command = spans
.first()
.filter(|content| content.as_str() == "sudo" || content.as_str() == "doas")
.is_some();
// Skip the last 'a' as span item
if is_last_span {
let offset = pos - flat.0.start;
if offset == 0 {
spans.push(String::new())
} else {
let mut current_span_str = current_span_str.to_string();
current_span_str.remove(offset);
spans.push(current_span_str);
}
// Read the current span to string
let current_span = working_set.get_span_contents(*span);
let current_span_str = String::from_utf8_lossy(current_span);
let is_last_span = span.contains(pos);
// Skip the last 'a' as span item
if is_last_span {
let offset = pos - span.start;
if offset == 0 {
spans.push(String::new())
} else {
let mut current_span_str = current_span_str.to_string();
current_span_str.remove(offset);
spans.push(current_span_str);
}
} else {
spans.push(current_span_str.to_string());
}
// Complete based on the last span
if is_last_span {
// Create a new span
let new_span = Span::new(span.start, span.end - 1);
// Parses the prefix. Completion should look up to the cursor position, not after.
let index = pos - span.start;
let prefix = &current_span[..index];
if let Expr::AttributeBlock(ab) = &element_expression.expr {
let last_attr = ab.attributes.last().expect("at least one attribute");
if let Expr::Garbage = last_attr.expr.expr {
return self.process_completion(
&mut AttributeCompletion,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
} else {
spans.push(current_span_str.to_string());
}
// Complete based on the last span
if is_last_span {
// Context variables
let most_left_var =
most_left_variable(flat_idx, &working_set, flattened.clone());
// Create a new span
let new_span = Span::new(flat.0.start, flat.0.end - 1);
// Parses the prefix. Completion should look up to the cursor position, not after.
let mut prefix = working_set.get_span_contents(flat.0).to_vec();
let index = pos - flat.0.start;
prefix.drain(index..);
// Variables completion
if prefix.starts_with(b"$") || most_left_var.is_some() {
let mut completer =
VariableCompletion::new(most_left_var.unwrap_or((vec![], vec![])));
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
// Flags completion
if prefix.starts_with(b"-") {
// Try to complete flag internally
let mut completer = FlagCompletion::new(pipeline_element.expr.clone());
let result = self.process_completion(
&mut completer,
&working_set,
prefix.clone(),
new_span,
fake_offset,
pos,
);
if !result.is_empty() {
return result;
}
// We got no results for internal completion
// now we can check if external completer is set and use it
if let Some(closure) = config.external_completer.as_ref() {
if let Some(external_result) =
self.external_completion(closure, &spans, fake_offset, new_span)
{
return external_result;
}
}
}
// specially check if it is currently empty - always complete commands
if (is_passthrough_command && flat_idx == 1)
|| (flat_idx == 0 && working_set.get_span_contents(new_span).is_empty())
{
let mut completer = CommandCompletion::new(
flattened.clone(),
// flat_idx,
FlatShape::String,
true,
);
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
// Completions that depends on the previous expression (e.g: use, source-env)
if (is_passthrough_command && flat_idx > 1) || flat_idx > 0 {
if let Some(previous_expr) = flattened.get(flat_idx - 1) {
// Read the content for the previous expression
let prev_expr_str =
working_set.get_span_contents(previous_expr.0).to_vec();
// Completion for .nu files
if prev_expr_str == b"use"
|| prev_expr_str == b"overlay use"
|| prev_expr_str == b"source-env"
{
let mut completer = DotNuCompletion::new();
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
} else if prev_expr_str == b"ls" {
let mut completer = FileCompletion::new();
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
}
}
// Match other types
match &flat.1 {
FlatShape::Custom(decl_id) => {
let mut completer = CustomCompletion::new(
self.stack.clone(),
*decl_id,
initial_line,
);
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
FlatShape::Directory => {
let mut completer = DirectoryCompletion::new();
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
FlatShape::Filepath | FlatShape::GlobPattern => {
let mut completer = FileCompletion::new();
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
flat_shape => {
let mut completer = CommandCompletion::new(
flattened.clone(),
// flat_idx,
flat_shape.clone(),
false,
);
let mut out: Vec<_> = self.process_completion(
&mut completer,
&working_set,
prefix.clone(),
new_span,
fake_offset,
pos,
);
if !out.is_empty() {
return out;
}
// Try to complete using an external completer (if set)
if let Some(closure) = config.external_completer.as_ref() {
if let Some(external_result) = self.external_completion(
closure,
&spans,
fake_offset,
new_span,
) {
return external_result;
}
}
// Check for file completion
let mut completer = FileCompletion::new();
out = self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
if !out.is_empty() {
return out;
}
}
};
return self.process_completion(
&mut AttributableCompletion,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
}
// Flags completion
if prefix.starts_with(b"-") {
// Try to complete flag internally
let mut completer = FlagCompletion::new(element_expression.clone());
let result = self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
if !result.is_empty() {
return result;
}
// We got no results for internal completion
// now we can check if external completer is set and use it
if let Some(closure) = config.completions.external.completer.as_ref() {
if let Some(external_result) =
self.external_completion(closure, &spans, fake_offset, new_span)
{
return external_result;
}
}
}
// specially check if it is currently empty - always complete commands
if (is_passthrough_command && flat_idx == 1)
|| (flat_idx == 0 && working_set.get_span_contents(new_span).is_empty())
{
let mut completer = CommandCompletion::new(
flattened.clone(),
// flat_idx,
FlatShape::String,
true,
);
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
// Completions that depends on the previous expression (e.g: use, source-env)
if (is_passthrough_command && flat_idx > 1) || flat_idx > 0 {
if let Some(previous_expr) = flattened.get(flat_idx - 1) {
// Read the content for the previous expression
let prev_expr_str = working_set.get_span_contents(previous_expr.0).to_vec();
// Completion for .nu files
if prev_expr_str == b"use"
|| prev_expr_str == b"overlay use"
|| prev_expr_str == b"source-env"
{
let mut completer = DotNuCompletion::new();
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
} else if prev_expr_str == b"ls" {
let mut completer = FileCompletion::new();
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
} else if matches!(
previous_expr.1,
FlatShape::Float
| FlatShape::Int
| FlatShape::String
| FlatShape::List
| FlatShape::Bool
| FlatShape::Variable(_)
) {
let mut completer = OperatorCompletion::new(element_expression.clone());
let operator_suggestion = self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
if !operator_suggestion.is_empty() {
return operator_suggestion;
}
}
}
}
// Match other types
match shape {
FlatShape::Custom(decl_id) => {
let mut completer = CustomCompletion::new(
self.stack.clone(),
*decl_id,
initial_line,
FileCompletion::new(),
);
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
FlatShape::Directory => {
let mut completer = DirectoryCompletion::new();
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
FlatShape::Filepath | FlatShape::GlobPattern => {
let mut completer = FileCompletion::new();
return self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
}
flat_shape => {
let mut completer = CommandCompletion::new(
flattened.clone(),
// flat_idx,
flat_shape.clone(),
false,
);
let mut out: Vec<_> = self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
if !out.is_empty() {
return out;
}
// Try to complete using an external completer (if set)
if let Some(closure) = config.completions.external.completer.as_ref() {
if let Some(external_result) =
self.external_completion(closure, &spans, fake_offset, new_span)
{
return external_result;
}
}
// Check for file completion
let mut completer = FileCompletion::new();
out = self.process_completion(
&mut completer,
&working_set,
prefix,
new_span,
fake_offset,
pos,
);
if !out.is_empty() {
return out;
}
}
};
}
}
@ -388,56 +540,6 @@ impl ReedlineCompleter for NuCompleter {
}
}
// reads the most left variable returning it's name (e.g: $myvar)
// and the depth (a.b.c)
fn most_left_variable(
idx: usize,
working_set: &StateWorkingSet<'_>,
flattened: Vec<(Span, FlatShape)>,
) -> Option<(Vec<u8>, Vec<Vec<u8>>)> {
// Reverse items to read the list backwards and truncate
// because the only items that matters are the ones before the current index
let mut rev = flattened;
rev.truncate(idx);
rev = rev.into_iter().rev().collect();
// Store the variables and sub levels found and reverse to correct order
let mut variables_found: Vec<Vec<u8>> = vec![];
let mut found_var = false;
for item in rev.clone() {
let result = working_set.get_span_contents(item.0).to_vec();
match item.1 {
FlatShape::Variable(_) => {
variables_found.push(result);
found_var = true;
break;
}
FlatShape::String => {
variables_found.push(result);
}
_ => {
break;
}
}
}
// If most left var was not found
if !found_var {
return None;
}
// Reverse the order back
variables_found = variables_found.into_iter().rev().collect();
// Extract the variable and the sublevels
let var = variables_found.first().unwrap_or(&vec![]).to_vec();
let sublevels: Vec<Vec<u8>> = variables_found.into_iter().skip(1).collect();
Some((var, sublevels))
}
pub fn map_value_completions<'a>(
list: impl Iterator<Item = &'a Value>,
span: Span,
@ -449,14 +551,11 @@ pub fn map_value_completions<'a>(
return Some(SemanticSuggestion {
suggestion: Suggestion {
value: s,
description: None,
style: None,
extra: None,
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
append_whitespace: false,
..Suggestion::default()
},
kind: Some(SuggestionKind::Type(x.get_type())),
});
@ -466,14 +565,11 @@ pub fn map_value_completions<'a>(
if let Ok(record) = x.as_record() {
let mut suggestion = Suggestion {
value: String::from(""), // Initialize with empty string
description: None,
style: None,
extra: None,
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
append_whitespace: false,
..Suggestion::default()
};
// Iterate the cols looking for `value` and `description`
@ -542,6 +638,11 @@ mod completer_tests {
let mut completer = NuCompleter::new(engine_state.into(), Arc::new(Stack::new()));
let dataset = [
("1 bit-sh", true, "b", vec!["bit-shl", "bit-shr"]),
("1.0 bit-sh", false, "b", vec![]),
("1 m", true, "m", vec!["mod"]),
("1.0 m", true, "m", vec!["mod"]),
("\"a\" s", true, "s", vec!["starts-with"]),
("sudo", false, "", Vec::new()),
("sudo l", true, "l", vec!["ls", "let", "lines", "loop"]),
(" sudo", false, "", Vec::new()),

284
crates/nu-cli/src/completions/completion_common.rs
View File

@ -1,72 +1,115 @@
use crate::completions::{matches, CompletionOptions};
use super::{completion_options::NuMatcher, MatchAlgorithm};
use crate::completions::CompletionOptions;
use nu_ansi_term::Style;
use nu_engine::env_to_string;
use nu_path::home_dir;
use nu_path::dots::expand_ndots;
use nu_path::{expand_to_real_path, home_dir};
use nu_protocol::{
engine::{EngineState, Stack, StateWorkingSet},
Span,
};
use nu_utils::get_ls_colors;
use std::path::{
is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR,
};
use nu_utils::IgnoreCaseExt;
use std::path::{is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP};
#[derive(Clone, Default)]
pub struct PathBuiltFromString {
cwd: PathBuf,
parts: Vec<String>,
isdir: bool,
}
/// Recursively goes through paths that match a given `partial`.
/// built: State struct for a valid matching path built so far.
///
/// `isdir`: whether the current partial path has a trailing slash.
/// Parsing a path string into a pathbuf loses that bit of information.
///
/// want_directory: Whether we want only directories as completion matches.
/// Some commands like `cd` can only be run on directories whereas others
/// like `ls` can be run on regular files as well.
fn complete_rec(
partial: &[&str],
built: &PathBuiltFromString,
cwd: &Path,
built_paths: &[PathBuiltFromString],
options: &CompletionOptions,
dir: bool,
want_directory: bool,
isdir: bool,
) -> Vec<PathBuiltFromString> {
let mut completions = vec![];
if let Some((&base, rest)) = partial.split_first() {
if (base == "." || base == "..") && (isdir || !rest.is_empty()) {
let mut built = built.clone();
built.parts.push(base.to_string());
built.isdir = true;
return complete_rec(rest, &built, cwd, options, dir, isdir);
if base.chars().all(|c| c == '.') && (isdir || !rest.is_empty()) {
let built_paths: Vec<_> = built_paths
.iter()
.map(|built| {
let mut built = built.clone();
built.parts.push(base.to_string());
built.isdir = true;
built
})
.collect();
return complete_rec(rest, &built_paths, options, want_directory, isdir);
}
}
let mut built_path = cwd.to_path_buf();
for part in &built.parts {
built_path.push(part);
let prefix = partial.first().unwrap_or(&"");
let mut matcher = NuMatcher::new(prefix, options.clone());
for built in built_paths {
let mut path = built.cwd.clone();
for part in &built.parts {
path.push(part);
}
let Ok(result) = path.read_dir() else {
continue;
};
for entry in result.filter_map(|e| e.ok()) {
let entry_name = entry.file_name().to_string_lossy().into_owned();
let entry_isdir = entry.path().is_dir() && !entry.path().is_symlink();
let mut built = built.clone();
built.parts.push(entry_name.clone());
built.isdir = entry_isdir;
if !want_directory || entry_isdir {
matcher.add(entry_name.clone(), (entry_name, built));
}
}
}
let Ok(result) = built_path.read_dir() else {
return completions;
};
for entry in result.filter_map(|e| e.ok()) {
let entry_name = entry.file_name().to_string_lossy().into_owned();
let entry_isdir = entry.path().is_dir();
let mut built = built.clone();
built.parts.push(entry_name.clone());
built.isdir = entry_isdir;
if !dir || entry_isdir {
match partial.split_first() {
Some((base, rest)) => {
if matches(base, &entry_name, options) {
if !rest.is_empty() || isdir {
completions
.extend(complete_rec(rest, &built, cwd, options, dir, isdir));
} else {
completions.push(built);
}
}
}
None => {
let mut completions = vec![];
for (entry_name, built) in matcher.results() {
match partial.split_first() {
Some((base, rest)) => {
// We use `isdir` to confirm that the current component has
// at least one next component or a slash.
// Serves as confirmation to ignore longer completions for
// components in between.
if !rest.is_empty() || isdir {
completions.extend(complete_rec(
rest,
&[built],
options,
want_directory,
isdir,
));
} else {
completions.push(built);
}
// For https://github.com/nushell/nushell/issues/13204
if isdir && options.match_algorithm == MatchAlgorithm::Prefix {
let exact_match = if options.case_sensitive {
entry_name.eq(base)
} else {
entry_name.to_folded_case().eq(&base.to_folded_case())
};
if exact_match {
break;
}
}
}
None => {
completions.push(built);
}
}
}
@ -81,16 +124,16 @@ enum OriginalCwd {
}
impl OriginalCwd {
fn apply(&self, mut p: PathBuiltFromString) -> String {
fn apply(&self, mut p: PathBuiltFromString, path_separator: char) -> String {
match self {
Self::None => {}
Self::Home => p.parts.insert(0, "~".to_string()),
Self::Prefix(s) => p.parts.insert(0, s.clone()),
};
let mut ret = p.parts.join(MAIN_SEPARATOR_STR);
let mut ret = p.parts.join(&path_separator.to_string());
if p.isdir {
ret.push(SEP);
ret.push(path_separator);
}
ret
}
@ -110,29 +153,58 @@ fn surround_remove(partial: &str) -> String {
partial.to_string()
}
pub struct FileSuggestion {
pub span: nu_protocol::Span,
pub path: String,
pub style: Option<Style>,
}
/// # Parameters
/// * `cwds` - A list of directories in which to search. The only reason this isn't a single string
/// is because dotnu_completions searches in multiple directories at once
pub fn complete_item(
want_directory: bool,
span: nu_protocol::Span,
partial: &str,
cwd: &str,
cwds: &[impl AsRef<str>],
options: &CompletionOptions,
engine_state: &EngineState,
stack: &Stack,
) -> Vec<(nu_protocol::Span, String, Option<Style>)> {
let partial = surround_remove(partial);
let isdir = partial.ends_with(is_separator);
let cwd_pathbuf = Path::new(cwd).to_path_buf();
let ls_colors = (engine_state.config.use_ls_colors_completions
&& engine_state.config.use_ansi_coloring)
.then(|| {
let ls_colors_env_str = match stack.get_env_var(engine_state, "LS_COLORS") {
Some(v) => env_to_string("LS_COLORS", &v, engine_state, stack).ok(),
None => None,
};
get_ls_colors(ls_colors_env_str)
});
) -> Vec<FileSuggestion> {
let cleaned_partial = surround_remove(partial);
let isdir = cleaned_partial.ends_with(is_separator);
let expanded_partial = expand_ndots(Path::new(&cleaned_partial));
let should_collapse_dots = expanded_partial != Path::new(&cleaned_partial);
let mut partial = expanded_partial.to_string_lossy().to_string();
let mut cwd = cwd_pathbuf.clone();
#[cfg(unix)]
let path_separator = SEP;
#[cfg(windows)]
let path_separator = cleaned_partial
.chars()
.rfind(|c: &char| is_separator(*c))
.unwrap_or(SEP);
// Handle the trailing dot case
if cleaned_partial.ends_with(&format!("{path_separator}.")) {
partial.push_str(&format!("{path_separator}."));
}
let cwd_pathbufs: Vec<_> = cwds
.iter()
.map(|cwd| Path::new(cwd.as_ref()).to_path_buf())
.collect();
let ls_colors = (engine_state.config.completions.use_ls_colors
&& engine_state.config.use_ansi_coloring.get(engine_state))
.then(|| {
let ls_colors_env_str = match stack.get_env_var(engine_state, "LS_COLORS") {
Some(v) => env_to_string("LS_COLORS", v, engine_state, stack).ok(),
None => None,
};
get_ls_colors(ls_colors_env_str)
});
let mut cwds = cwd_pathbufs.clone();
let mut prefix_len = 0;
let mut original_cwd = OriginalCwd::None;
@ -140,25 +212,21 @@ pub fn complete_item(
match components.peek().cloned() {
Some(c @ Component::Prefix(..)) => {
// windows only by definition
components.next();
if let Some(Component::RootDir) = components.peek().cloned() {
components.next();
};
cwd = [c, Component::RootDir].iter().collect();
cwds = vec![[c, Component::RootDir].iter().collect()];
prefix_len = c.as_os_str().len();
original_cwd = OriginalCwd::Prefix(c.as_os_str().to_string_lossy().into_owned());
}
Some(c @ Component::RootDir) => {
components.next();
// This is kind of a hack. When joining an empty string with the rest,
// we add the slash automagically
cwd = PathBuf::from(c.as_os_str());
cwds = vec![PathBuf::from(c.as_os_str())];
prefix_len = 1;
original_cwd = OriginalCwd::Prefix(String::new());
}
Some(Component::Normal(home)) if home.to_string_lossy() == "~" => {
components.next();
cwd = home_dir().unwrap_or(cwd_pathbuf);
cwds = home_dir()
.map(|dir| vec![dir.into()])
.unwrap_or(cwd_pathbufs);
prefix_len = 1;
original_cwd = OriginalCwd::Home;
}
@ -175,21 +243,39 @@ pub fn complete_item(
complete_rec(
partial.as_slice(),
&PathBuiltFromString::default(),
&cwd,
&cwds
.into_iter()
.map(|cwd| PathBuiltFromString {
cwd,
parts: Vec::new(),
isdir: false,
})
.collect::<Vec<_>>(),
options,
want_directory,
isdir,
)
.into_iter()
.map(|p| {
let path = original_cwd.apply(p);
.map(|mut p| {
if should_collapse_dots {
p = collapse_ndots(p);
}
let path = original_cwd.apply(p, path_separator);
let style = ls_colors.as_ref().map(|lsc| {
lsc.style_for_path_with_metadata(&path, std::fs::symlink_metadata(&path).ok().as_ref())
.map(lscolors::Style::to_nu_ansi_term_style)
.unwrap_or_default()
lsc.style_for_path_with_metadata(
&path,
std::fs::symlink_metadata(expand_to_real_path(&path))
.ok()
.as_ref(),
)
.map(lscolors::Style::to_nu_ansi_term_style)
.unwrap_or_default()
});
(span, escape_path(path, want_directory), style)
FileSuggestion {
span,
path: escape_path(path, want_directory),
style,
}
})
.collect()
}
@ -197,8 +283,9 @@ pub fn complete_item(
// Fix files or folders with quotes or hashes
pub fn escape_path(path: String, dir: bool) -> String {
// make glob pattern have the highest priority.
let glob_contaminated = path.contains(['[', '*', ']', '?']);
if glob_contaminated {
if nu_glob::is_glob(path.as_str()) {
let pathbuf = nu_path::expand_tilde(path);
let path = pathbuf.to_string_lossy();
return if path.contains('\'') {
// decide to use double quote, also need to escape `"` in path
// or else users can't do anything with completed path either.
@ -211,8 +298,10 @@ pub fn escape_path(path: String, dir: bool) -> String {
let filename_contaminated = !dir && path.contains(['\'', '"', ' ', '#', '(', ')']);
let dirname_contaminated = dir && path.contains(['\'', '"', ' ', '#']);
let maybe_flag = path.starts_with('-');
let maybe_variable = path.starts_with('$');
let maybe_number = path.parse::<f64>().is_ok();
if filename_contaminated || dirname_contaminated || maybe_flag || maybe_number {
if filename_contaminated || dirname_contaminated || maybe_flag || maybe_variable || maybe_number
{
format!("`{path}`")
} else {
path
@ -251,3 +340,38 @@ pub fn adjust_if_intermediate(
readjusted,
}
}
/// Collapse multiple ".." components into n-dots.
///
/// It performs the reverse operation of `expand_ndots`, collapsing sequences of ".." into n-dots,
/// such as "..." and "....".
///
/// The resulting path will use platform-specific path separators, regardless of what path separators were used in the input.
fn collapse_ndots(path: PathBuiltFromString) -> PathBuiltFromString {
let mut result = PathBuiltFromString {
parts: Vec::with_capacity(path.parts.len()),
isdir: path.isdir,
cwd: path.cwd,
};
let mut dot_count = 0;
for part in path.parts {
if part == ".." {
dot_count += 1;
} else {
if dot_count > 0 {
result.parts.push(".".repeat(dot_count + 1));
dot_count = 0;
}
result.parts.push(part);
}
}
// Add any remaining dots
if dot_count > 0 {
result.parts.push(".".repeat(dot_count + 1));
}
result
}

276
crates/nu-cli/src/completions/completion_options.rs
View File

@ -1,17 +1,16 @@
use fuzzy_matcher::{skim::SkimMatcherV2, FuzzyMatcher};
use nu_parser::trim_quotes_str;
use nu_protocol::CompletionAlgorithm;
use std::fmt::Display;
use nu_protocol::{CompletionAlgorithm, CompletionSort};
use nu_utils::IgnoreCaseExt;
use nucleo_matcher::{
pattern::{Atom, AtomKind, CaseMatching, Normalization},
Config, Matcher, Utf32Str,
};
use std::{borrow::Cow, fmt::Display};
#[derive(Copy, Clone)]
pub enum SortBy {
LevenshteinDistance,
Ascending,
None,
}
use super::SemanticSuggestion;
/// Describes how suggestions should be matched.
#[derive(Copy, Clone, Debug)]
#[derive(Copy, Clone, Debug, PartialEq)]
pub enum MatchAlgorithm {
/// Only show suggestions which begin with the given input
///
@ -26,33 +25,170 @@ pub enum MatchAlgorithm {
Fuzzy,
}
impl MatchAlgorithm {
/// Returns whether the `needle` search text matches the given `haystack`.
pub fn matches_str(&self, haystack: &str, needle: &str) -> bool {
let haystack = trim_quotes_str(haystack);
let needle = trim_quotes_str(needle);
match *self {
MatchAlgorithm::Prefix => haystack.starts_with(needle),
pub struct NuMatcher<T> {
options: CompletionOptions,
needle: String,
state: State<T>,
}
enum State<T> {
Prefix {
/// Holds (haystack, item)
items: Vec<(String, T)>,
},
Fuzzy {
matcher: Matcher,
atom: Atom,
/// Holds (haystack, item, score)
items: Vec<(String, T, u16)>,
},
}
/// Filters and sorts suggestions
impl<T> NuMatcher<T> {
/// # Arguments
///
/// * `needle` - The text to search for
pub fn new(needle: impl AsRef<str>, options: CompletionOptions) -> NuMatcher<T> {
let needle = trim_quotes_str(needle.as_ref());
match options.match_algorithm {
MatchAlgorithm::Prefix => {
let lowercase_needle = if options.case_sensitive {
needle.to_owned()
} else {
needle.to_folded_case()
};
NuMatcher {
options,
needle: lowercase_needle,
state: State::Prefix { items: Vec::new() },
}
}
MatchAlgorithm::Fuzzy => {
let matcher = SkimMatcherV2::default();
matcher.fuzzy_match(haystack, needle).is_some()
let atom = Atom::new(
needle,
if options.case_sensitive {
CaseMatching::Respect
} else {
CaseMatching::Ignore
},
Normalization::Smart,
AtomKind::Fuzzy,
false,
);
NuMatcher {
options,
needle: needle.to_owned(),
state: State::Fuzzy {
matcher: Matcher::new(Config::DEFAULT),
atom,
items: Vec::new(),
},
}
}
}
}
/// Returns whether the `needle` search text matches the given `haystack`.
pub fn matches_u8(&self, haystack: &[u8], needle: &[u8]) -> bool {
match *self {
MatchAlgorithm::Prefix => haystack.starts_with(needle),
MatchAlgorithm::Fuzzy => {
let haystack_str = String::from_utf8_lossy(haystack);
let needle_str = String::from_utf8_lossy(needle);
let matcher = SkimMatcherV2::default();
matcher.fuzzy_match(&haystack_str, &needle_str).is_some()
/// Returns whether or not the haystack matches the needle. If it does, `item` is added
/// to the list of matches (if given).
///
/// Helper to avoid code duplication between [NuMatcher::add] and [NuMatcher::matches].
fn matches_aux(&mut self, haystack: &str, item: Option<T>) -> bool {
let haystack = trim_quotes_str(haystack);
match &mut self.state {
State::Prefix { items } => {
let haystack_folded = if self.options.case_sensitive {
Cow::Borrowed(haystack)
} else {
Cow::Owned(haystack.to_folded_case())
};
let matches = if self.options.positional {
haystack_folded.starts_with(self.needle.as_str())
} else {
haystack_folded.contains(self.needle.as_str())
};
if matches {
if let Some(item) = item {
items.push((haystack.to_string(), item));
}
}
matches
}
State::Fuzzy {
matcher,
atom,
items,
} => {
let mut haystack_buf = Vec::new();
let haystack_utf32 = Utf32Str::new(trim_quotes_str(haystack), &mut haystack_buf);
let mut indices = Vec::new();
let Some(score) = atom.indices(haystack_utf32, matcher, &mut indices) else {
return false;
};
if let Some(item) = item {
items.push((haystack.to_string(), item, score));
}
true
}
}
}
/// Add the given item if the given haystack matches the needle.
///
/// Returns whether the item was added.
pub fn add(&mut self, haystack: impl AsRef<str>, item: T) -> bool {
self.matches_aux(haystack.as_ref(), Some(item))
}
/// Returns whether the haystack matches the needle.
pub fn matches(&mut self, haystack: &str) -> bool {
self.matches_aux(haystack, None)
}
/// Get all the items that matched (sorted)
pub fn results(self) -> Vec<T> {
match self.state {
State::Prefix { mut items, .. } => {
items.sort_by(|(haystack1, _), (haystack2, _)| {
let cmp_sensitive = haystack1.cmp(haystack2);
if self.options.case_sensitive {
cmp_sensitive
} else {
haystack1
.to_folded_case()
.cmp(&haystack2.to_folded_case())
.then(cmp_sensitive)
}
});
items.into_iter().map(|(_, item)| item).collect::<Vec<_>>()
}
State::Fuzzy { mut items, .. } => {
match self.options.sort {
CompletionSort::Alphabetical => {
items.sort_by(|(haystack1, _, _), (haystack2, _, _)| {
haystack1.cmp(haystack2)
});
}
CompletionSort::Smart => {
items.sort_by(|(haystack1, _, score1), (haystack2, _, score2)| {
score2.cmp(score1).then(haystack1.cmp(haystack2))
});
}
}
items
.into_iter()
.map(|(_, item, _)| item)
.collect::<Vec<_>>()
}
}
}
}
impl NuMatcher<SemanticSuggestion> {
pub fn add_semantic_suggestion(&mut self, sugg: SemanticSuggestion) -> bool {
let value = sugg.suggestion.value.to_string();
self.add(value, sugg)
}
}
impl From<CompletionAlgorithm> for MatchAlgorithm {
@ -96,6 +232,7 @@ pub struct CompletionOptions {
pub case_sensitive: bool,
pub positional: bool,
pub match_algorithm: MatchAlgorithm,
pub sort: CompletionSort,
}
impl Default for CompletionOptions {
@ -104,41 +241,74 @@ impl Default for CompletionOptions {
case_sensitive: true,
positional: true,
match_algorithm: MatchAlgorithm::Prefix,
sort: Default::default(),
}
}
}
#[cfg(test)]
mod test {
use super::MatchAlgorithm;
use rstest::rstest;
#[test]
fn match_algorithm_prefix() {
let algorithm = MatchAlgorithm::Prefix;
use super::{CompletionOptions, MatchAlgorithm, NuMatcher};
assert!(algorithm.matches_str("example text", ""));
assert!(algorithm.matches_str("example text", "examp"));
assert!(!algorithm.matches_str("example text", "text"));
assert!(algorithm.matches_u8(&[1, 2, 3], &[]));
assert!(algorithm.matches_u8(&[1, 2, 3], &[1, 2]));
assert!(!algorithm.matches_u8(&[1, 2, 3], &[2, 3]));
#[rstest]
#[case(MatchAlgorithm::Prefix, "example text", "", true)]
#[case(MatchAlgorithm::Prefix, "example text", "examp", true)]
#[case(MatchAlgorithm::Prefix, "example text", "text", false)]
#[case(MatchAlgorithm::Fuzzy, "example text", "", true)]
#[case(MatchAlgorithm::Fuzzy, "example text", "examp", true)]
#[case(MatchAlgorithm::Fuzzy, "example text", "ext", true)]
#[case(MatchAlgorithm::Fuzzy, "example text", "mplxt", true)]
#[case(MatchAlgorithm::Fuzzy, "example text", "mpp", false)]
fn match_algorithm_simple(
#[case] match_algorithm: MatchAlgorithm,
#[case] haystack: &str,
#[case] needle: &str,
#[case] should_match: bool,
) {
let options = CompletionOptions {
match_algorithm,
..Default::default()
};
let mut matcher = NuMatcher::new(needle, options);
matcher.add(haystack, haystack);
if should_match {
assert_eq!(vec![haystack], matcher.results());
} else {
assert_ne!(vec![haystack], matcher.results());
}
}
#[test]
fn match_algorithm_fuzzy() {
let algorithm = MatchAlgorithm::Fuzzy;
fn match_algorithm_fuzzy_sort_score() {
let options = CompletionOptions {
match_algorithm: MatchAlgorithm::Fuzzy,
..Default::default()
};
let mut matcher = NuMatcher::new("fob", options);
for item in ["foo/bar", "fob", "foo bar"] {
matcher.add(item, item);
}
// Sort by score, then in alphabetical order
assert_eq!(vec!["fob", "foo bar", "foo/bar"], matcher.results());
}
assert!(algorithm.matches_str("example text", ""));
assert!(algorithm.matches_str("example text", "examp"));
assert!(algorithm.matches_str("example text", "ext"));
assert!(algorithm.matches_str("example text", "mplxt"));
assert!(!algorithm.matches_str("example text", "mpp"));
assert!(algorithm.matches_u8(&[1, 2, 3], &[]));
assert!(algorithm.matches_u8(&[1, 2, 3], &[1, 2]));
assert!(algorithm.matches_u8(&[1, 2, 3], &[2, 3]));
assert!(algorithm.matches_u8(&[1, 2, 3], &[1, 3]));
assert!(!algorithm.matches_u8(&[1, 2, 3], &[2, 2]));
#[test]
fn match_algorithm_fuzzy_sort_strip() {
let options = CompletionOptions {
match_algorithm: MatchAlgorithm::Fuzzy,
..Default::default()
};
let mut matcher = NuMatcher::new("'love spaces' ", options);
for item in [
"'i love spaces'",
"'i love spaces' so much",
"'lovespaces' ",
] {
matcher.add(item, item);
}
// Make sure the spaces are respected
assert_eq!(vec!["'i love spaces' so much"], matcher.results());
}
}

183
crates/nu-cli/src/completions/custom_completions.rs
View File

@ -1,45 +1,45 @@
use crate::completions::{
completer::map_value_completions, Completer, CompletionOptions, MatchAlgorithm,
SemanticSuggestion, SortBy,
completer::map_value_completions, Completer, CompletionOptions, SemanticSuggestion,
};
use nu_engine::eval_call;
use nu_protocol::{
ast::{Argument, Call, Expr, Expression},
debugger::WithoutDebug,
engine::{Stack, StateWorkingSet},
PipelineData, Span, Type, Value,
DeclId, PipelineData, Span, Type, Value,
};
use nu_utils::IgnoreCaseExt;
use std::collections::HashMap;
pub struct CustomCompletion {
use super::completion_options::NuMatcher;
pub struct CustomCompletion<T: Completer> {
stack: Stack,
decl_id: usize,
decl_id: DeclId,
line: String,
sort_by: SortBy,
fallback: T,
}
impl CustomCompletion {
pub fn new(stack: Stack, decl_id: usize, line: String) -> Self {
impl<T: Completer> CustomCompletion<T> {
pub fn new(stack: Stack, decl_id: DeclId, line: String, fallback: T) -> Self {
Self {
stack,
decl_id,
line,
sort_by: SortBy::None,
fallback,
}
}
}
impl Completer for CustomCompletion {
impl<T: Completer> Completer for CustomCompletion<T> {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
_stack: &Stack,
prefix: Vec<u8>,
stack: &Stack,
prefix: &[u8],
span: Span,
offset: usize,
pos: usize,
completion_options: &CompletionOptions,
orig_options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
// Line position
let line_pos = pos - offset;
@ -52,30 +52,28 @@ impl Completer for CustomCompletion {
decl_id: self.decl_id,
head: span,
arguments: vec![
Argument::Positional(Expression {
span: Span::unknown(),
ty: Type::String,
expr: Expr::String(self.line.clone()),
custom_completion: None,
}),
Argument::Positional(Expression {
span: Span::unknown(),
ty: Type::Int,
expr: Expr::Int(line_pos as i64),
custom_completion: None,
}),
Argument::Positional(Expression::new_unknown(
Expr::String(self.line.clone()),
Span::unknown(),
Type::String,
)),
Argument::Positional(Expression::new_unknown(
Expr::Int(line_pos as i64),
Span::unknown(),
Type::Int,
)),
],
parser_info: HashMap::new(),
},
PipelineData::empty(),
);
let mut custom_completion_options = None;
let mut completion_options = orig_options.clone();
let mut should_sort = true;
// Parse result
let suggestions = result
.and_then(|data| data.into_value(span))
.map(|value| match &value {
let suggestions = match result.and_then(|data| data.into_value(span)) {
Ok(value) => match &value {
Value::Record { val, .. } => {
let completions = val
.get("completions")
@ -88,81 +86,70 @@ impl Completer for CustomCompletion {
let options = val.get("options");
if let Some(Value::Record { val: options, .. }) = &options {
let should_sort = options
.get("sort")
.and_then(|val| val.as_bool().ok())
.unwrap_or(false);
if should_sort {
self.sort_by = SortBy::Ascending;
if let Some(sort) = options.get("sort").and_then(|val| val.as_bool().ok()) {
should_sort = sort;
}
custom_completion_options = Some(CompletionOptions {
case_sensitive: options
.get("case_sensitive")
.and_then(|val| val.as_bool().ok())
.unwrap_or(true),
positional: options
.get("positional")
.and_then(|val| val.as_bool().ok())
.unwrap_or(true),
match_algorithm: match options.get("completion_algorithm") {
Some(option) => option
.coerce_string()
.ok()
.and_then(|option| option.try_into().ok())
.unwrap_or(MatchAlgorithm::Prefix),
None => completion_options.match_algorithm,
},
});
if let Some(case_sensitive) = options
.get("case_sensitive")
.and_then(|val| val.as_bool().ok())
{
completion_options.case_sensitive = case_sensitive;
}
if let Some(positional) =
options.get("positional").and_then(|val| val.as_bool().ok())
{
completion_options.positional = positional;
}
if let Some(algorithm) = options
.get("completion_algorithm")
.and_then(|option| option.coerce_string().ok())
.and_then(|option| option.try_into().ok())
{
completion_options.match_algorithm = algorithm;
}
}
completions
}
Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset),
_ => vec![],
})
.unwrap_or_default();
if let Some(custom_completion_options) = custom_completion_options {
filter(&prefix, suggestions, &custom_completion_options)
} else {
filter(&prefix, suggestions, completion_options)
}
}
fn get_sort_by(&self) -> SortBy {
self.sort_by
}
}
fn filter(
prefix: &[u8],
items: Vec<SemanticSuggestion>,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
items
.into_iter()
.filter(|it| match options.match_algorithm {
MatchAlgorithm::Prefix => match (options.case_sensitive, options.positional) {
(true, true) => it.suggestion.value.as_bytes().starts_with(prefix),
(true, false) => it
.suggestion
.value
.contains(std::str::from_utf8(prefix).unwrap_or("")),
(false, positional) => {
let value = it.suggestion.value.to_folded_case();
let prefix = std::str::from_utf8(prefix).unwrap_or("").to_folded_case();
if positional {
value.starts_with(&prefix)
} else {
value.contains(&prefix)
}
Value::Nothing { .. } => {
return self.fallback.fetch(
working_set,
stack,
prefix,
span,
offset,
pos,
orig_options,
);
}
_ => {
log::error!(
"Custom completer returned invalid value of type {}",
value.get_type().to_string()
);
return vec![];
}
},
MatchAlgorithm::Fuzzy => options
.match_algorithm
.matches_u8(it.suggestion.value.as_bytes(), prefix),
})
.collect()
Err(e) => {
log::error!("Error getting custom completions: {e}");
return vec![];
}
};
let mut matcher = NuMatcher::new(String::from_utf8_lossy(prefix), completion_options);
if should_sort {
for sugg in suggestions {
matcher.add_semantic_suggestion(sugg);
}
matcher.results()
} else {
suggestions
.into_iter()
.filter(|sugg| matcher.matches(&sugg.suggestion.value))
.collect()
}
}
}

63
crates/nu-cli/src/completions/directory_completions.rs
View File

@ -1,16 +1,15 @@
use crate::completions::{
completion_common::{adjust_if_intermediate, complete_item, AdjustView},
Completer, CompletionOptions, SortBy,
Completer, CompletionOptions,
};
use nu_ansi_term::Style;
use nu_protocol::{
engine::{EngineState, Stack, StateWorkingSet},
levenshtein_distance, Span,
Span,
};
use reedline::Suggestion;
use std::path::{Path, MAIN_SEPARATOR as SEP};
use std::path::Path;
use super::SemanticSuggestion;
use super::{completion_common::FileSuggestion, SemanticSuggestion};
#[derive(Clone, Default)]
pub struct DirectoryCompletion {}
@ -26,17 +25,17 @@ impl Completer for DirectoryCompletion {
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
prefix: Vec<u8>,
prefix: &[u8],
span: Span,
offset: usize,
_pos: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
let AdjustView { prefix, span, .. } = adjust_if_intermediate(&prefix, working_set, span);
let AdjustView { prefix, span, .. } = adjust_if_intermediate(prefix, working_set, span);
// Filter only the folders
#[allow(deprecated)]
let output: Vec<_> = directory_completion(
let items: Vec<_> = directory_completion(
span,
&prefix,
&working_set.permanent_state.current_work_dir(),
@ -47,56 +46,24 @@ impl Completer for DirectoryCompletion {
.into_iter()
.map(move |x| SemanticSuggestion {
suggestion: Suggestion {
value: x.1,
description: None,
style: x.2,
extra: None,
value: x.path,
style: x.style,
span: reedline::Span {
start: x.0.start - offset,
end: x.0.end - offset,
start: x.span.start - offset,
end: x.span.end - offset,
},
append_whitespace: false,
..Suggestion::default()
},
// TODO????
kind: None,
})
.collect();
output
}
// Sort results prioritizing the non hidden folders
fn sort(&self, items: Vec<SemanticSuggestion>, prefix: Vec<u8>) -> Vec<SemanticSuggestion> {
let prefix_str = String::from_utf8_lossy(&prefix).to_string();
// Sort items
let mut sorted_items = items;
match self.get_sort_by() {
SortBy::Ascending => {
sorted_items.sort_by(|a, b| {
// Ignore trailing slashes in folder names when sorting
a.suggestion
.value
.trim_end_matches(SEP)
.cmp(b.suggestion.value.trim_end_matches(SEP))
});
}
SortBy::LevenshteinDistance => {
sorted_items.sort_by(|a, b| {
let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value);
let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value);
a_distance.cmp(&b_distance)
});
}
_ => (),
}
// Separate the results between hidden and non hidden
let mut hidden: Vec<SemanticSuggestion> = vec![];
let mut non_hidden: Vec<SemanticSuggestion> = vec![];
for item in sorted_items.into_iter() {
for item in items.into_iter() {
let item_path = Path::new(&item.suggestion.value);
if let Some(value) = item_path.file_name() {
@ -124,6 +91,6 @@ pub fn directory_completion(
options: &CompletionOptions,
engine_state: &EngineState,
stack: &Stack,
) -> Vec<(nu_protocol::Span, String, Option<Style>)> {
complete_item(true, span, partial, cwd, options, engine_state, stack)
) -> Vec<FileSuggestion> {
complete_item(true, span, partial, &[cwd], options, engine_state, stack)
}

224
crates/nu-cli/src/completions/dotnu_completions.rs
View File

@ -1,12 +1,16 @@
use crate::completions::{file_path_completion, Completer, CompletionOptions, SortBy};
use crate::completions::{file_path_completion, Completer, CompletionOptions};
use nu_path::expand_tilde;
use nu_protocol::{
engine::{Stack, StateWorkingSet},
Span,
};
use reedline::Suggestion;
use std::path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR};
use std::{
collections::HashSet,
path::{is_separator, PathBuf, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR},
};
use super::SemanticSuggestion;
use super::{SemanticSuggestion, SuggestionKind};
#[derive(Clone, Default)]
pub struct DotNuCompletion {}
@ -22,119 +26,143 @@ impl Completer for DotNuCompletion {
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
prefix: Vec<u8>,
prefix: &[u8],
span: Span,
offset: usize,
_pos: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
let prefix_str = String::from_utf8_lossy(&prefix).replace('`', "");
let mut search_dirs: Vec<String> = vec![];
let prefix_str = String::from_utf8_lossy(prefix);
let start_with_backquote = prefix_str.starts_with('`');
let end_with_backquote = prefix_str.ends_with('`');
let prefix_str = prefix_str.replace('`', "");
// e.g. `./`, `..\`, `/`
let not_lib_dirs = prefix_str
.chars()
.find(|c| *c != '.')
.is_some_and(is_separator);
let mut search_dirs: Vec<PathBuf> = vec![];
// If prefix_str is only a word we want to search in the current dir
let (base, partial) = prefix_str
.rsplit_once(is_separator)
.unwrap_or((".", &prefix_str));
let (base, partial) = if let Some((parent, remain)) = prefix_str.rsplit_once(is_separator) {
// If prefix_str is only a word we want to search in the current dir.
// "/xx" should be split to "/" and "xx".
if parent.is_empty() {
(MAIN_SEPARATOR_STR, remain)
} else {
(parent, remain)
}
} else {
(".", prefix_str.as_str())
};
let base_dir = base.replace(is_separator, MAIN_SEPARATOR_STR);
let mut partial = partial.to_string();
// On windows, this standardizes paths to use \
let mut is_current_folder = false;
// Fetch the lib dirs
let lib_dirs: Vec<String> = if let Some(lib_dirs) = working_set.get_env_var("NU_LIB_DIRS") {
lib_dirs
.as_list()
.into_iter()
.flat_map(|it| {
it.iter().map(|x| {
x.to_path()
.expect("internal error: failed to convert lib path")
})
})
.map(|it| {
it.into_os_string()
.into_string()
.expect("internal error: failed to convert OS path")
})
.collect()
} else {
vec![]
};
// NOTE: 2 ways to setup `NU_LIB_DIRS`
// 1. `const NU_LIB_DIRS = [paths]`, equal to `nu -I paths`
// 2. `$env.NU_LIB_DIRS = [paths]`
let const_lib_dirs = working_set
.find_variable(b"$NU_LIB_DIRS")
.and_then(|vid| working_set.get_variable(vid).const_val.as_ref());
let env_lib_dirs = working_set.get_env_var("NU_LIB_DIRS");
let lib_dirs: HashSet<PathBuf> = [const_lib_dirs, env_lib_dirs]
.into_iter()
.flatten()
.flat_map(|lib_dirs| {
lib_dirs
.as_list()
.into_iter()
.flat_map(|it| it.iter().filter_map(|x| x.to_path().ok()))
.map(expand_tilde)
})
.collect();
// Check if the base_dir is a folder
// rsplit_once removes the separator
let cwd = working_set.permanent_state.cwd(None);
if base_dir != "." {
// Add the base dir into the directories to be searched
search_dirs.push(base_dir.clone());
// Reset the partial adding the basic dir back
// in order to make the span replace work properly
let mut base_dir_partial = base_dir;
base_dir_partial.push_str(&partial);
partial = base_dir_partial;
let expanded_base_dir = expand_tilde(&base_dir);
let is_base_dir_relative = expanded_base_dir.is_relative();
// Search in base_dir as well as lib_dirs.
// After expanded, base_dir can be a relative path or absolute path.
// If relative, we join "current working dir" with it to get subdirectory and add to search_dirs.
// If absolute, we add it to search_dirs.
if let Ok(mut cwd) = cwd {
if is_base_dir_relative {
cwd.push(&base_dir);
search_dirs.push(cwd.into_std_path_buf());
} else {
search_dirs.push(expanded_base_dir);
}
}
if !not_lib_dirs {
search_dirs.extend(lib_dirs.into_iter().map(|mut dir| {
dir.push(&base_dir);
dir
}));
}
} else {
// Fetch the current folder
#[allow(deprecated)]
let current_folder = working_set.permanent_state.current_work_dir();
is_current_folder = true;
// Add the current folder and the lib dirs into the
// directories to be searched
search_dirs.push(current_folder);
search_dirs.extend(lib_dirs);
if let Ok(cwd) = cwd {
search_dirs.push(cwd.into_std_path_buf());
}
if !not_lib_dirs {
search_dirs.extend(lib_dirs);
}
}
// Fetch the files filtering the ones that ends with .nu
// and transform them into suggestions
let output: Vec<SemanticSuggestion> = search_dirs
let completions = file_path_completion(
span,
partial,
&search_dirs
.iter()
.filter_map(|d| d.to_str())
.collect::<Vec<_>>(),
options,
working_set.permanent_state,
stack,
);
completions
.into_iter()
.flat_map(|search_dir| {
let completions = file_path_completion(
span,
&partial,
&search_dir,
options,
working_set.permanent_state,
stack,
);
completions
.into_iter()
.filter(move |it| {
// Different base dir, so we list the .nu files or folders
if !is_current_folder {
it.1.ends_with(".nu") || it.1.ends_with(SEP)
} else {
// Lib dirs, so we filter only the .nu files or directory modules
if it.1.ends_with(SEP) {
Path::new(&search_dir).join(&it.1).join("mod.nu").exists()
} else {
it.1.ends_with(".nu")
}
}
})
.map(move |x| SemanticSuggestion {
suggestion: Suggestion {
value: x.1,
description: None,
style: x.2,
extra: None,
span: reedline::Span {
start: x.0.start - offset,
end: x.0.end - offset,
},
append_whitespace: true,
},
// TODO????
kind: None,
})
// Different base dir, so we list the .nu files or folders
.filter(|it| {
// for paths with spaces in them
let path = it.path.trim_end_matches('`');
path.ends_with(".nu") || path.ends_with(SEP)
})
.collect();
output
}
fn get_sort_by(&self) -> SortBy {
SortBy::LevenshteinDistance
.map(|x| {
let append_whitespace =
x.path.ends_with(".nu") && (!start_with_backquote || end_with_backquote);
// Re-calculate the span to replace
let mut span_offset = 0;
let mut value = x.path.to_string();
// Complete only the last path component
if base_dir == MAIN_SEPARATOR_STR {
span_offset = base_dir.len()
} else if base_dir != "." {
span_offset = base_dir.len() + 1
}
// Retain only one '`'
if start_with_backquote {
value = value.trim_start_matches('`').to_string();
span_offset += 1;
}
// Add the backquote back
if end_with_backquote && !value.ends_with('`') {
value.push('`');
}
let end = x.span.end - offset;
let start = std::cmp::min(end, x.span.start - offset + span_offset);
SemanticSuggestion {
suggestion: Suggestion {
value,
style: x.style,
span: reedline::Span { start, end },
append_whitespace,
..Suggestion::default()
},
kind: Some(SuggestionKind::Module),
}
})
.collect::<Vec<_>>()
}
}

79
crates/nu-cli/src/completions/file_completions.rs
View File

@ -1,17 +1,15 @@
use crate::completions::{
completion_common::{adjust_if_intermediate, complete_item, AdjustView},
Completer, CompletionOptions, SortBy,
Completer, CompletionOptions,
};
use nu_ansi_term::Style;
use nu_protocol::{
engine::{EngineState, Stack, StateWorkingSet},
levenshtein_distance, Span,
Span,
};
use nu_utils::IgnoreCaseExt;
use reedline::Suggestion;
use std::path::{Path, MAIN_SEPARATOR as SEP};
use std::path::Path;
use super::SemanticSuggestion;
use super::{completion_common::FileSuggestion, SemanticSuggestion};
#[derive(Clone, Default)]
pub struct FileCompletion {}
@ -27,7 +25,7 @@ impl Completer for FileCompletion {
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
prefix: Vec<u8>,
prefix: &[u8],
span: Span,
offset: usize,
_pos: usize,
@ -37,14 +35,14 @@ impl Completer for FileCompletion {
prefix,
span,
readjusted,
} = adjust_if_intermediate(&prefix, working_set, span);
} = adjust_if_intermediate(prefix, working_set, span);
#[allow(deprecated)]
let output: Vec<_> = complete_item(
let items: Vec<_> = complete_item(
readjusted,
span,
&prefix,
&working_set.permanent_state.current_work_dir(),
&[&working_set.permanent_state.current_work_dir()],
options,
working_set.permanent_state,
stack,
@ -52,56 +50,26 @@ impl Completer for FileCompletion {
.into_iter()
.map(move |x| SemanticSuggestion {
suggestion: Suggestion {
value: x.1,
description: None,
style: x.2,
extra: None,
value: x.path,
style: x.style,
span: reedline::Span {
start: x.0.start - offset,
end: x.0.end - offset,
start: x.span.start - offset,
end: x.span.end - offset,
},
append_whitespace: false,
..Suggestion::default()
},
// TODO????
kind: None,
})
.collect();
output
}
// Sort results prioritizing the non hidden folders
fn sort(&self, items: Vec<SemanticSuggestion>, prefix: Vec<u8>) -> Vec<SemanticSuggestion> {
let prefix_str = String::from_utf8_lossy(&prefix).to_string();
// Sort items
let mut sorted_items = items;
match self.get_sort_by() {
SortBy::Ascending => {
sorted_items.sort_by(|a, b| {
// Ignore trailing slashes in folder names when sorting
a.suggestion
.value
.trim_end_matches(SEP)
.cmp(b.suggestion.value.trim_end_matches(SEP))
});
}
SortBy::LevenshteinDistance => {
sorted_items.sort_by(|a, b| {
let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value);
let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value);
a_distance.cmp(&b_distance)
});
}
_ => (),
}
// Sort results prioritizing the non hidden folders
// Separate the results between hidden and non hidden
let mut hidden: Vec<SemanticSuggestion> = vec![];
let mut non_hidden: Vec<SemanticSuggestion> = vec![];
for item in sorted_items.into_iter() {
for item in items.into_iter() {
let item_path = Path::new(&item.suggestion.value);
if let Some(value) = item_path.file_name() {
@ -125,21 +93,10 @@ impl Completer for FileCompletion {
pub fn file_path_completion(
span: nu_protocol::Span,
partial: &str,
cwd: &str,
cwds: &[impl AsRef<str>],
options: &CompletionOptions,
engine_state: &EngineState,
stack: &Stack,
) -> Vec<(nu_protocol::Span, String, Option<Style>)> {
complete_item(false, span, partial, cwd, options, engine_state, stack)
}
pub fn matches(partial: &str, from: &str, options: &CompletionOptions) -> bool {
// Check for case sensitive
if !options.case_sensitive {
return options
.match_algorithm
.matches_str(&from.to_folded_case(), &partial.to_folded_case());
}
options.match_algorithm.matches_str(from, partial)
) -> Vec<FileSuggestion> {
complete_item(false, span, partial, cwds, options, engine_state, stack)
}

66
crates/nu-cli/src/completions/flag_completions.rs
View File

@ -1,4 +1,4 @@
use crate::completions::{Completer, CompletionOptions};
use crate::completions::{completion_options::NuMatcher, Completer, CompletionOptions};
use nu_protocol::{
ast::{Expr, Expression},
engine::{Stack, StateWorkingSet},
@ -24,7 +24,7 @@ impl Completer for FlagCompletion {
&mut self,
working_set: &StateWorkingSet,
_stack: &Stack,
prefix: Vec<u8>,
prefix: &[u8],
span: Span,
offset: usize,
_pos: usize,
@ -35,7 +35,7 @@ impl Completer for FlagCompletion {
let decl = working_set.get_decl(call.decl_id);
let sig = decl.signature();
let mut output = vec![];
let mut matcher = NuMatcher::new(String::from_utf8_lossy(prefix), options.clone());
for named in &sig.named {
let flag_desc = &named.desc;
@ -44,23 +44,20 @@ impl Completer for FlagCompletion {
short.encode_utf8(&mut named);
named.insert(0, b'-');
if options.match_algorithm.matches_u8(&named, &prefix) {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(&named).to_string(),
description: Some(flag_desc.to_string()),
style: None,
extra: None,
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
append_whitespace: true,
matcher.add_semantic_suggestion(SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(&named).to_string(),
description: Some(flag_desc.to_string()),
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
// TODO????
kind: None,
});
}
append_whitespace: true,
..Suggestion::default()
},
// TODO????
kind: None,
});
}
if named.long.is_empty() {
@ -71,26 +68,23 @@ impl Completer for FlagCompletion {
named.insert(0, b'-');
named.insert(0, b'-');
if options.match_algorithm.matches_u8(&named, &prefix) {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(&named).to_string(),
description: Some(flag_desc.to_string()),
style: None,
extra: None,
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
append_whitespace: true,
matcher.add_semantic_suggestion(SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(&named).to_string(),
description: Some(flag_desc.to_string()),
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
// TODO????
kind: None,
});
}
append_whitespace: true,
..Suggestion::default()
},
// TODO????
kind: None,
});
}
return output;
return matcher.results();
}
vec![]

10
crates/nu-cli/src/completions/mod.rs
View File

@ -1,4 +1,6 @@
mod attribute_completions;
mod base;
mod cell_path_completions;
mod command_completions;
mod completer;
mod completion_common;
@ -8,15 +10,19 @@ mod directory_completions;
mod dotnu_completions;
mod file_completions;
mod flag_completions;
mod operator_completions;
mod variable_completions;
pub use attribute_completions::{AttributableCompletion, AttributeCompletion};
pub use base::{Completer, SemanticSuggestion, SuggestionKind};
pub use cell_path_completions::CellPathCompletion;
pub use command_completions::CommandCompletion;
pub use completer::NuCompleter;
pub use completion_options::{CompletionOptions, MatchAlgorithm, SortBy};
pub use completion_options::{CompletionOptions, MatchAlgorithm};
pub use custom_completions::CustomCompletion;
pub use directory_completions::DirectoryCompletion;
pub use dotnu_completions::DotNuCompletion;
pub use file_completions::{file_path_completion, matches, FileCompletion};
pub use file_completions::{file_path_completion, FileCompletion};
pub use flag_completions::FlagCompletion;
pub use operator_completions::OperatorCompletion;
pub use variable_completions::VariableCompletion;

170
crates/nu-cli/src/completions/operator_completions.rs Normal file
View File

@ -0,0 +1,170 @@
use crate::completions::{
completion_options::NuMatcher, Completer, CompletionOptions, SemanticSuggestion, SuggestionKind,
};
use nu_protocol::{
ast::{Expr, Expression},
engine::{Stack, StateWorkingSet},
Span, Type,
};
use reedline::Suggestion;
#[derive(Clone)]
pub struct OperatorCompletion {
previous_expr: Expression,
}
impl OperatorCompletion {
pub fn new(previous_expr: Expression) -> Self {
OperatorCompletion { previous_expr }
}
}
impl Completer for OperatorCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
_stack: &Stack,
_prefix: &[u8],
span: Span,
offset: usize,
_pos: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
//Check if int, float, or string
let partial = std::str::from_utf8(working_set.get_span_contents(span)).unwrap_or("");
let op = match &self.previous_expr.expr {
Expr::BinaryOp(x, _, _) => &x.expr,
_ => {
return vec![];
}
};
let possible_operations = match op {
Expr::Int(_) => vec![
("+", "Add (Plus)"),
("-", "Subtract (Minus)"),
("*", "Multiply"),
("/", "Divide"),
("==", "Equal to"),
("!=", "Not equal to"),
("//", "Floor division"),
("<", "Less than"),
(">", "Greater than"),
("<=", "Less than or equal to"),
(">=", "Greater than or equal to"),
("mod", "Floor division remainder (Modulo)"),
("**", "Power of"),
("bit-or", "Bitwise OR"),
("bit-xor", "Bitwise exclusive OR"),
("bit-and", "Bitwise AND"),
("bit-shl", "Bitwise shift left"),
("bit-shr", "Bitwise shift right"),
("in", "Is a member of (doesn't use regex)"),
("not-in", "Is not a member of (doesn't use regex)"),
],
Expr::String(_) => vec![
("=~", "Contains regex match"),
("like", "Contains regex match"),
("!~", "Does not contain regex match"),
("not-like", "Does not contain regex match"),
(
"++",
"Concatenates two lists, two strings, or two binary values",
),
("in", "Is a member of (doesn't use regex)"),
("not-in", "Is not a member of (doesn't use regex)"),
("starts-with", "Starts with"),
("ends-with", "Ends with"),
],
Expr::Float(_) => vec![
("+", "Add (Plus)"),
("-", "Subtract (Minus)"),
("*", "Multiply"),
("/", "Divide"),
("==", "Equal to"),
("!=", "Not equal to"),
("//", "Floor division"),
("<", "Less than"),
(">", "Greater than"),
("<=", "Less than or equal to"),
(">=", "Greater than or equal to"),
("mod", "Floor division remainder (Modulo)"),
("**", "Power of"),
("in", "Is a member of (doesn't use regex)"),
("not-in", "Is not a member of (doesn't use regex)"),
],
Expr::Bool(_) => vec![
(
"and",
"Both values are true (short-circuits when first value is false)",
),
(
"or",
"Either value is true (short-circuits when first value is true)",
),
("xor", "One value is true and the other is false"),
("not", "Negates a value or expression"),
("in", "Is a member of (doesn't use regex)"),
("not-in", "Is not a member of (doesn't use regex)"),
],
Expr::FullCellPath(path) => match path.head.expr {
Expr::List(_) => vec![
(
"++",
"Concatenates two lists, two strings, or two binary values",
),
("has", "Contains a value of (doesn't use regex)"),
("not-has", "Does not contain a value of (doesn't use regex)"),
],
Expr::Var(id) => get_variable_completions(id, working_set),
_ => vec![],
},
_ => vec![],
};
let mut matcher = NuMatcher::new(partial, options.clone());
for (symbol, desc) in possible_operations.into_iter() {
matcher.add_semantic_suggestion(SemanticSuggestion {
suggestion: Suggestion {
value: symbol.to_string(),
description: Some(desc.to_string()),
span: reedline::Span::new(span.start - offset, span.end - offset),
append_whitespace: true,
..Suggestion::default()
},
kind: Some(SuggestionKind::Command(
nu_protocol::engine::CommandType::Builtin,
)),
});
}
matcher.results()
}
}
pub fn get_variable_completions<'a>(
id: nu_protocol::Id<nu_protocol::marker::Var>,
working_set: &StateWorkingSet,
) -> Vec<(&'a str, &'a str)> {
let var = working_set.get_variable(id);
if !var.mutable {
return vec![];
}
match var.ty {
Type::List(_) | Type::String | Type::Binary => vec![
(
"++=",
"Concatenates two lists, two strings, or two binary values",
),
("=", "Assigns a value to a variable."),
],
Type::Int | Type::Float => vec![
("=", "Assigns a value to a variable."),
("+=", "Adds a value to a variable."),
("-=", "Subtracts a value from a variable."),
("*=", "Multiplies a variable by a value"),
("/=", "Divides a variable by a value."),
],
_ => vec![],
}
}

317
crates/nu-cli/src/completions/variable_completions.rs
View File

@ -1,201 +1,70 @@
use crate::completions::{
Completer, CompletionOptions, MatchAlgorithm, SemanticSuggestion, SuggestionKind,
};
use nu_engine::{column::get_columns, eval_variable};
use crate::completions::{Completer, CompletionOptions, SemanticSuggestion, SuggestionKind};
use nu_protocol::{
engine::{Stack, StateWorkingSet},
Span, Value,
Span, VarId,
};
use reedline::Suggestion;
use std::str;
#[derive(Clone)]
pub struct VariableCompletion {
var_context: (Vec<u8>, Vec<Vec<u8>>), // tuple with $var and the sublevels (.b.c.d)
}
use super::completion_options::NuMatcher;
impl VariableCompletion {
pub fn new(var_context: (Vec<u8>, Vec<Vec<u8>>)) -> Self {
Self { var_context }
}
}
pub struct VariableCompletion {}
impl Completer for VariableCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
prefix: Vec<u8>,
_stack: &Stack,
prefix: &[u8],
span: Span,
offset: usize,
_pos: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
let mut output = vec![];
let builtins = ["$nu", "$in", "$env"];
let var_str = std::str::from_utf8(&self.var_context.0).unwrap_or("");
let var_id = working_set.find_variable(&self.var_context.0);
let prefix_str = String::from_utf8_lossy(prefix);
let mut matcher = NuMatcher::new(prefix_str, options.clone());
let current_span = reedline::Span {
start: span.start - offset,
end: span.end - offset,
};
let sublevels_count = self.var_context.1.len();
// Completions for the given variable
if !var_str.is_empty() {
// Completion for $env.<tab>
if var_str == "$env" {
let env_vars = stack.get_env_vars(working_set.permanent_state);
// Return nested values
if sublevels_count > 0 {
// Extract the target var ($env.<target-var>)
let target_var = self.var_context.1[0].clone();
let target_var_str =
str::from_utf8(&target_var).unwrap_or_default().to_string();
// Everything after the target var is the nested level ($env.<target-var>.<nested_levels>...)
let nested_levels: Vec<Vec<u8>> =
self.var_context.1.clone().into_iter().skip(1).collect();
if let Some(val) = env_vars.get(&target_var_str) {
for suggestion in nested_suggestions(val, &nested_levels, current_span) {
if options.match_algorithm.matches_u8_insensitive(
options.case_sensitive,
suggestion.suggestion.value.as_bytes(),
&prefix,
) {
output.push(suggestion);
}
}
return output;
}
} else {
// No nesting provided, return all env vars
for env_var in env_vars {
if options.match_algorithm.matches_u8_insensitive(
options.case_sensitive,
env_var.0.as_bytes(),
&prefix,
) {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: env_var.0,
description: None,
style: None,
extra: None,
span: current_span,
append_whitespace: false,
},
kind: Some(SuggestionKind::Type(env_var.1.get_type())),
});
}
}
return output;
}
}
// Completions for $nu.<tab>
if var_str == "$nu" {
// Eval nu var
if let Ok(nuval) = eval_variable(
working_set.permanent_state,
stack,
nu_protocol::NU_VARIABLE_ID,
nu_protocol::Span::new(current_span.start, current_span.end),
) {
for suggestion in nested_suggestions(&nuval, &self.var_context.1, current_span)
{
if options.match_algorithm.matches_u8_insensitive(
options.case_sensitive,
suggestion.suggestion.value.as_bytes(),
&prefix,
) {
output.push(suggestion);
}
}
return output;
}
}
// Completion other variable types
if let Some(var_id) = var_id {
// Extract the variable value from the stack
let var = stack.get_var(var_id, Span::new(span.start, span.end));
// If the value exists and it's of type Record
if let Ok(value) = var {
for suggestion in nested_suggestions(&value, &self.var_context.1, current_span)
{
if options.match_algorithm.matches_u8_insensitive(
options.case_sensitive,
suggestion.suggestion.value.as_bytes(),
&prefix,
) {
output.push(suggestion);
}
}
return output;
}
}
}
// Variable completion (e.g: $en<tab> to complete $env)
let builtins = ["$nu", "$in", "$env"];
for builtin in builtins {
if options.match_algorithm.matches_u8_insensitive(
options.case_sensitive,
builtin.as_bytes(),
&prefix,
) {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: builtin.to_string(),
description: None,
style: None,
extra: None,
span: current_span,
append_whitespace: false,
},
// TODO is there a way to get the VarId to get the type???
kind: None,
});
}
matcher.add_semantic_suggestion(SemanticSuggestion {
suggestion: Suggestion {
value: builtin.to_string(),
span: current_span,
..Suggestion::default()
},
// TODO is there a way to get the VarId to get the type???
kind: None,
});
}
let mut add_candidate = |name, var_id: &VarId| {
matcher.add_semantic_suggestion(SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(name).to_string(),
span: current_span,
..Suggestion::default()
},
kind: Some(SuggestionKind::Type(
working_set.get_variable(*var_id).ty.clone(),
)),
})
};
// TODO: The following can be refactored (see find_commands_by_predicate() used in
// command_completions).
let mut removed_overlays = vec![];
// Working set scope vars
for scope_frame in working_set.delta.scope.iter().rev() {
for overlay_frame in scope_frame.active_overlays(&mut removed_overlays).rev() {
for v in &overlay_frame.vars {
if options.match_algorithm.matches_u8_insensitive(
options.case_sensitive,
v.0,
&prefix,
) {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(v.0).to_string(),
description: None,
style: None,
extra: None,
span: current_span,
append_whitespace: false,
},
kind: Some(SuggestionKind::Type(
working_set.get_variable(*v.1).ty.clone(),
)),
});
}
for (name, var_id) in &overlay_frame.vars {
add_candidate(name, var_id);
}
}
}
// Permanent state vars
// for scope in &self.engine_state.scope {
for overlay_frame in working_set
@ -203,125 +72,11 @@ impl Completer for VariableCompletion {
.active_overlays(&removed_overlays)
.rev()
{
for v in &overlay_frame.vars {
if options.match_algorithm.matches_u8_insensitive(
options.case_sensitive,
v.0,
&prefix,
) {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(v.0).to_string(),
description: None,
style: None,
extra: None,
span: current_span,
append_whitespace: false,
},
kind: Some(SuggestionKind::Type(
working_set.get_variable(*v.1).ty.clone(),
)),
});
}
for (name, var_id) in &overlay_frame.vars {
add_candidate(name, var_id);
}
}
output.dedup(); // TODO: Removes only consecutive duplicates, is it intended?
output
}
}
// Find recursively the values for sublevels
// if no sublevels are set it returns the current value
fn nested_suggestions(
val: &Value,
sublevels: &[Vec<u8>],
current_span: reedline::Span,
) -> Vec<SemanticSuggestion> {
let mut output: Vec<SemanticSuggestion> = vec![];
let value = recursive_value(val, sublevels).unwrap_or_else(Value::nothing);
let kind = SuggestionKind::Type(value.get_type());
match value {
Value::Record { val, .. } => {
// Add all the columns as completion
for col in val.columns() {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: col.clone(),
description: None,
style: None,
extra: None,
span: current_span,
append_whitespace: false,
},
kind: Some(kind.clone()),
});
}
output
}
Value::List { vals, .. } => {
for column_name in get_columns(vals.as_slice()) {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: column_name,
description: None,
style: None,
extra: None,
span: current_span,
append_whitespace: false,
},
kind: Some(kind.clone()),
});
}
output
}
_ => output,
}
}
// Extracts the recursive value (e.g: $var.a.b.c)
fn recursive_value(val: &Value, sublevels: &[Vec<u8>]) -> Result<Value, Span> {
// Go to next sublevel
if let Some((sublevel, next_sublevels)) = sublevels.split_first() {
let span = val.span();
match val {
Value::Record { val, .. } => {
if let Some((_, value)) = val.iter().find(|(key, _)| key.as_bytes() == sublevel) {
// If matches try to fetch recursively the next
recursive_value(value, next_sublevels)
} else {
// Current sublevel value not found
Err(span)
}
}
Value::List { vals, .. } => {
for col in get_columns(vals.as_slice()) {
if col.as_bytes() == *sublevel {
let val = val.get_data_by_key(&col).ok_or(span)?;
return recursive_value(&val, next_sublevels);
}
}
// Current sublevel value not found
Err(span)
}
_ => Ok(val.clone()),
}
} else {
Ok(val.clone())
}
}
impl MatchAlgorithm {
pub fn matches_u8_insensitive(&self, sensitive: bool, haystack: &[u8], needle: &[u8]) -> bool {
if sensitive {
self.matches_u8(haystack, needle)
} else {
self.matches_u8(&haystack.to_ascii_lowercase(), &needle.to_ascii_lowercase())
}
matcher.results()
}
}

161
crates/nu-cli/src/config_files.rs
View File

@ -2,13 +2,13 @@ use crate::util::eval_source;
#[cfg(feature = "plugin")]
use nu_path::canonicalize_with;
#[cfg(feature = "plugin")]
use nu_protocol::{engine::StateWorkingSet, report_error, ParseError, PluginRegistryFile, Spanned};
use nu_protocol::{engine::StateWorkingSet, ParseError, PluginRegistryFile, Spanned};
use nu_protocol::{
engine::{EngineState, Stack},
report_error_new, HistoryFileFormat, PipelineData,
report_shell_error, PipelineData,
};
#[cfg(feature = "plugin")]
use nu_utils::utils::perf;
use nu_utils::perf;
use std::path::PathBuf;
#[cfg(feature = "plugin")]
@ -16,16 +16,9 @@ const PLUGIN_FILE: &str = "plugin.msgpackz";
#[cfg(feature = "plugin")]
const OLD_PLUGIN_FILE: &str = "plugin.nu";
const HISTORY_FILE_TXT: &str = "history.txt";
const HISTORY_FILE_SQLITE: &str = "history.sqlite3";
#[cfg(feature = "plugin")]
pub fn read_plugin_file(
engine_state: &mut EngineState,
plugin_file: Option<Spanned<String>>,
storage_path: &str,
) {
use nu_protocol::ShellError;
pub fn read_plugin_file(engine_state: &mut EngineState, plugin_file: Option<Spanned<String>>) {
use nu_protocol::{shell_error::io::IoError, ShellError};
use std::path::Path;
let span = plugin_file.as_ref().map(|s| s.span);
@ -36,7 +29,7 @@ pub fn read_plugin_file(
.and_then(|p| Path::new(&p.item).extension())
.is_some_and(|ext| ext == "nu")
{
report_error_new(
report_shell_error(
engine_state,
&ShellError::GenericError {
error: "Wrong plugin file format".into(),
@ -52,14 +45,14 @@ pub fn read_plugin_file(
let mut start_time = std::time::Instant::now();
// Reading signatures from plugin registry file
// The plugin.msgpackz file stores the parsed signature collected from each registered plugin
add_plugin_file(engine_state, plugin_file.clone(), storage_path);
perf(
add_plugin_file(engine_state, plugin_file.clone());
perf!(
"add plugin file to engine_state",
start_time,
file!(),
line!(),
column!(),
engine_state.get_config().use_ansi_coloring,
engine_state
.get_config()
.use_ansi_coloring
.get(engine_state)
);
start_time = std::time::Instant::now();
@ -73,8 +66,7 @@ pub fn read_plugin_file(
log::warn!("Plugin file not found: {}", plugin_path.display());
// Try migration of an old plugin file if this wasn't a custom plugin file
if plugin_file.is_none() && migrate_old_plugin_file(engine_state, storage_path)
{
if plugin_file.is_none() && migrate_old_plugin_file(engine_state) {
let Ok(file) = std::fs::File::open(&plugin_path) else {
log::warn!("Failed to load newly migrated plugin file");
return;
@ -84,18 +76,14 @@ pub fn read_plugin_file(
return;
}
} else {
report_error_new(
report_shell_error(
engine_state,
&ShellError::GenericError {
error: format!(
"Error while opening plugin registry file: {}",
plugin_path.display()
),
msg: "plugin path defined here".into(),
span,
help: None,
inner: vec![err.into()],
},
&ShellError::Io(IoError::new_internal_with_path(
err.kind(),
"Could not open plugin registry file",
nu_protocol::location!(),
plugin_path,
)),
);
return;
}
@ -116,7 +104,7 @@ pub fn read_plugin_file(
Ok(contents) => contents,
Err(err) => {
log::warn!("Failed to read plugin registry file: {err:?}");
report_error_new(
report_shell_error(
engine_state,
&ShellError::GenericError {
error: format!(
@ -137,13 +125,13 @@ pub fn read_plugin_file(
}
};
perf(
perf!(
&format!("read plugin file {}", plugin_path.display()),
start_time,
file!(),
line!(),
column!(),
engine_state.get_config().use_ansi_coloring,
engine_state
.get_config()
.use_ansi_coloring
.get(engine_state)
);
start_time = std::time::Instant::now();
@ -152,30 +140,26 @@ pub fn read_plugin_file(
nu_plugin_engine::load_plugin_file(&mut working_set, &contents, span);
if let Err(err) = engine_state.merge_delta(working_set.render()) {
report_error_new(engine_state, &err);
report_shell_error(engine_state, &err);
return;
}
perf(
perf!(
&format!("load plugin file {}", plugin_path.display()),
start_time,
file!(),
line!(),
column!(),
engine_state.get_config().use_ansi_coloring,
engine_state
.get_config()
.use_ansi_coloring
.get(engine_state)
);
}
}
#[cfg(feature = "plugin")]
pub fn add_plugin_file(
engine_state: &mut EngineState,
plugin_file: Option<Spanned<String>>,
storage_path: &str,
) {
pub fn add_plugin_file(engine_state: &mut EngineState, plugin_file: Option<Spanned<String>>) {
use std::path::Path;
let working_set = StateWorkingSet::new(engine_state);
use nu_protocol::report_parse_error;
if let Ok(cwd) = engine_state.cwd_as_string(None) {
if let Some(plugin_file) = plugin_file {
@ -190,18 +174,18 @@ pub fn add_plugin_file(
engine_state.plugin_path = Some(path)
} else {
// It's an error if the directory for the plugin file doesn't exist.
report_error(
&working_set,
report_parse_error(
&StateWorkingSet::new(engine_state),
&ParseError::FileNotFound(
path_dir.to_string_lossy().into_owned(),
plugin_file.span,
),
);
}
} else if let Some(mut plugin_path) = nu_path::config_dir() {
} else if let Some(plugin_path) = nu_path::nu_config_dir() {
// Path to store plugins signatures
plugin_path.push(storage_path);
let mut plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
let mut plugin_path =
canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path.into());
plugin_path.push(PLUGIN_FILE);
let plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
engine_state.plugin_path = Some(plugin_path);
@ -222,7 +206,8 @@ pub fn eval_config_contents(
let prev_file = engine_state.file.take();
engine_state.file = Some(config_path.clone());
eval_source(
// TODO: ignore this error?
let _ = eval_source(
engine_state,
stack,
&contents,
@ -235,36 +220,18 @@ pub fn eval_config_contents(
engine_state.file = prev_file;
// Merge the environment in case env vars changed in the config
match engine_state.cwd(Some(stack)) {
Ok(cwd) => {
if let Err(e) = engine_state.merge_env(stack, cwd) {
report_error_new(engine_state, &e);
}
}
Err(e) => {
report_error_new(engine_state, &e);
}
if let Err(e) = engine_state.merge_env(stack) {
report_shell_error(engine_state, &e);
}
}
}
}
pub(crate) fn get_history_path(storage_path: &str, mode: HistoryFileFormat) -> Option<PathBuf> {
nu_path::config_dir().map(|mut history_path| {
history_path.push(storage_path);
history_path.push(match mode {
HistoryFileFormat::PlainText => HISTORY_FILE_TXT,
HistoryFileFormat::Sqlite => HISTORY_FILE_SQLITE,
});
history_path
})
}
#[cfg(feature = "plugin")]
pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -> bool {
pub fn migrate_old_plugin_file(engine_state: &EngineState) -> bool {
use nu_protocol::{
PluginExample, PluginIdentity, PluginRegistryItem, PluginRegistryItemData, PluginSignature,
ShellError,
shell_error::io::IoError, PluginExample, PluginIdentity, PluginRegistryItem,
PluginRegistryItemData, PluginSignature, ShellError,
};
use std::collections::BTreeMap;
@ -274,10 +241,9 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -
return false;
};
let Some(config_dir) = nu_path::config_dir().and_then(|mut dir| {
dir.push(storage_path);
nu_path::canonicalize_with(dir, &cwd).ok()
}) else {
let Some(config_dir) =
nu_path::nu_config_dir().and_then(|dir| nu_path::canonicalize_with(dir, &cwd).ok())
else {
return false;
};
@ -288,7 +254,7 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -
let old_contents = match std::fs::read(&old_plugin_file_path) {
Ok(old_contents) => old_contents,
Err(err) => {
report_error_new(
report_shell_error(
engine_state,
&ShellError::GenericError {
error: "Can't read old plugin file to migrate".into(),
@ -344,17 +310,28 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -
name: identity.name().to_owned(),
filename: identity.filename().to_owned(),
shell: identity.shell().map(|p| p.to_owned()),
data: PluginRegistryItemData::Valid { commands },
data: PluginRegistryItemData::Valid {
metadata: Default::default(),
commands,
},
});
}
// Write the new file
let new_plugin_file_path = config_dir.join(PLUGIN_FILE);
if let Err(err) = std::fs::File::create(&new_plugin_file_path)
.map_err(|e| e.into())
.map_err(|err| {
IoError::new_internal_with_path(
err.kind(),
"Could not create new plugin file",
nu_protocol::location!(),
new_plugin_file_path.clone(),
)
})
.map_err(ShellError::from)
.and_then(|file| contents.write_to(file, None))
{
report_error_new(
report_shell_error(
&engine_state,
&ShellError::GenericError {
error: "Failed to save migrated plugin file".into(),
@ -378,13 +355,13 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -
);
}
perf(
perf!(
"migrate old plugin file",
start_time,
file!(),
line!(),
column!(),
engine_state.get_config().use_ansi_coloring,
engine_state
.get_config()
.use_ansi_coloring
.get(&engine_state)
);
true
}

64
crates/nu-cli/src/eval_cmds.rs
View File

@ -1,42 +1,76 @@
use log::info;
use nu_engine::{convert_env_values, eval_block};
use nu_engine::eval_block;
use nu_parser::parse;
use nu_protocol::{
cli_error::report_compile_error,
debugger::WithoutDebug,
engine::{EngineState, Stack, StateWorkingSet},
report_error, PipelineData, ShellError, Spanned, Value,
report_parse_error, report_parse_warning, PipelineData, ShellError, Spanned, Value,
};
use std::sync::Arc;
use crate::util::print_pipeline;
#[derive(Default)]
pub struct EvaluateCommandsOpts {
pub table_mode: Option<Value>,
pub error_style: Option<Value>,
pub no_newline: bool,
}
/// Run a command (or commands) given to us by the user
pub fn evaluate_commands(
commands: &Spanned<String>,
engine_state: &mut EngineState,
stack: &mut Stack,
input: PipelineData,
table_mode: Option<Value>,
no_newline: bool,
opts: EvaluateCommandsOpts,
) -> Result<(), ShellError> {
// Translate environment variables from Strings to Values
convert_env_values(engine_state, stack)?;
let EvaluateCommandsOpts {
table_mode,
error_style,
no_newline,
} = opts;
// Handle the configured error style early
if let Some(e_style) = error_style {
match e_style.coerce_str()?.parse() {
Ok(e_style) => {
Arc::make_mut(&mut engine_state.config).error_style = e_style;
}
Err(err) => {
return Err(ShellError::GenericError {
error: "Invalid value for `--error-style`".into(),
msg: err.into(),
span: Some(e_style.span()),
help: None,
inner: vec![],
});
}
}
}
// Parse the source code
let (block, delta) = {
if let Some(ref t_mode) = table_mode {
let mut config = engine_state.get_config().clone();
config.table_mode = t_mode.coerce_str()?.parse().unwrap_or_default();
engine_state.set_config(config);
Arc::make_mut(&mut engine_state.config).table.mode =
t_mode.coerce_str()?.parse().unwrap_or_default();
}
let mut working_set = StateWorkingSet::new(engine_state);
let output = parse(&mut working_set, None, commands.item.as_bytes(), false);
if let Some(warning) = working_set.parse_warnings.first() {
report_error(&working_set, warning);
report_parse_warning(&working_set, warning);
}
if let Some(err) = working_set.parse_errors.first() {
report_error(&working_set, err);
report_parse_error(&working_set, err);
std::process::exit(1);
}
if let Some(err) = working_set.compile_errors.first() {
report_compile_error(&working_set, err);
std::process::exit(1);
}
@ -54,15 +88,11 @@ pub fn evaluate_commands(
}
if let Some(t_mode) = table_mode {
Arc::make_mut(&mut engine_state.config).table_mode =
Arc::make_mut(&mut engine_state.config).table.mode =
t_mode.coerce_str()?.parse().unwrap_or_default();
}
if let Some(status) = pipeline.print(engine_state, stack, no_newline, false)? {
if status.code() != 0 {
std::process::exit(status.code())
}
}
print_pipeline(engine_state, stack, pipeline, no_newline)?;
info!("evaluate {}:{}:{}", file!(), line!(), column!());

70
crates/nu-cli/src/eval_file.rs
View File

@ -1,14 +1,17 @@
use crate::util::eval_source;
use crate::util::{eval_source, print_pipeline};
use log::{info, trace};
use nu_engine::{convert_env_values, eval_block};
use nu_engine::eval_block;
use nu_parser::parse;
use nu_path::canonicalize_with;
use nu_protocol::{
cli_error::report_compile_error,
debugger::WithoutDebug,
engine::{EngineState, Stack, StateWorkingSet},
report_error, PipelineData, ShellError, Span, Value,
report_parse_error, report_parse_warning,
shell_error::io::*,
PipelineData, ShellError, Span, Value,
};
use std::sync::Arc;
use std::{path::PathBuf, sync::Arc};
/// Entry point for evaluating a file.
///
@ -21,16 +24,16 @@ pub fn evaluate_file(
stack: &mut Stack,
input: PipelineData,
) -> Result<(), ShellError> {
// Convert environment variables from Strings to Values and store them in the engine state.
convert_env_values(engine_state, stack)?;
let cwd = engine_state.cwd_as_string(Some(stack))?;
let file_path =
canonicalize_with(&path, cwd).map_err(|err| ShellError::FileNotFoundCustom {
msg: format!("Could not access file '{path}': {err}"),
span: Span::unknown(),
})?;
let file_path = canonicalize_with(&path, cwd).map_err(|err| {
IoError::new_internal_with_path(
err.kind().not_found_as(NotFound::File),
"Could not access file",
nu_protocol::location!(),
PathBuf::from(&path),
)
})?;
let file_path_str = file_path
.to_str()
@ -42,18 +45,24 @@ pub fn evaluate_file(
span: Span::unknown(),
})?;
let file = std::fs::read(&file_path).map_err(|err| ShellError::FileNotFoundCustom {
msg: format!("Could not read file '{file_path_str}': {err}"),
span: Span::unknown(),
let file = std::fs::read(&file_path).map_err(|err| {
IoError::new_internal_with_path(
err.kind().not_found_as(NotFound::File),
"Could not read file",
nu_protocol::location!(),
file_path.clone(),
)
})?;
engine_state.file = Some(file_path.clone());
let parent = file_path
.parent()
.ok_or_else(|| ShellError::FileNotFoundCustom {
msg: format!("The file path '{file_path_str}' does not have a parent"),
span: Span::unknown(),
})?;
let parent = file_path.parent().ok_or_else(|| {
IoError::new_internal_with_path(
ErrorKind::DirectoryNotFound,
"The file path does not have a parent",
nu_protocol::location!(),
file_path.clone(),
)
})?;
stack.add_env_var(
"FILE_PWD".to_string(),
@ -76,9 +85,18 @@ pub fn evaluate_file(
trace!("parsing file: {}", file_path_str);
let block = parse(&mut working_set, Some(file_path_str), &file, false);
if let Some(warning) = working_set.parse_warnings.first() {
report_parse_warning(&working_set, warning);
}
// If any parse errors were found, report the first error and exit.
if let Some(err) = working_set.parse_errors.first() {
report_error(&working_set, err);
report_parse_error(&working_set, err);
std::process::exit(1);
}
if let Some(err) = working_set.compile_errors.first() {
report_compile_error(&working_set, err);
std::process::exit(1);
}
@ -109,11 +127,7 @@ pub fn evaluate_file(
};
// Print the pipeline output of the last command of the file.
if let Some(status) = pipeline.print(engine_state, stack, true, false)? {
if status.code() != 0 {
std::process::exit(status.code())
}
}
print_pipeline(engine_state, stack, pipeline, true)?;
// Invoke the main command with arguments.
// Arguments with whitespaces are quoted, thus can be safely concatenated by whitespace.
@ -131,7 +145,7 @@ pub fn evaluate_file(
};
if exit_code != 0 {
std::process::exit(exit_code)
std::process::exit(exit_code);
}
info!("evaluate {}:{}:{}", file!(), line!(), column!());

4
crates/nu-cli/src/lib.rs
View File

@ -1,3 +1,4 @@
#![doc = include_str!("../README.md")]
mod commands;
mod completions;
mod config_files;
@ -17,10 +18,9 @@ mod validation;
pub use commands::add_cli_context;
pub use completions::{FileCompletion, NuCompleter, SemanticSuggestion, SuggestionKind};
pub use config_files::eval_config_contents;
pub use eval_cmds::evaluate_commands;
pub use eval_cmds::{evaluate_commands, EvaluateCommandsOpts};
pub use eval_file::evaluate_file;
pub use menus::NuHelpCompleter;
pub use nu_cmd_base::util::get_init_cwd;
pub use nu_highlight::NuHighlight;
pub use print::Print;
pub use prompt::NushellPrompt;

84
crates/nu-cli/src/menus/help_completions.rs
View File

@ -1,62 +1,73 @@
use nu_engine::documentation::get_flags_section;
use nu_protocol::{engine::EngineState, levenshtein_distance};
use nu_engine::documentation::{get_flags_section, HelpStyle};
use nu_protocol::{engine::EngineState, levenshtein_distance, Config};
use nu_utils::IgnoreCaseExt;
use reedline::{Completer, Suggestion};
use std::{fmt::Write, sync::Arc};
pub struct NuHelpCompleter(Arc<EngineState>);
pub struct NuHelpCompleter {
engine_state: Arc<EngineState>,
config: Arc<Config>,
}
impl NuHelpCompleter {
pub fn new(engine_state: Arc<EngineState>) -> Self {
Self(engine_state)
pub fn new(engine_state: Arc<EngineState>, config: Arc<Config>) -> Self {
Self {
engine_state,
config,
}
}
fn completion_helper(&self, line: &str, pos: usize) -> Vec<Suggestion> {
let full_commands = self.0.get_signatures_with_examples(false);
let folded_line = line.to_folded_case();
//Vec<(Signature, Vec<Example>, bool, bool)> {
let mut commands = full_commands
.iter()
.filter(|(sig, _, _, _, _)| {
sig.name.to_folded_case().contains(&folded_line)
|| sig.usage.to_folded_case().contains(&folded_line)
|| sig
.search_terms
.iter()
let mut help_style = HelpStyle::default();
help_style.update_from_config(&self.engine_state, &self.config);
let mut commands = self
.engine_state
.get_decls_sorted(false)
.into_iter()
.filter_map(|(_, decl_id)| {
let decl = self.engine_state.get_decl(decl_id);
(decl.name().to_folded_case().contains(&folded_line)
|| decl.description().to_folded_case().contains(&folded_line)
|| decl
.search_terms()
.into_iter()
.any(|term| term.to_folded_case().contains(&folded_line))
|| sig.extra_usage.to_folded_case().contains(&folded_line)
|| decl
.extra_description()
.to_folded_case()
.contains(&folded_line))
.then_some(decl)
})
.collect::<Vec<_>>();
commands.sort_by(|(a, _, _, _, _), (b, _, _, _, _)| {
let a_distance = levenshtein_distance(line, &a.name);
let b_distance = levenshtein_distance(line, &b.name);
a_distance.cmp(&b_distance)
});
commands.sort_by_cached_key(|decl| levenshtein_distance(line, decl.name()));
commands
.into_iter()
.map(|(sig, examples, _, _, _)| {
.map(|decl| {
let mut long_desc = String::new();
let usage = &sig.usage;
if !usage.is_empty() {
long_desc.push_str(usage);
let description = decl.description();
if !description.is_empty() {
long_desc.push_str(description);
long_desc.push_str("\r\n\r\n");
}
let extra_usage = &sig.extra_usage;
if !extra_usage.is_empty() {
long_desc.push_str(extra_usage);
let extra_desc = decl.extra_description();
if !extra_desc.is_empty() {
long_desc.push_str(extra_desc);
long_desc.push_str("\r\n\r\n");
}
let sig = decl.signature();
let _ = write!(long_desc, "Usage:\r\n > {}\r\n", sig.call_signature());
if !sig.named.is_empty() {
long_desc.push_str(&get_flags_section(Some(&*self.0.clone()), sig, |v| {
v.to_parsable_string(", ", &self.0.config)
long_desc.push_str(&get_flags_section(&sig, &help_style, |v| {
v.to_parsable_string(", ", &self.config)
}))
}
@ -72,7 +83,7 @@ impl NuHelpCompleter {
let opt_suffix = if let Some(value) = &positional.default_value {
format!(
" (optional, default: {})",
&value.to_parsable_string(", ", &self.0.config),
&value.to_parsable_string(", ", &self.config),
)
} else {
(" (optional)").to_string()
@ -93,21 +104,21 @@ impl NuHelpCompleter {
}
}
let extra: Vec<String> = examples
let extra: Vec<String> = decl
.examples()
.iter()
.map(|example| example.example.replace('\n', "\r\n"))
.collect();
Suggestion {
value: sig.name.clone(),
value: decl.name().into(),
description: Some(long_desc),
style: None,
extra: Some(extra),
span: reedline::Span {
start: pos - line.len(),
end: pos,
},
append_whitespace: false,
..Suggestion::default()
}
})
.collect()
@ -138,7 +149,8 @@ mod test {
) {
let engine_state =
nu_command::add_shell_command_context(nu_cmd_lang::create_default_context());
let mut completer = NuHelpCompleter::new(engine_state.into());
let config = engine_state.get_config().clone();
let mut completer = NuHelpCompleter::new(engine_state.into(), config);
let suggestions = completer.complete(line, end);
assert_eq!(

16
crates/nu-cli/src/menus/menu_completions.rs
View File

@ -2,7 +2,7 @@ use nu_engine::eval_block;
use nu_protocol::{
debugger::WithoutDebug,
engine::{EngineState, Stack},
IntoPipelineData, Span, Value,
BlockId, IntoPipelineData, Span, Value,
};
use reedline::{menu_functions::parse_selection_char, Completer, Suggestion};
use std::sync::Arc;
@ -10,7 +10,7 @@ use std::sync::Arc;
const SELECTION_CHAR: char = '!';
pub struct NuMenuCompleter {
block_id: usize,
block_id: BlockId,
span: Span,
stack: Stack,
engine_state: Arc<EngineState>,
@ -19,7 +19,7 @@ pub struct NuMenuCompleter {
impl NuMenuCompleter {
pub fn new(
block_id: usize,
block_id: BlockId,
span: Span,
stack: Stack,
engine_state: Arc<EngineState>,
@ -28,7 +28,7 @@ impl NuMenuCompleter {
Self {
block_id,
span,
stack: stack.reset_out_dest().capture(),
stack: stack.reset_out_dest().collect_value(),
engine_state,
only_buffer_difference,
}
@ -142,10 +142,9 @@ fn convert_to_suggestions(
vec![Suggestion {
value: text,
description,
style: None,
extra,
span,
append_whitespace: false,
..Suggestion::default()
}]
}
Value::List { vals, .. } => vals
@ -154,9 +153,6 @@ fn convert_to_suggestions(
.collect(),
_ => vec![Suggestion {
value: format!("Not a record: {value:?}"),
description: None,
style: None,
extra: None,
span: reedline::Span {
start: if only_buffer_difference {
pos - line.len()
@ -169,7 +165,7 @@ fn convert_to_suggestions(
line.len()
},
},
append_whitespace: false,
..Suggestion::default()
}],
}
}

15
crates/nu-cli/src/nu_highlight.rs
View File

@ -1,3 +1,5 @@
use std::sync::Arc;
use nu_engine::command_prelude::*;
use reedline::{Highlighter, StyledText};
@ -15,7 +17,7 @@ impl Command for NuHighlight {
.input_output_types(vec![(Type::String, Type::String)])
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"Syntax highlight the input string."
}
@ -32,14 +34,11 @@ impl Command for NuHighlight {
) -> Result<PipelineData, ShellError> {
let head = call.head;
let ctrlc = engine_state.ctrlc.clone();
let engine_state = std::sync::Arc::new(engine_state.clone());
let config = engine_state.get_config().clone();
let signals = engine_state.signals();
let highlighter = crate::NuHighlighter {
engine_state,
stack: std::sync::Arc::new(stack.clone()),
config,
engine_state: Arc::new(engine_state.clone()),
stack: Arc::new(stack.clone()),
};
input.map(
@ -50,7 +49,7 @@ impl Command for NuHighlight {
}
Err(err) => Value::error(err, head),
},
ctrlc,
signals,
)
}

42
crates/nu-cli/src/print.rs
View File

@ -1,4 +1,5 @@
use nu_engine::command_prelude::*;
use nu_protocol::ByteStreamSource;
#[derive(Clone)]
pub struct Print;
@ -22,14 +23,19 @@ impl Command for Print {
Some('n'),
)
.switch("stderr", "print to stderr instead of stdout", Some('e'))
.switch(
"raw",
"print without formatting (including binary data)",
Some('r'),
)
.category(Category::Strings)
}
fn usage(&self) -> &str {
fn description(&self) -> &str {
"Print the given values to stdout."
}
fn extra_usage(&self) -> &str {
fn extra_description(&self) -> &str {
r#"Unlike `echo`, this command does not return any value (`print | describe` will return "nothing").
Since this command has no output, there is no point in piping it with other commands.
@ -45,20 +51,39 @@ Since this command has no output, there is no point in piping it with other comm
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
mut input: PipelineData,
) -> Result<PipelineData, ShellError> {
let args: Vec<Value> = call.rest(engine_state, stack, 0)?;
let no_newline = call.has_flag(engine_state, stack, "no-newline")?;
let to_stderr = call.has_flag(engine_state, stack, "stderr")?;
let raw = call.has_flag(engine_state, stack, "raw")?;
// This will allow for easy printing of pipelines as well
if !args.is_empty() {
for arg in args {
arg.into_pipeline_data()
.print(engine_state, stack, no_newline, to_stderr)?;
if raw {
arg.into_pipeline_data()
.print_raw(engine_state, no_newline, to_stderr)?;
} else {
arg.into_pipeline_data().print_table(
engine_state,
stack,
no_newline,
to_stderr,
)?;
}
}
} else if !input.is_nothing() {
input.print(engine_state, stack, no_newline, to_stderr)?;
if let PipelineData::ByteStream(stream, _) = &mut input {
if let ByteStreamSource::Child(child) = stream.source_mut() {
child.ignore_error(true);
}
}
if raw {
input.print_raw(engine_state, no_newline, to_stderr)?;
} else {
input.print_table(engine_state, stack, no_newline, to_stderr)?;
}
}
Ok(PipelineData::empty())
@ -76,6 +101,11 @@ Since this command has no output, there is no point in piping it with other comm
example: r#"print (2 + 3)"#,
result: None,
},
Example {
description: "Print 'ABC' from binary data",
example: r#"0x[41 42 43] | print --raw"#,
result: None,
},
]
}
}

12
crates/nu-cli/src/prompt.rs
View File

@ -1,10 +1,7 @@
use crate::prompt_update::{
POST_PROMPT_MARKER, PRE_PROMPT_MARKER, VSCODE_POST_PROMPT_MARKER, VSCODE_PRE_PROMPT_MARKER,
};
use nu_protocol::{
engine::{EngineState, Stack},
Value,
};
use nu_protocol::engine::{EngineState, Stack};
#[cfg(windows)]
use nu_utils::enable_vt_processing;
use reedline::{
@ -124,8 +121,11 @@ impl Prompt for NushellPrompt {
.replace('\n', "\r\n");
if self.shell_integration_osc633 {
if self.stack.get_env_var(&self.engine_state, "TERM_PROGRAM")
== Some(Value::test_string("vscode"))
if self
.stack
.get_env_var(&self.engine_state, "TERM_PROGRAM")
.and_then(|v| v.as_str().ok())
== Some("vscode")
{
// We're in vscode and we have osc633 enabled
format!("{VSCODE_PRE_PROMPT_MARKER}{prompt}{VSCODE_POST_PROMPT_MARKER}").into()

53
crates/nu-cli/src/prompt_update.rs
View File

@ -1,9 +1,9 @@
use crate::NushellPrompt;
use log::trace;
use log::{trace, warn};
use nu_engine::ClosureEvalOnce;
use nu_protocol::{
engine::{EngineState, Stack},
report_error_new, Config, PipelineData, Value,
report_shell_error, Config, PipelineData, Value,
};
use reedline::Prompt;
@ -30,27 +30,21 @@ pub(crate) const TRANSIENT_PROMPT_MULTILINE_INDICATOR: &str =
pub(crate) const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
pub(crate) const POST_PROMPT_MARKER: &str = "\x1b]133;B\x1b\\";
pub(crate) const PRE_EXECUTION_MARKER: &str = "\x1b]133;C\x1b\\";
#[allow(dead_code)]
pub(crate) const POST_EXECUTION_MARKER_PREFIX: &str = "\x1b]133;D;";
#[allow(dead_code)]
pub(crate) const POST_EXECUTION_MARKER_SUFFIX: &str = "\x1b\\";
// OSC633 is the same as OSC133 but specifically for VSCode
pub(crate) const VSCODE_PRE_PROMPT_MARKER: &str = "\x1b]633;A\x1b\\";
pub(crate) const VSCODE_POST_PROMPT_MARKER: &str = "\x1b]633;B\x1b\\";
#[allow(dead_code)]
pub(crate) const VSCODE_PRE_EXECUTION_MARKER: &str = "\x1b]633;C\x1b\\";
#[allow(dead_code)]
//"\x1b]633;D;{}\x1b\\"
pub(crate) const VSCODE_POST_EXECUTION_MARKER_PREFIX: &str = "\x1b]633;D;";
#[allow(dead_code)]
pub(crate) const VSCODE_POST_EXECUTION_MARKER_SUFFIX: &str = "\x1b\\";
#[allow(dead_code)]
pub(crate) const VSCODE_COMMANDLINE_MARKER: &str = "\x1b]633;E\x1b\\";
#[allow(dead_code)]
//"\x1b]633;E;{}\x1b\\"
pub(crate) const VSCODE_COMMANDLINE_MARKER_PREFIX: &str = "\x1b]633;E;";
pub(crate) const VSCODE_COMMANDLINE_MARKER_SUFFIX: &str = "\x1b\\";
// "\x1b]633;P;Cwd={}\x1b\\"
pub(crate) const VSCODE_CWD_PROPERTY_MARKER_PREFIX: &str = "\x1b]633;P;Cwd=";
#[allow(dead_code)]
pub(crate) const VSCODE_CWD_PROPERTY_MARKER_SUFFIX: &str = "\x1b\\";
pub(crate) const RESET_APPLICATION_MODE: &str = "\x1b[?1l";
@ -65,7 +59,7 @@ fn get_prompt_string(
.get_env_var(engine_state, prompt)
.and_then(|v| match v {
Value::Closure { val, .. } => {
let result = ClosureEvalOnce::new(engine_state, stack, *val)
let result = ClosureEvalOnce::new(engine_state, stack, val.as_ref().clone())
.run_with_input(PipelineData::Empty);
trace!(
@ -77,7 +71,7 @@ fn get_prompt_string(
result
.map_err(|err| {
report_error_new(engine_state, &err);
report_shell_error(engine_state, &err);
})
.ok()
}
@ -86,18 +80,19 @@ fn get_prompt_string(
})
.and_then(|pipeline_data| {
let output = pipeline_data.collect_string("", config).ok();
let ansi_output = output.map(|mut x| {
// Always reset the color at the start of the right prompt
// to ensure there is no ansi bleed over
if x.is_empty() && prompt == PROMPT_COMMAND_RIGHT {
x.insert_str(0, "\x1b[0m")
};
output.map(|mut x| {
// Just remove the very last newline.
if x.ends_with('\n') {
x.pop();
}
if x.ends_with('\r') {
x.pop();
}
x
})
});
// Let's keep this for debugging purposes with nu --log-level warn
warn!("{}:{}:{} {:?}", file!(), line!(), column!(), ansi_output);
ansi_output
})
}
@ -115,13 +110,17 @@ pub(crate) fn update_prompt(
// Now that we have the prompt string lets ansify it.
// <133 A><prompt><133 B><command><133 C><command output>
let left_prompt_string = if config.shell_integration_osc633 {
if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) {
let left_prompt_string = if config.shell_integration.osc633 {
if stack
.get_env_var(engine_state, "TERM_PROGRAM")
.and_then(|v| v.as_str().ok())
== Some("vscode")
{
// We're in vscode and we have osc633 enabled
Some(format!(
"{VSCODE_PRE_PROMPT_MARKER}{configured_left_prompt_string}{VSCODE_POST_PROMPT_MARKER}"
))
} else if config.shell_integration_osc133 {
} else if config.shell_integration.osc133 {
// If we're in VSCode but we don't find the env var, but we have osc133 set, then use it
Some(format!(
"{PRE_PROMPT_MARKER}{configured_left_prompt_string}{POST_PROMPT_MARKER}"
@ -129,7 +128,7 @@ pub(crate) fn update_prompt(
} else {
configured_left_prompt_string.into()
}
} else if config.shell_integration_osc133 {
} else if config.shell_integration.osc133 {
Some(format!(
"{PRE_PROMPT_MARKER}{configured_left_prompt_string}{POST_PROMPT_MARKER}"
))

718
crates/nu-cli/src/reedline_config.rs
View File

File diff suppressed because it is too large Load Diff

750
crates/nu-cli/src/repl.rs
View File

File diff suppressed because it is too large Load Diff

67
crates/nu-cli/src/syntax_highlight.rs
View File

@ -6,7 +6,7 @@ use nu_parser::{flatten_block, parse, FlatShape};
use nu_protocol::{
ast::{Block, Expr, Expression, PipelineRedirection, RecordItem},
engine::{EngineState, Stack, StateWorkingSet},
Config, Span,
Span,
};
use reedline::{Highlighter, StyledText};
use std::sync::Arc;
@ -14,15 +14,14 @@ use std::sync::Arc;
pub struct NuHighlighter {
pub engine_state: Arc<EngineState>,
pub stack: Arc<Stack>,
pub config: Config,
}
impl Highlighter for NuHighlighter {
fn highlight(&self, line: &str, _cursor: usize) -> StyledText {
trace!("highlighting: {}", line);
let highlight_resolved_externals =
self.engine_state.get_config().highlight_resolved_externals;
let config = self.stack.get_config(&self.engine_state);
let highlight_resolved_externals = config.highlight_resolved_externals;
let mut working_set = StateWorkingSet::new(&self.engine_state);
let block = parse(&mut working_set, None, line.as_bytes(), false);
let (shapes, global_span_offset) = {
@ -88,7 +87,7 @@ impl Highlighter for NuHighlighter {
.to_string();
let mut add_colored_token = |shape: &FlatShape, text: String| {
output.push((get_shape_color(shape.as_str(), &self.config), text));
output.push((get_shape_color(shape.as_str(), &config), text));
};
match shape.1 {
@ -128,9 +127,9 @@ impl Highlighter for NuHighlighter {
let start = part.start - span.start;
let end = part.end - span.start;
let text = next_token[start..end].to_string();
let mut style = get_shape_color(shape.as_str(), &self.config);
let mut style = get_shape_color(shape.as_str(), &config);
if highlight {
style = get_matching_brackets_style(style, &self.config);
style = get_matching_brackets_style(style, &config);
}
output.push((style, text));
}
@ -138,14 +137,13 @@ impl Highlighter for NuHighlighter {
FlatShape::Filepath => add_colored_token(&shape.1, next_token),
FlatShape::Directory => add_colored_token(&shape.1, next_token),
FlatShape::GlobInterpolation => add_colored_token(&shape.1, next_token),
FlatShape::GlobPattern => add_colored_token(&shape.1, next_token),
FlatShape::Variable(_) | FlatShape::VarDecl(_) => {
add_colored_token(&shape.1, next_token)
}
FlatShape::Flag => add_colored_token(&shape.1, next_token),
FlatShape::Pipe => add_colored_token(&shape.1, next_token),
FlatShape::And => add_colored_token(&shape.1, next_token),
FlatShape::Or => add_colored_token(&shape.1, next_token),
FlatShape::Redirection => add_colored_token(&shape.1, next_token),
FlatShape::Custom(..) => add_colored_token(&shape.1, next_token),
FlatShape::MatchPattern => add_colored_token(&shape.1, next_token),
@ -311,6 +309,7 @@ fn find_matching_block_end_in_expr(
.unwrap_or(expression.span.start);
return match &expression.expr {
// TODO: Can't these be handled with an `_ => None` branch? Refactor
Expr::Bool(_) => None,
Expr::Int(_) => None,
Expr::Float(_) => None,
@ -337,6 +336,28 @@ fn find_matching_block_end_in_expr(
Expr::Nothing => None,
Expr::Garbage => None,
Expr::AttributeBlock(ab) => ab
.attributes
.iter()
.find_map(|attr| {
find_matching_block_end_in_expr(
line,
working_set,
&attr.expr,
global_span_offset,
global_cursor_offset,
)
})
.or_else(|| {
find_matching_block_end_in_expr(
line,
working_set,
&ab.item,
global_span_offset,
global_cursor_offset,
)
}),
Expr::Table(table) => {
if expr_last == global_cursor_offset {
// cursor is at table end
@ -429,6 +450,14 @@ fn find_matching_block_end_in_expr(
)
}),
Expr::Collect(_, expr) => find_matching_block_end_in_expr(
line,
working_set,
expr,
global_span_offset,
global_cursor_offset,
),
Expr::Block(block_id)
| Expr::Closure(block_id)
| Expr::RowCondition(block_id)
@ -452,15 +481,17 @@ fn find_matching_block_end_in_expr(
}
}
Expr::StringInterpolation(exprs) => exprs.iter().find_map(|expr| {
find_matching_block_end_in_expr(
line,
working_set,
expr,
global_span_offset,
global_cursor_offset,
)
}),
Expr::StringInterpolation(exprs) | Expr::GlobInterpolation(exprs, _) => {
exprs.iter().find_map(|expr| {
find_matching_block_end_in_expr(
line,
working_set,
expr,
global_span_offset,
global_cursor_offset,
)
})
}
Expr::List(list) => {
if expr_last == global_cursor_offset {

122
crates/nu-cli/src/util.rs
View File

@ -1,14 +1,18 @@
#![allow(clippy::byte_char_slices)]
use nu_cmd_base::hook::eval_hook;
use nu_engine::{eval_block, eval_block_with_early_return};
use nu_parser::{escape_quote_string, lex, parse, unescape_unquote_string, Token, TokenContents};
use nu_parser::{lex, parse, unescape_unquote_string, Token, TokenContents};
use nu_protocol::{
cli_error::report_compile_error,
debugger::WithoutDebug,
engine::{EngineState, Stack, StateWorkingSet},
report_error, report_error_new, PipelineData, ShellError, Span, Value,
report_parse_error, report_parse_warning, report_shell_error, PipelineData, ShellError, Span,
Value,
};
#[cfg(windows)]
use nu_utils::enable_vt_processing;
use nu_utils::utils::perf;
use nu_utils::{escape_quote_string, perf};
use std::path::Path;
// This will collect environment variables from std::env and adds them to a stack.
@ -39,7 +43,7 @@ fn gather_env_vars(
init_cwd: &Path,
) {
fn report_capture_error(engine_state: &EngineState, env_str: &str, msg: &str) {
report_error_new(
report_shell_error(
engine_state,
&ShellError::GenericError {
error: format!("Environment variable was not captured: {env_str}"),
@ -70,7 +74,7 @@ fn gather_env_vars(
}
None => {
// Could not capture current working directory
report_error_new(
report_shell_error(
engine_state,
&ShellError::GenericError {
error: "Current directory is not a valid utf-8 path".into(),
@ -128,7 +132,7 @@ fn gather_env_vars(
working_set.error(err);
}
if working_set.parse_errors.first().is_some() {
if !working_set.parse_errors.is_empty() {
report_capture_error(
engine_state,
&String::from_utf8_lossy(contents),
@ -172,7 +176,7 @@ fn gather_env_vars(
working_set.error(err);
}
if working_set.parse_errors.first().is_some() {
if !working_set.parse_errors.is_empty() {
report_capture_error(
engine_state,
&String::from_utf8_lossy(contents),
@ -199,6 +203,35 @@ fn gather_env_vars(
}
}
/// Print a pipeline with formatting applied based on display_output hook.
///
/// This function should be preferred when printing values resulting from a completed evaluation.
/// For values printed as part of a command's execution, such as values printed by the `print` command,
/// the `PipelineData::print_table` function should be preferred instead as it is not config-dependent.
///
/// `no_newline` controls if we need to attach newline character to output.
pub fn print_pipeline(
engine_state: &mut EngineState,
stack: &mut Stack,
pipeline: PipelineData,
no_newline: bool,
) -> Result<(), ShellError> {
if let Some(hook) = engine_state.get_config().hooks.display_output.clone() {
let pipeline = eval_hook(
engine_state,
stack,
Some(pipeline),
vec![],
&hook,
"display_output",
)?;
pipeline.print_raw(engine_state, no_newline, false)
} else {
// if display_output isn't set, we should still prefer to print with some formatting
pipeline.print_table(engine_state, stack, no_newline, false)
}
}
pub fn eval_source(
engine_state: &mut EngineState,
stack: &mut Stack,
@ -210,31 +243,32 @@ pub fn eval_source(
let start_time = std::time::Instant::now();
let exit_code = match evaluate_source(engine_state, stack, source, fname, input, allow_return) {
Ok(code) => code.unwrap_or(0),
Ok(failed) => {
let code = failed.into();
stack.set_last_exit_code(code, Span::unknown());
code
}
Err(err) => {
report_error_new(engine_state, &err);
1
report_shell_error(engine_state, &err);
let code = err.exit_code();
stack.set_last_error(&err);
code.unwrap_or(0)
}
};
stack.add_env_var(
"LAST_EXIT_CODE".to_string(),
Value::int(exit_code.into(), Span::unknown()),
);
// reset vt processing, aka ansi because illbehaved externals can break it
#[cfg(windows)]
{
let _ = enable_vt_processing();
}
perf(
perf!(
&format!("eval_source {}", &fname),
start_time,
file!(),
line!(),
column!(),
engine_state.get_config().use_ansi_coloring,
engine_state
.get_config()
.use_ansi_coloring
.get(engine_state)
);
exit_code
@ -247,7 +281,7 @@ fn evaluate_source(
fname: &str,
input: PipelineData,
allow_return: bool,
) -> Result<Option<i32>, ShellError> {
) -> Result<bool, ShellError> {
let (block, delta) = {
let mut working_set = StateWorkingSet::new(engine_state);
let output = parse(
@ -257,12 +291,17 @@ fn evaluate_source(
false,
);
if let Some(warning) = working_set.parse_warnings.first() {
report_error(&working_set, warning);
report_parse_warning(&working_set, warning);
}
if let Some(err) = working_set.parse_errors.first() {
report_error(&working_set, err);
return Ok(Some(1));
report_parse_error(&working_set, err);
return Ok(true);
}
if let Some(err) = working_set.compile_errors.first() {
report_compile_error(&working_set, err);
return Ok(true);
}
(output, working_set.render())
@ -276,25 +315,10 @@ fn evaluate_source(
eval_block::<WithoutDebug>(engine_state, stack, &block, input)
}?;
let status = if let PipelineData::ByteStream(stream, ..) = pipeline {
stream.print(false)?
} else {
if let Some(hook) = engine_state.get_config().hooks.display_output.clone() {
let pipeline = eval_hook(
engine_state,
stack,
Some(pipeline),
vec![],
&hook,
"display_output",
)?;
pipeline.print(engine_state, stack, false, false)
} else {
pipeline.print(engine_state, stack, true, false)
}?
};
let no_newline = matches!(&pipeline, &PipelineData::ByteStream(..));
print_pipeline(engine_state, stack, pipeline, no_newline)?;
Ok(status.map(|status| status.code()))
Ok(false)
}
#[cfg(test)]
@ -319,16 +343,10 @@ mod test {
let env = engine_state.render_env_vars();
assert!(
matches!(env.get(&"FOO".to_string()), Some(&Value::String { val, .. }) if val == "foo")
);
assert!(
matches!(env.get(&"SYMBOLS".to_string()), Some(&Value::String { val, .. }) if val == symbols)
);
assert!(
matches!(env.get(&symbols.to_string()), Some(&Value::String { val, .. }) if val == "symbols")
);
assert!(env.get(&"PWD".to_string()).is_some());
assert!(matches!(env.get("FOO"), Some(&Value::String { val, .. }) if val == "foo"));
assert!(matches!(env.get("SYMBOLS"), Some(&Value::String { val, .. }) if val == symbols));
assert!(matches!(env.get(symbols), Some(&Value::String { val, .. }) if val == "symbols"));
assert!(env.contains_key("PWD"));
assert_eq!(env.len(), 4);
}
}

296
crates/nu-cli/tests/commands/history_import.rs Normal file
View File

@ -0,0 +1,296 @@
use nu_protocol::HistoryFileFormat;
use nu_test_support::{nu, Outcome};
use reedline::{
FileBackedHistory, History, HistoryItem, HistoryItemId, ReedlineError, SearchQuery,
SqliteBackedHistory,
};
use rstest::rstest;
use tempfile::TempDir;
struct Test {
cfg_dir: TempDir,
}
impl Test {
fn new(history_format: &'static str) -> Self {
let cfg_dir = tempfile::Builder::new()
.prefix("history_import_test")
.tempdir()
.unwrap();
// Assigning to $env.config.history.file_format seems to work only in startup
// configuration.
std::fs::write(
cfg_dir.path().join("env.nu"),
format!("$env.config.history.file_format = {history_format:?}"),
)
.unwrap();
Self { cfg_dir }
}
fn nu(&self, cmd: impl AsRef<str>) -> Outcome {
let env = [(
"XDG_CONFIG_HOME".to_string(),
self.cfg_dir.path().to_str().unwrap().to_string(),
)];
let env_config = self.cfg_dir.path().join("env.nu");
nu!(envs: env, env_config: env_config, cmd.as_ref())
}
fn open_plaintext(&self) -> Result<FileBackedHistory, ReedlineError> {
FileBackedHistory::with_file(
100,
self.cfg_dir
.path()
.join("nushell")
.join(HistoryFileFormat::Plaintext.default_file_name()),
)
}
fn open_sqlite(&self) -> Result<SqliteBackedHistory, ReedlineError> {
SqliteBackedHistory::with_file(
self.cfg_dir
.path()
.join("nushell")
.join(HistoryFileFormat::Sqlite.default_file_name()),
None,
None,
)
}
fn open_backend(&self, format: HistoryFileFormat) -> Result<Box<dyn History>, ReedlineError> {
fn boxed(be: impl History + 'static) -> Box<dyn History> {
Box::new(be)
}
use HistoryFileFormat::*;
match format {
Plaintext => self.open_plaintext().map(boxed),
Sqlite => self.open_sqlite().map(boxed),
}
}
}
enum HistorySource {
Vec(Vec<HistoryItem>),
Command(&'static str),
}
struct TestCase {
dst_format: HistoryFileFormat,
dst_history: Vec<HistoryItem>,
src_history: HistorySource,
want_history: Vec<HistoryItem>,
}
const EMPTY_TEST_CASE: TestCase = TestCase {
dst_format: HistoryFileFormat::Plaintext,
dst_history: Vec::new(),
src_history: HistorySource::Vec(Vec::new()),
want_history: Vec::new(),
};
impl TestCase {
fn run(self) {
use HistoryFileFormat::*;
let test = Test::new(match self.dst_format {
Plaintext => "plaintext",
Sqlite => "sqlite",
});
save_all(
&mut *test.open_backend(self.dst_format).unwrap(),
self.dst_history,
)
.unwrap();
let outcome = match self.src_history {
HistorySource::Vec(src_history) => {
let src_format = match self.dst_format {
Plaintext => Sqlite,
Sqlite => Plaintext,
};
save_all(&mut *test.open_backend(src_format).unwrap(), src_history).unwrap();
test.nu("history import")
}
HistorySource::Command(cmd) => {
let mut cmd = cmd.to_string();
cmd.push_str(" | history import");
test.nu(cmd)
}
};
assert!(outcome.status.success());
let got = query_all(&*test.open_backend(self.dst_format).unwrap()).unwrap();
// Compare just the commands first, for readability.
fn commands_only(items: &[HistoryItem]) -> Vec<&str> {
items
.iter()
.map(|item| item.command_line.as_str())
.collect()
}
assert_eq!(commands_only(&got), commands_only(&self.want_history));
// If commands match, compare full items.
assert_eq!(got, self.want_history);
}
}
fn query_all(history: &dyn History) -> Result<Vec<HistoryItem>, ReedlineError> {
history.search(SearchQuery::everything(
reedline::SearchDirection::Forward,
None,
))
}
fn save_all(history: &mut dyn History, items: Vec<HistoryItem>) -> Result<(), ReedlineError> {
for item in items {
history.save(item)?;
}
Ok(())
}
const EMPTY_ITEM: HistoryItem = HistoryItem {
command_line: String::new(),
id: None,
start_timestamp: None,
session_id: None,
hostname: None,
cwd: None,
duration: None,
exit_status: None,
more_info: None,
};
#[test]
fn history_import_pipe_string() {
TestCase {
dst_format: HistoryFileFormat::Plaintext,
src_history: HistorySource::Command("echo bar"),
want_history: vec![HistoryItem {
id: Some(HistoryItemId::new(0)),
command_line: "bar".to_string(),
..EMPTY_ITEM
}],
..EMPTY_TEST_CASE
}
.run();
}
#[test]
fn history_import_pipe_record() {
TestCase {
dst_format: HistoryFileFormat::Sqlite,
src_history: HistorySource::Command("[[cwd command]; [/tmp some_command]]"),
want_history: vec![HistoryItem {
id: Some(HistoryItemId::new(1)),
command_line: "some_command".to_string(),
cwd: Some("/tmp".to_string()),
..EMPTY_ITEM
}],
..EMPTY_TEST_CASE
}
.run();
}
#[test]
fn to_empty_plaintext() {
TestCase {
dst_format: HistoryFileFormat::Plaintext,
src_history: HistorySource::Vec(vec![
HistoryItem {
command_line: "foo".to_string(),
..EMPTY_ITEM
},
HistoryItem {
command_line: "bar".to_string(),
..EMPTY_ITEM
},
]),
want_history: vec![
HistoryItem {
id: Some(HistoryItemId::new(0)),
command_line: "foo".to_string(),
..EMPTY_ITEM
},
HistoryItem {
id: Some(HistoryItemId::new(1)),
command_line: "bar".to_string(),
..EMPTY_ITEM
},
],
..EMPTY_TEST_CASE
}
.run()
}
#[test]
fn to_empty_sqlite() {
TestCase {
dst_format: HistoryFileFormat::Sqlite,
src_history: HistorySource::Vec(vec![
HistoryItem {
command_line: "foo".to_string(),
..EMPTY_ITEM
},
HistoryItem {
command_line: "bar".to_string(),
..EMPTY_ITEM
},
]),
want_history: vec![
HistoryItem {
id: Some(HistoryItemId::new(1)),
command_line: "foo".to_string(),
..EMPTY_ITEM
},
HistoryItem {
id: Some(HistoryItemId::new(2)),
command_line: "bar".to_string(),
..EMPTY_ITEM
},
],
..EMPTY_TEST_CASE
}
.run()
}
#[rstest]
#[case::plaintext(HistoryFileFormat::Plaintext)]
#[case::sqlite(HistoryFileFormat::Sqlite)]
fn to_existing(#[case] dst_format: HistoryFileFormat) {
TestCase {
dst_format,
dst_history: vec![
HistoryItem {
id: Some(HistoryItemId::new(0)),
command_line: "original-1".to_string(),
..EMPTY_ITEM
},
HistoryItem {
id: Some(HistoryItemId::new(1)),
command_line: "original-2".to_string(),
..EMPTY_ITEM
},
],
src_history: HistorySource::Vec(vec![HistoryItem {
id: Some(HistoryItemId::new(1)),
command_line: "new".to_string(),
..EMPTY_ITEM
}]),
want_history: vec![
HistoryItem {
id: Some(HistoryItemId::new(0)),
command_line: "original-1".to_string(),
..EMPTY_ITEM
},
HistoryItem {
id: Some(HistoryItemId::new(1)),
command_line: "original-2".to_string(),
..EMPTY_ITEM
},
HistoryItem {
id: Some(HistoryItemId::new(2)),
command_line: "new".to_string(),
..EMPTY_ITEM
},
],
}
.run()
}

7
crates/nu-cli/tests/commands/keybindings_list.rs Normal file
View File

@ -0,0 +1,7 @@
use nu_test_support::nu;
#[test]
fn not_empty() {
let result = nu!("keybindings list | is-not-empty");
assert_eq!(result.out, "true");
}

2
crates/nu-cli/tests/commands/mod.rs
View File

@ -1 +1,3 @@
mod history_import;
mod keybindings_list;
mod nu_highlight;

1273
crates/nu-cli/tests/completions/mod.rs
View File

File diff suppressed because it is too large Load Diff

95
crates/nu-cli/tests/completions/support/completions_helpers.rs
View File

@ -1,5 +1,6 @@
use nu_engine::eval_block;
use nu_parser::parse;
use nu_path::{AbsolutePathBuf, PathBuf};
use nu_protocol::{
debugger::WithoutDebug,
engine::{EngineState, Stack, StateWorkingSet},
@ -7,14 +8,14 @@ use nu_protocol::{
};
use nu_test_support::fs;
use reedline::Suggestion;
use std::path::{PathBuf, MAIN_SEPARATOR};
use std::path::MAIN_SEPARATOR;
fn create_default_context() -> EngineState {
nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
}
// creates a new engine with the current path into the completions fixtures folder
pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
pub fn new_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
// Target folder inside assets
let dir = fs::fixtures().join("completions");
let dir_str = dir
@ -62,13 +63,71 @@ pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
);
// Merge environment into the permanent state
let merge_result = engine_state.merge_env(&mut stack, &dir);
let merge_result = engine_state.merge_env(&mut stack);
assert!(merge_result.is_ok());
(dir, dir_str, engine_state, stack)
}
pub fn new_quote_engine() -> (PathBuf, String, EngineState, Stack) {
// creates a new engine with the current path into the completions fixtures folder
pub fn new_dotnu_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
// Target folder inside assets
let dir = fs::fixtures().join("dotnu_completions");
let dir_str = dir
.clone()
.into_os_string()
.into_string()
.unwrap_or_default();
let dir_span = nu_protocol::Span::new(0, dir_str.len());
// Create a new engine with default context
let mut engine_state = create_default_context();
// Add $nu
engine_state.generate_nu_constant();
// const $NU_LIB_DIRS
let mut working_set = StateWorkingSet::new(&engine_state);
let var_id = working_set.add_variable(
b"$NU_LIB_DIRS".into(),
Span::unknown(),
nu_protocol::Type::List(Box::new(nu_protocol::Type::String)),
false,
);
working_set.set_variable_const_val(
var_id,
Value::test_list(vec![
Value::string(file(dir.join("lib-dir1")), dir_span),
Value::string(file(dir.join("lib-dir3")), dir_span),
]),
);
let _ = engine_state.merge_delta(working_set.render());
// New stack
let mut stack = Stack::new();
// Add pwd as env var
stack.add_env_var("PWD".to_string(), Value::string(dir_str.clone(), dir_span));
stack.add_env_var(
"TEST".to_string(),
Value::string("NUSHELL".to_string(), dir_span),
);
stack.add_env_var(
"NU_LIB_DIRS".into(),
Value::test_list(vec![
Value::string(file(dir.join("lib-dir2")), dir_span),
Value::string(file(dir.join("lib-dir3")), dir_span),
]),
);
// Merge environment into the permanent state
let merge_result = engine_state.merge_env(&mut stack);
assert!(merge_result.is_ok());
(dir, dir_str, engine_state, stack)
}
pub fn new_quote_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
// Target folder inside assets
let dir = fs::fixtures().join("quoted_completions");
let dir_str = dir
@ -97,13 +156,13 @@ pub fn new_quote_engine() -> (PathBuf, String, EngineState, Stack) {
);
// Merge environment into the permanent state
let merge_result = engine_state.merge_env(&mut stack, &dir);
let merge_result = engine_state.merge_env(&mut stack);
assert!(merge_result.is_ok());
(dir, dir_str, engine_state, stack)
}
pub fn new_partial_engine() -> (PathBuf, String, EngineState, Stack) {
pub fn new_partial_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
// Target folder inside assets
let dir = fs::fixtures().join("partial_completions");
let dir_str = dir
@ -132,14 +191,14 @@ pub fn new_partial_engine() -> (PathBuf, String, EngineState, Stack) {
);
// Merge environment into the permanent state
let merge_result = engine_state.merge_env(&mut stack, &dir);
let merge_result = engine_state.merge_env(&mut stack);
assert!(merge_result.is_ok());
(dir, dir_str, engine_state, stack)
}
// match a list of suggestions with the expected values
pub fn match_suggestions(expected: Vec<String>, suggestions: Vec<Suggestion>) {
pub fn match_suggestions(expected: &Vec<String>, suggestions: &Vec<Suggestion>) {
let expected_len = expected.len();
let suggestions_len = suggestions.len();
if expected_len != suggestions_len {
@ -149,22 +208,25 @@ pub fn match_suggestions(expected: Vec<String>, suggestions: Vec<Suggestion>) {
Expected: {expected:#?}\n"
)
}
expected.iter().zip(suggestions).for_each(|it| {
assert_eq!(it.0, &it.1.value);
});
let suggestoins_str = suggestions
.iter()
.map(|it| it.value.clone())
.collect::<Vec<_>>();
assert_eq!(expected, &suggestoins_str);
}
// append the separator to the converted path
pub fn folder(path: PathBuf) -> String {
pub fn folder(path: impl Into<PathBuf>) -> String {
let mut converted_path = file(path);
converted_path.push(MAIN_SEPARATOR);
converted_path
}
// convert a given path to string
pub fn file(path: PathBuf) -> String {
path.into_os_string().into_string().unwrap_or_default()
pub fn file(path: impl Into<PathBuf>) -> String {
path.into().into_os_string().into_string().unwrap()
}
// merge_input executes the given input into the engine
@ -173,7 +235,6 @@ pub fn merge_input(
input: &[u8],
engine_state: &mut EngineState,
stack: &mut Stack,
dir: PathBuf,
) -> Result<(), ShellError> {
let (block, delta) = {
let mut working_set = StateWorkingSet::new(engine_state);
@ -196,5 +257,5 @@ pub fn merge_input(
.is_ok());
// Merge environment into the permanent state
engine_state.merge_env(stack, &dir)
engine_state.merge_env(stack)
}

13
crates/nu-cmd-base/Cargo.toml
View File

@ -5,15 +5,18 @@ edition = "2021"
license = "MIT"
name = "nu-cmd-base"
repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-base"
version = "0.93.1"
version = "0.102.1"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lints]
workspace = true
[dependencies]
nu-engine = { path = "../nu-engine", version = "0.93.1" }
nu-parser = { path = "../nu-parser", version = "0.93.1" }
nu-path = { path = "../nu-path", version = "0.93.1" }
nu-protocol = { path = "../nu-protocol", version = "0.93.1" }
nu-engine = { path = "../nu-engine", version = "0.102.1", default-features = false }
nu-parser = { path = "../nu-parser", version = "0.102.1" }
nu-path = { path = "../nu-path", version = "0.102.1" }
nu-protocol = { path = "../nu-protocol", version = "0.102.1", default-features = false }
indexmap = { workspace = true }
miette = { workspace = true }

5
crates/nu-cmd-base/README.md Normal file
View File

@ -0,0 +1,5 @@
Utilities used by the different `nu-command`/`nu-cmd-*` crates, should not contain any full `Command` implementations.
## Internal Nushell crate
This crate implements components of Nushell and is not designed to support plugin authors or other users directly.

146
crates/nu-cmd-base/src/hook.rs
View File

@ -1,61 +1,61 @@
use crate::util::get_guaranteed_cwd;
use miette::Result;
use nu_engine::{eval_block, eval_block_with_early_return};
use nu_parser::parse;
use nu_protocol::{
cli_error::{report_error, report_error_new},
cli_error::{report_parse_error, report_shell_error},
debugger::WithoutDebug,
engine::{Closure, EngineState, Stack, StateWorkingSet},
PipelineData, PositionalArg, ShellError, Span, Type, Value, VarId,
};
use std::sync::Arc;
use std::{collections::HashMap, sync::Arc};
pub fn eval_env_change_hook(
env_change_hook: Option<Value>,
env_change_hook: &HashMap<String, Vec<Value>>,
engine_state: &mut EngineState,
stack: &mut Stack,
) -> Result<(), ShellError> {
if let Some(hook) = env_change_hook {
match hook {
Value::Record { val, .. } => {
for (env_name, hook_value) in &val {
let before = engine_state
.previous_env_vars
.get(env_name)
.cloned()
.unwrap_or_default();
for (env, hooks) in env_change_hook {
let before = engine_state.previous_env_vars.get(env);
let after = stack.get_env_var(engine_state, env);
if before != after {
let before = before.cloned().unwrap_or_default();
let after = after.cloned().unwrap_or_default();
let after = stack
.get_env_var(engine_state, env_name)
.unwrap_or_default();
eval_hooks(
engine_state,
stack,
vec![("$before".into(), before), ("$after".into(), after.clone())],
hooks,
"env_change",
)?;
if before != after {
eval_hook(
engine_state,
stack,
None,
vec![("$before".into(), before), ("$after".into(), after.clone())],
hook_value,
"env_change",
)?;
Arc::make_mut(&mut engine_state.previous_env_vars)
.insert(env_name.to_string(), after);
}
}
}
x => {
return Err(ShellError::TypeMismatch {
err_message: "record for the 'env_change' hook".to_string(),
span: x.span(),
});
}
Arc::make_mut(&mut engine_state.previous_env_vars).insert(env.clone(), after);
}
}
Ok(())
}
pub fn eval_hooks(
engine_state: &mut EngineState,
stack: &mut Stack,
arguments: Vec<(String, Value)>,
hooks: &[Value],
hook_name: &str,
) -> Result<(), ShellError> {
for hook in hooks {
eval_hook(
engine_state,
stack,
None,
arguments.clone(),
hook,
&format!("{hook_name} list, recursive"),
)?;
}
Ok(())
}
pub fn eval_hook(
engine_state: &mut EngineState,
stack: &mut Stack,
@ -91,12 +91,13 @@ pub fn eval_hook(
false,
);
if let Some(err) = working_set.parse_errors.first() {
report_error(&working_set, err);
return Err(ShellError::UnsupportedConfigValue {
expected: "valid source code".into(),
value: "source code with syntax errors".into(),
span,
report_parse_error(&working_set, err);
return Err(ShellError::GenericError {
error: format!("Failed to run {hook_name} hook"),
msg: "source code has errors".into(),
span: Some(span),
help: None,
inner: Vec::new(),
});
}
@ -123,7 +124,7 @@ pub fn eval_hook(
output = pipeline_data;
}
Err(err) => {
report_error_new(engine_state, &err);
report_shell_error(engine_state, &err);
}
}
@ -132,16 +133,7 @@ pub fn eval_hook(
}
}
Value::List { vals, .. } => {
for val in vals {
eval_hook(
engine_state,
stack,
None,
arguments.clone(),
val,
&format!("{hook_name} list, recursive"),
)?;
}
eval_hooks(engine_state, stack, arguments, vals, hook_name)?;
}
Value::Record { val, .. } => {
// Hooks can optionally be a record in this form:
@ -167,10 +159,10 @@ pub fn eval_hook(
{
val
} else {
return Err(ShellError::UnsupportedConfigValue {
expected: "boolean output".to_string(),
value: "other PipelineData variant".to_string(),
span: other_span,
return Err(ShellError::RuntimeTypeMismatch {
expected: Type::Bool,
actual: pipeline_data.get_type(),
span: pipeline_data.span().unwrap_or(other_span),
});
}
}
@ -179,9 +171,9 @@ pub fn eval_hook(
}
}
} else {
return Err(ShellError::UnsupportedConfigValue {
expected: "block".to_string(),
value: format!("{}", condition.get_type()),
return Err(ShellError::RuntimeTypeMismatch {
expected: Type::Closure,
actual: condition.get_type(),
span: other_span,
});
}
@ -194,7 +186,7 @@ pub fn eval_hook(
let Some(follow) = val.get("code") else {
return Err(ShellError::CantFindColumn {
col_name: "code".into(),
span,
span: Some(span),
src_span: span,
});
};
@ -223,12 +215,13 @@ pub fn eval_hook(
false,
);
if let Some(err) = working_set.parse_errors.first() {
report_error(&working_set, err);
return Err(ShellError::UnsupportedConfigValue {
expected: "valid source code".into(),
value: "source code with syntax errors".into(),
span: source_span,
report_parse_error(&working_set, err);
return Err(ShellError::GenericError {
error: format!("Failed to run {hook_name} hook"),
msg: "source code has errors".into(),
span: Some(span),
help: None,
inner: Vec::new(),
});
}
@ -251,7 +244,7 @@ pub fn eval_hook(
output = pipeline_data;
}
Err(err) => {
report_error_new(engine_state, &err);
report_shell_error(engine_state, &err);
}
}
@ -263,9 +256,9 @@ pub fn eval_hook(
run_hook(engine_state, stack, val, input, arguments, source_span)?;
}
other => {
return Err(ShellError::UnsupportedConfigValue {
expected: "block or string".to_string(),
value: format!("{}", other.get_type()),
return Err(ShellError::RuntimeTypeMismatch {
expected: Type::custom("string or closure"),
actual: other.get_type(),
span: source_span,
});
}
@ -276,16 +269,15 @@ pub fn eval_hook(
output = run_hook(engine_state, stack, val, input, arguments, span)?;
}
other => {
return Err(ShellError::UnsupportedConfigValue {
expected: "string, block, record, or list of commands".into(),
value: format!("{}", other.get_type()),
return Err(ShellError::RuntimeTypeMismatch {
expected: Type::custom("string, closure, record, or list"),
actual: other.get_type(),
span: other.span(),
});
}
}
let cwd = get_guaranteed_cwd(engine_state, stack);
engine_state.merge_env(stack, cwd)?;
engine_state.merge_env(stack)?;
Ok(output)
}

10
crates/nu-cmd-base/src/input_handler.rs
View File

@ -1,5 +1,5 @@
use nu_protocol::{ast::CellPath, PipelineData, ShellError, Span, Value};
use std::sync::{atomic::AtomicBool, Arc};
use nu_protocol::{ast::CellPath, PipelineData, ShellError, Signals, Span, Value};
use std::sync::Arc;
pub trait CmdArgument {
fn take_cell_paths(&mut self) -> Option<Vec<CellPath>>;
@ -40,7 +40,7 @@ pub fn operate<C, A>(
mut arg: A,
input: PipelineData,
span: Span,
ctrlc: Option<Arc<AtomicBool>>,
signals: &Signals,
) -> Result<PipelineData, ShellError>
where
A: CmdArgument + Send + Sync + 'static,
@ -55,7 +55,7 @@ where
_ => cmd(&v, &arg, span),
}
},
ctrlc,
signals,
),
Some(column_paths) => {
let arg = Arc::new(arg);
@ -79,7 +79,7 @@ where
}
v
},
ctrlc,
signals,
)
}
}

1
crates/nu-cmd-base/src/lib.rs
View File

@ -1,3 +1,4 @@
#![doc = include_str!("../README.md")]
pub mod formats;
pub mod hook;
pub mod input_handler;

21
crates/nu-cmd-base/src/util.rs
View File

@ -2,24 +2,11 @@ use nu_protocol::{
engine::{EngineState, Stack},
Range, ShellError, Span, Value,
};
use std::{ops::Bound, path::PathBuf};
pub fn get_init_cwd() -> PathBuf {
std::env::current_dir().unwrap_or_else(|_| {
std::env::var("PWD")
.map(Into::into)
.unwrap_or_else(|_| nu_path::home_dir().unwrap_or_default())
})
}
pub fn get_guaranteed_cwd(engine_state: &EngineState, stack: &Stack) -> PathBuf {
engine_state
.cwd(Some(stack))
.unwrap_or(crate::util::get_init_cwd())
}
use std::ops::Bound;
type MakeRangeError = fn(&str, Span) -> ShellError;
/// Returns a inclusive pair of boundary in given `range`.
pub fn process_range(range: &Range) -> Result<(isize, isize), MakeRangeError> {
match range {
Range::IntRange(range) => {
@ -91,10 +78,10 @@ pub fn get_editor(
get_editor_commandline(&config.buffer_editor, "$env.config.buffer_editor")
{
Ok(buff_editor)
} else if let Some(value) = env_vars.get("EDITOR") {
get_editor_commandline(value, "$env.EDITOR")
} else if let Some(value) = env_vars.get("VISUAL") {
get_editor_commandline(value, "$env.VISUAL")
} else if let Some(value) = env_vars.get("EDITOR") {
get_editor_commandline(value, "$env.EDITOR")
} else {
Err(ShellError::GenericError {
error: "No editor configured".into(),

75
crates/nu-cmd-dataframe/Cargo.toml
View File

@ -1,75 +0,0 @@
[package]
authors = ["The Nushell Project Developers"]
description = "Nushell's dataframe commands based on polars."
edition = "2021"
license = "MIT"
name = "nu-cmd-dataframe"
repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-dataframe"
version = "0.93.1"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
bench = false
[dependencies]
nu-engine = { path = "../nu-engine", version = "0.93.1" }
nu-parser = { path = "../nu-parser", version = "0.93.1" }
nu-protocol = { path = "../nu-protocol", version = "0.93.1" }
# Potential dependencies for extras
chrono = { workspace = true, features = ["std", "unstable-locales"], default-features = false }
chrono-tz = { workspace = true }
fancy-regex = { workspace = true }
indexmap = { workspace = true }
num = { version = "0.4", optional = true }
serde = { workspace = true, features = ["derive"] }
# keep sqlparser at 0.39.0 until we can update polars
sqlparser = { version = "0.45", optional = true }
polars-io = { version = "0.39", features = ["avro"], optional = true }
polars-arrow = { version = "0.39", optional = true }
polars-ops = { version = "0.39", optional = true }
polars-plan = { version = "0.39", features = ["regex"], optional = true }
polars-utils = { version = "0.39", optional = true }
[dependencies.polars]
features = [
"arg_where",
"checked_arithmetic",
"concat_str",
"cross_join",
"csv",
"cum_agg",
"dtype-categorical",
"dtype-datetime",
"dtype-struct",
"dtype-i8",
"dtype-i16",
"dtype-u8",
"dtype-u16",
"dynamic_group_by",
"ipc",
"is_in",
"json",
"lazy",
"object",
"parquet",
"random",
"rolling_window",
"rows",
"serde",
"serde-lazy",
"strings",
"temporal",
"to_dummies",
]
default-features = false
optional = true
version = "0.39"
[features]
dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"]
default = []
[dev-dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" }

12
crates/nu-cmd-dataframe/src/dataframe/README.md
View File

@ -1,12 +0,0 @@
# Dataframe
This dataframe directory holds all of the definitions of the dataframe data structures and commands.
There are three sections of commands:
* [eager](./eager)
* [series](./series)
* [values](./values)
For more details see the
[Nushell book section on dataframes](https://www.nushell.sh/book/dataframes.html)

134
crates/nu-cmd-dataframe/src/dataframe/eager/append.rs
View File

@ -1,134 +0,0 @@
use crate::dataframe::values::{Axis, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct AppendDF;
impl Command for AppendDF {
fn name(&self) -> &str {
"dfr append"
}
fn usage(&self) -> &str {
"Appends a new dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("other", SyntaxShape::Any, "dataframe to be appended")
.switch("col", "appends in col orientation", Some('c'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Appends a dataframe as new columns",
example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
$a | dfr append $a"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"a_x".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b_x".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Appends a dataframe merging at the end of columns",
example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
$a | dfr append $a --col"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![
Value::test_int(1),
Value::test_int(3),
Value::test_int(1),
Value::test_int(3),
],
),
Column::new(
"b".to_string(),
vec![
Value::test_int(2),
Value::test_int(4),
Value::test_int(2),
Value::test_int(4),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let other: Value = call.req(engine_state, stack, 0)?;
let axis = if call.has_flag(engine_state, stack, "col")? {
Axis::Column
} else {
Axis::Row
};
let df_other = NuDataFrame::try_from_value(other)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.append_df(&df_other, axis, call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(AppendDF {})])
}
}

195
crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs
View File

@ -1,195 +0,0 @@
use crate::dataframe::values::{str_to_dtype, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
use polars::prelude::*;
#[derive(Clone)]
pub struct CastDF;
impl Command for CastDF {
fn name(&self) -> &str {
"dfr cast"
}
fn usage(&self) -> &str {
"Cast a column to a different dtype."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.required(
"dtype",
SyntaxShape::String,
"The dtype to cast the column to",
)
.optional(
"column",
SyntaxShape::String,
"The column to cast. Required when used with a dataframe.",
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Cast a column in a dataframe to a different dtype",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr cast u8 a | dfr schema",
result: Some(Value::record(
record! {
"a" => Value::string("u8", Span::test_data()),
"b" => Value::string("i64", Span::test_data()),
},
Span::test_data(),
)),
},
Example {
description: "Cast a column in a lazy dataframe to a different dtype",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-lazy | dfr cast u8 a | dfr schema",
result: Some(Value::record(
record! {
"a" => Value::string("u8", Span::test_data()),
"b" => Value::string("i64", Span::test_data()),
},
Span::test_data(),
)),
},
Example {
description: "Cast a column in a expression to a different dtype",
example: r#"[[a b]; [1 2] [1 4]] | dfr into-df | dfr group-by a | dfr agg [ (dfr col b | dfr cast u8 | dfr min | dfr as "b_min") ] | dfr schema"#,
result: None
}
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuLazyFrame::can_downcast(&value) {
let (dtype, column_nm) = df_args(engine_state, stack, call)?;
let df = NuLazyFrame::try_from_value(value)?;
command_lazy(call, column_nm, dtype, df)
} else if NuDataFrame::can_downcast(&value) {
let (dtype, column_nm) = df_args(engine_state, stack, call)?;
let df = NuDataFrame::try_from_value(value)?;
command_eager(call, column_nm, dtype, df)
} else {
let dtype: String = call.req(engine_state, stack, 0)?;
let dtype = str_to_dtype(&dtype, call.head)?;
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().cast(dtype).into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn df_args(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<(DataType, String), ShellError> {
let dtype = dtype_arg(engine_state, stack, call)?;
let column_nm: String =
call.opt(engine_state, stack, 1)?
.ok_or(ShellError::MissingParameter {
param_name: "column_name".into(),
span: call.head,
})?;
Ok((dtype, column_nm))
}
fn dtype_arg(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<DataType, ShellError> {
let dtype: String = call.req(engine_state, stack, 0)?;
str_to_dtype(&dtype, call.head)
}
fn command_lazy(
call: &Call,
column_nm: String,
dtype: DataType,
lazy: NuLazyFrame,
) -> Result<PipelineData, ShellError> {
let column = col(&column_nm).cast(dtype);
let lazy = lazy.into_polars().with_columns(&[column]);
let lazy = NuLazyFrame::new(false, lazy);
Ok(PipelineData::Value(
NuLazyFrame::into_value(lazy, call.head)?,
None,
))
}
fn command_eager(
call: &Call,
column_nm: String,
dtype: DataType,
nu_df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let mut df = nu_df.df;
let column = df
.column(&column_nm)
.map_err(|e| ShellError::GenericError {
error: format!("{e}"),
msg: "".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let casted = column.cast(&dtype).map_err(|e| ShellError::GenericError {
error: format!("{e}"),
msg: "".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let _ = df
.with_column(casted)
.map_err(|e| ShellError::GenericError {
error: format!("{e}"),
msg: "".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let df = NuDataFrame::new(false, df);
Ok(PipelineData::Value(df.into_value(call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(CastDF {})])
}
}

73
crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs
View File

@ -1,73 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ColumnsDF;
impl Command for ColumnsDF {
fn name(&self) -> &str {
"dfr columns"
}
fn usage(&self) -> &str {
"Show dataframe columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Dataframe columns",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr columns",
result: Some(Value::list(
vec![Value::test_string("a"), Value::test_string("b")],
Span::test_data(),
)),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let names: Vec<Value> = df
.as_ref()
.get_column_names()
.iter()
.map(|v| Value::string(*v, call.head))
.collect();
let names = Value::list(names, call.head);
Ok(PipelineData::Value(names, None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ColumnsDF {})])
}
}

115
crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs
View File

@ -1,115 +0,0 @@
use crate::dataframe::values::{utils::convert_columns, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct DropDF;
impl Command for DropDF {
fn name(&self) -> &str {
"dfr drop"
}
fn usage(&self) -> &str {
"Creates a new dataframe by dropping the selected columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest("rest", SyntaxShape::Any, "column names to be dropped")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "drop column a",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr drop a",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
let (col_string, col_span) = convert_columns(columns, call.head)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let new_df = col_string
.first()
.ok_or_else(|| ShellError::GenericError {
error: "Empty names list".into(),
msg: "No column names were found".into(),
span: Some(col_span),
help: None,
inner: vec![],
})
.and_then(|col| {
df.as_ref()
.drop(&col.item)
.map_err(|e| ShellError::GenericError {
error: "Error dropping column".into(),
msg: e.to_string(),
span: Some(col.span),
help: None,
inner: vec![],
})
})?;
// If there are more columns in the drop selection list, these
// are added from the resulting dataframe
col_string
.iter()
.skip(1)
.try_fold(new_df, |new_df, col| {
new_df
.drop(&col.item)
.map_err(|e| ShellError::GenericError {
error: "Error dropping column".into(),
msg: e.to_string(),
span: Some(col.span),
help: None,
inner: vec![],
})
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DropDF {})])
}
}

119
crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs
View File

@ -1,119 +0,0 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::UniqueKeepStrategy;
#[derive(Clone)]
pub struct DropDuplicates;
impl Command for DropDuplicates {
fn name(&self) -> &str {
"dfr drop-duplicates"
}
fn usage(&self) -> &str {
"Drops duplicate values in dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional(
"subset",
SyntaxShape::Table(vec![]),
"subset of columns to drop duplicates",
)
.switch("maintain", "maintain order", Some('m'))
.switch(
"last",
"keeps last duplicate value (by default keeps first)",
Some('l'),
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "drop duplicates",
example: "[[a b]; [1 2] [3 4] [1 2]] | dfr into-df | dfr drop-duplicates",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(3), Value::test_int(1)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(4), Value::test_int(2)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let columns: Option<Vec<Value>> = call.opt(engine_state, stack, 0)?;
let (subset, col_span) = match columns {
Some(cols) => {
let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
(Some(agg_string), col_span)
}
None => (None, call.head),
};
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let subset_slice = subset.as_ref().map(|cols| &cols[..]);
let keep_strategy = if call.has_flag(engine_state, stack, "last")? {
UniqueKeepStrategy::Last
} else {
UniqueKeepStrategy::First
};
df.as_ref()
.unique(subset_slice, keep_strategy, None)
.map_err(|e| ShellError::GenericError {
error: "Error dropping duplicates".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DropDuplicates {})])
}
}

137
crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs
View File

@ -1,137 +0,0 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct DropNulls;
impl Command for DropNulls {
fn name(&self) -> &str {
"dfr drop-nulls"
}
fn usage(&self) -> &str {
"Drops null values in dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional(
"subset",
SyntaxShape::Table(vec![]),
"subset of columns to drop nulls",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "drop null values in dataframe",
example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | dfr into-df);
let res = ($df.b / $df.b);
let a = ($df | dfr with-column $res --name res);
$a | dfr drop-nulls"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(1)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(2)],
),
Column::new(
"res".to_string(),
vec![Value::test_int(1), Value::test_int(1)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "drop null values in dataframe",
example: r#"let s = ([1 2 0 0 3 4] | dfr into-df);
($s / $s) | dfr drop-nulls"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"div_0_0".to_string(),
vec![
Value::test_int(1),
Value::test_int(1),
Value::test_int(1),
Value::test_int(1),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let columns: Option<Vec<Value>> = call.opt(engine_state, stack, 0)?;
let (subset, col_span) = match columns {
Some(cols) => {
let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
(Some(agg_string), col_span)
}
None => (None, call.head),
};
let subset_slice = subset.as_ref().map(|cols| &cols[..]);
df.as_ref()
.drop_nulls(subset_slice)
.map_err(|e| ShellError::GenericError {
error: "Error dropping nulls".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::super::WithColumn;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DropNulls {}), Box::new(WithColumn {})])
}
}

104
crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs
View File

@ -1,104 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct DataTypes;
impl Command for DataTypes {
fn name(&self) -> &str {
"dfr dtypes"
}
fn usage(&self) -> &str {
"Show dataframe data types."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Dataframe dtypes",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dtypes",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"column".to_string(),
vec![Value::test_string("a"), Value::test_string("b")],
),
Column::new(
"dtype".to_string(),
vec![Value::test_string("i64"), Value::test_string("i64")],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut dtypes: Vec<Value> = Vec::new();
let names: Vec<Value> = df
.as_ref()
.get_column_names()
.iter()
.map(|v| {
let dtype = df
.as_ref()
.column(v)
.expect("using name from list of names from dataframe")
.dtype();
let dtype_str = dtype.to_string();
dtypes.push(Value::string(dtype_str, call.head));
Value::string(*v, call.head)
})
.collect();
let names_col = Column::new("column".to_string(), names);
let dtypes_col = Column::new("dtype".to_string(), dtypes);
NuDataFrame::try_from_columns(vec![names_col, dtypes_col], None)
.map(|df| PipelineData::Value(df.into_value(call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DataTypes {})])
}
}

107
crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs
View File

@ -1,107 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::{prelude::*, series::Series};
#[derive(Clone)]
pub struct Dummies;
impl Command for Dummies {
fn name(&self) -> &str {
"dfr dummies"
}
fn usage(&self) -> &str {
"Creates a new dataframe with dummy variables."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.switch("drop-first", "Drop first row", Some('d'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create new dataframe with dummy variables from a dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dummies",
result: Some(
NuDataFrame::try_from_series(
vec![
Series::new("a_1", &[1_u8, 0]),
Series::new("a_3", &[0_u8, 1]),
Series::new("b_2", &[1_u8, 0]),
Series::new("b_4", &[0_u8, 1]),
],
Span::test_data(),
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Create new dataframe with dummy variables from a series",
example: "[1 2 2 3 3] | dfr into-df | dfr dummies",
result: Some(
NuDataFrame::try_from_series(
vec![
Series::new("0_1", &[1_u8, 0, 0, 0, 0]),
Series::new("0_2", &[0_u8, 1, 1, 0, 0]),
Series::new("0_3", &[0_u8, 0, 0, 1, 1]),
],
Span::test_data(),
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let drop_first: bool = call.has_flag(engine_state, stack, "drop-first")?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.as_ref()
.to_dummies(None, drop_first)
.map_err(|e| ShellError::GenericError {
error: "Error calculating dummies".into(),
msg: e.to_string(),
span: Some(call.head),
help: Some("The only allowed column types for dummies are String or Int".into()),
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(Dummies {})])
}
}

144
crates/nu-cmd-dataframe/src/dataframe/eager/first.rs
View File

@ -1,144 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct FirstDF;
impl Command for FirstDF {
fn name(&self) -> &str {
"dfr first"
}
fn usage(&self) -> &str {
"Show only the first number of rows or create a first expression"
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional(
"rows",
SyntaxShape::Int,
"starting from the front, the number of rows to return",
)
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Return the first row of a dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(1)]),
Column::new("b".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Return the first two rows of a dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first 2",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Creates a first expression from a column",
example: "dfr col a | dfr first",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(value)?;
command(engine_state, stack, call, df)
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().first().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
let rows = rows.unwrap_or(1);
let res = df.as_ref().head(Some(rows));
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples_dataframe() {
let mut engine_state = build_test_engine_state(vec![Box::new(FirstDF {})]);
test_dataframe_example(&mut engine_state, &FirstDF.examples()[0]);
test_dataframe_example(&mut engine_state, &FirstDF.examples()[1]);
}
#[test]
fn test_examples_expression() {
let mut engine_state = build_test_engine_state(vec![
Box::new(FirstDF {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &FirstDF.examples()[2]);
}
}

87
crates/nu-cmd-dataframe/src/dataframe/eager/get.rs
View File

@ -1,87 +0,0 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct GetDF;
impl Command for GetDF {
fn name(&self) -> &str {
"dfr get"
}
fn usage(&self) -> &str {
"Creates dataframe with the selected columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest("rest", SyntaxShape::Any, "column names to sort dataframe")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns the selected column",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr get a",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
let (col_string, col_span) = convert_columns_string(columns, call.head)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.as_ref()
.select(col_string)
.map_err(|e| ShellError::GenericError {
error: "Error selecting columns".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetDF {})])
}
}

118
crates/nu-cmd-dataframe/src/dataframe/eager/last.rs
View File

@ -1,118 +0,0 @@
use crate::dataframe::values::{utils::DEFAULT_ROWS, Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LastDF;
impl Command for LastDF {
fn name(&self) -> &str {
"dfr last"
}
fn usage(&self) -> &str {
"Creates new dataframe with tail rows or creates a last expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional("rows", SyntaxShape::Int, "Number of rows for tail")
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create new dataframe with last rows",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr last 1",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(3)]),
Column::new("b".to_string(), vec![Value::test_int(4)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Creates a last expression from a column",
example: "dfr col a | dfr last",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(value)?;
command(engine_state, stack, call, df)
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().last().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
let rows = rows.unwrap_or(DEFAULT_ROWS);
let res = df.as_ref().tail(Some(rows));
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples_dataframe() {
let mut engine_state = build_test_engine_state(vec![Box::new(LastDF {})]);
test_dataframe_example(&mut engine_state, &LastDF.examples()[0]);
}
#[test]
fn test_examples_expression() {
let mut engine_state = build_test_engine_state(vec![
Box::new(LastDF {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &LastDF.examples()[1]);
}
}

68
crates/nu-cmd-dataframe/src/dataframe/eager/list.rs
View File

@ -1,68 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ListDF;
impl Command for ListDF {
fn name(&self) -> &str {
"dfr ls"
}
fn usage(&self) -> &str {
"Lists stored dataframes."
}
fn signature(&self) -> Signature {
Signature::build(self.name()).category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Creates a new dataframe and shows it in the dataframe list",
example: r#"let test = ([[a b];[1 2] [3 4]] | dfr into-df);
ls"#,
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let mut vals: Vec<(String, Value)> = vec![];
for overlay_frame in engine_state.active_overlays(&[]) {
for var in &overlay_frame.vars {
if let Ok(value) = stack.get_var(*var.1, call.head) {
let name = String::from_utf8_lossy(var.0).to_string();
vals.push((name, value));
}
}
}
let vals = vals
.into_iter()
.filter_map(|(name, value)| {
NuDataFrame::try_from_value(value).ok().map(|df| (name, df))
})
.map(|(name, df)| {
Value::record(
record! {
"name" => Value::string(name, call.head),
"columns" => Value::int(df.as_ref().width() as i64, call.head),
"rows" => Value::int(df.as_ref().height() as i64, call.head),
},
call.head,
)
})
.collect::<Vec<Value>>();
let list = Value::list(vals, call.head);
Ok(list.into_pipeline_data())
}
}

248
crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs
View File

@ -1,248 +0,0 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct MeltDF;
impl Command for MeltDF {
fn name(&self) -> &str {
"dfr melt"
}
fn usage(&self) -> &str {
"Unpivot a DataFrame from wide to long format."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required_named(
"columns",
SyntaxShape::Table(vec![]),
"column names for melting",
Some('c'),
)
.required_named(
"values",
SyntaxShape::Table(vec![]),
"column names used as value columns",
Some('v'),
)
.named(
"variable-name",
SyntaxShape::String,
"optional name for variable column",
Some('r'),
)
.named(
"value-name",
SyntaxShape::String,
"optional name for value column",
Some('l'),
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "melt dataframe",
example:
"[[a b c d]; [x 1 4 a] [y 2 5 b] [z 3 6 c]] | dfr into-df | dfr melt -c [b c] -v [a d]",
result: Some(
NuDataFrame::try_from_columns(vec![
Column::new(
"b".to_string(),
vec![
Value::test_int(1),
Value::test_int(2),
Value::test_int(3),
Value::test_int(1),
Value::test_int(2),
Value::test_int(3),
],
),
Column::new(
"c".to_string(),
vec![
Value::test_int(4),
Value::test_int(5),
Value::test_int(6),
Value::test_int(4),
Value::test_int(5),
Value::test_int(6),
],
),
Column::new(
"variable".to_string(),
vec![
Value::test_string("a"),
Value::test_string("a"),
Value::test_string("a"),
Value::test_string("d"),
Value::test_string("d"),
Value::test_string("d"),
],
),
Column::new(
"value".to_string(),
vec![
Value::test_string("x"),
Value::test_string("y"),
Value::test_string("z"),
Value::test_string("a"),
Value::test_string("b"),
Value::test_string("c"),
],
),
], None)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let id_col: Vec<Value> = call
.get_flag(engine_state, stack, "columns")?
.expect("required value");
let val_col: Vec<Value> = call
.get_flag(engine_state, stack, "values")?
.expect("required value");
let value_name: Option<Spanned<String>> = call.get_flag(engine_state, stack, "value-name")?;
let variable_name: Option<Spanned<String>> =
call.get_flag(engine_state, stack, "variable-name")?;
let (id_col_string, id_col_span) = convert_columns_string(id_col, call.head)?;
let (val_col_string, val_col_span) = convert_columns_string(val_col, call.head)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
check_column_datatypes(df.as_ref(), &id_col_string, id_col_span)?;
check_column_datatypes(df.as_ref(), &val_col_string, val_col_span)?;
let mut res = df
.as_ref()
.melt(&id_col_string, &val_col_string)
.map_err(|e| ShellError::GenericError {
error: "Error calculating melt".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
if let Some(name) = &variable_name {
res.rename("variable", &name.item)
.map_err(|e| ShellError::GenericError {
error: "Error renaming column".into(),
msg: e.to_string(),
span: Some(name.span),
help: None,
inner: vec![],
})?;
}
if let Some(name) = &value_name {
res.rename("value", &name.item)
.map_err(|e| ShellError::GenericError {
error: "Error renaming column".into(),
msg: e.to_string(),
span: Some(name.span),
help: None,
inner: vec![],
})?;
}
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
fn check_column_datatypes<T: AsRef<str>>(
df: &polars::prelude::DataFrame,
cols: &[T],
col_span: Span,
) -> Result<(), ShellError> {
if cols.is_empty() {
return Err(ShellError::GenericError {
error: "Merge error".into(),
msg: "empty column list".into(),
span: Some(col_span),
help: None,
inner: vec![],
});
}
// Checking if they are same type
if cols.len() > 1 {
for w in cols.windows(2) {
let l_series = df
.column(w[0].as_ref())
.map_err(|e| ShellError::GenericError {
error: "Error selecting columns".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})?;
let r_series = df
.column(w[1].as_ref())
.map_err(|e| ShellError::GenericError {
error: "Error selecting columns".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})?;
if l_series.dtype() != r_series.dtype() {
return Err(ShellError::GenericError {
error: "Merge error".into(),
msg: "found different column types in list".into(),
span: Some(col_span),
help: Some(format!(
"datatypes {} and {} are incompatible",
l_series.dtype(),
r_series.dtype()
)),
inner: vec![],
});
}
}
}
Ok(())
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(MeltDF {})])
}
}

114
crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs
View File

@ -1,114 +0,0 @@
mod append;
mod cast;
mod columns;
mod drop;
mod drop_duplicates;
mod drop_nulls;
mod dtypes;
mod dummies;
mod filter_with;
mod first;
mod get;
mod last;
mod list;
mod melt;
mod open;
mod query_df;
mod rename;
mod sample;
mod schema;
mod shape;
mod slice;
mod sql_context;
mod sql_expr;
mod summary;
mod take;
mod to_arrow;
mod to_avro;
mod to_csv;
mod to_df;
mod to_json_lines;
mod to_nu;
mod to_parquet;
mod with_column;
use nu_protocol::engine::StateWorkingSet;
pub use self::open::OpenDataFrame;
pub use append::AppendDF;
pub use cast::CastDF;
pub use columns::ColumnsDF;
pub use drop::DropDF;
pub use drop_duplicates::DropDuplicates;
pub use drop_nulls::DropNulls;
pub use dtypes::DataTypes;
pub use dummies::Dummies;
pub use filter_with::FilterWith;
pub use first::FirstDF;
pub use get::GetDF;
pub use last::LastDF;
pub use list::ListDF;
pub use melt::MeltDF;
pub use query_df::QueryDf;
pub use rename::RenameDF;
pub use sample::SampleDF;
pub use schema::SchemaDF;
pub use shape::ShapeDF;
pub use slice::SliceDF;
pub use sql_context::SQLContext;
pub use summary::Summary;
pub use take::TakeDF;
pub use to_arrow::ToArrow;
pub use to_avro::ToAvro;
pub use to_csv::ToCSV;
pub use to_df::ToDataFrame;
pub use to_json_lines::ToJsonLines;
pub use to_nu::ToNu;
pub use to_parquet::ToParquet;
pub use with_column::WithColumn;
pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
macro_rules! bind_command {
( $command:expr ) => {
working_set.add_decl(Box::new($command));
};
( $( $command:expr ),* ) => {
$( working_set.add_decl(Box::new($command)); )*
};
}
// Dataframe commands
bind_command!(
AppendDF,
CastDF,
ColumnsDF,
DataTypes,
Summary,
DropDF,
DropDuplicates,
DropNulls,
Dummies,
FilterWith,
FirstDF,
GetDF,
LastDF,
ListDF,
MeltDF,
OpenDataFrame,
QueryDf,
RenameDF,
SampleDF,
SchemaDF,
ShapeDF,
SliceDF,
TakeDF,
ToArrow,
ToAvro,
ToCSV,
ToDataFrame,
ToNu,
ToParquet,
ToJsonLines,
WithColumn
);
}

518
crates/nu-cmd-dataframe/src/dataframe/eager/open.rs
View File

@ -1,518 +0,0 @@
use crate::dataframe::values::{NuDataFrame, NuLazyFrame, NuSchema};
use nu_engine::command_prelude::*;
use polars::prelude::{
CsvEncoding, CsvReader, IpcReader, JsonFormat, JsonReader, LazyCsvReader, LazyFileListReader,
LazyFrame, ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader,
};
use polars_io::{avro::AvroReader, HiveOptions};
use std::{fs::File, io::BufReader, path::PathBuf};
#[derive(Clone)]
pub struct OpenDataFrame;
impl Command for OpenDataFrame {
fn name(&self) -> &str {
"dfr open"
}
fn usage(&self) -> &str {
"Opens CSV, JSON, JSON lines, arrow, avro, or parquet file to create dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"file",
SyntaxShape::Filepath,
"file path to load values from",
)
.switch("lazy", "creates a lazy dataframe", Some('l'))
.named(
"type",
SyntaxShape::String,
"File type: csv, tsv, json, parquet, arrow, avro. If omitted, derive from file extension",
Some('t'),
)
.named(
"delimiter",
SyntaxShape::String,
"file delimiter character. CSV file",
Some('d'),
)
.switch(
"no-header",
"Indicates if file doesn't have header. CSV file",
None,
)
.named(
"infer-schema",
SyntaxShape::Number,
"Number of rows to infer the schema of the file. CSV file",
None,
)
.named(
"skip-rows",
SyntaxShape::Number,
"Number of rows to skip from file. CSV file",
None,
)
.named(
"columns",
SyntaxShape::List(Box::new(SyntaxShape::String)),
"Columns to be selected from csv file. CSV and Parquet file",
None,
)
.named(
"schema",
SyntaxShape::Record(vec![]),
r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
Some('s')
)
.input_output_type(Type::Any, Type::Custom("dataframe".into()))
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Takes a file name and creates a dataframe",
example: "dfr open test.csv",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<PipelineData, ShellError> {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let type_option: Option<Spanned<String>> = call.get_flag(engine_state, stack, "type")?;
let type_id = match &type_option {
Some(ref t) => Some((t.item.to_owned(), "Invalid type", t.span)),
None => file.item.extension().map(|e| {
(
e.to_string_lossy().into_owned(),
"Invalid extension",
file.span,
)
}),
};
match type_id {
Some((e, msg, blamed)) => match e.as_str() {
"csv" | "tsv" => from_csv(engine_state, stack, call),
"parquet" | "parq" => from_parquet(engine_state, stack, call),
"ipc" | "arrow" => from_ipc(engine_state, stack, call),
"json" => from_json(engine_state, stack, call),
"jsonl" => from_jsonl(engine_state, stack, call),
"avro" => from_avro(engine_state, stack, call),
_ => Err(ShellError::FileNotFoundCustom {
msg: format!(
"{msg}. Supported values: csv, tsv, parquet, ipc, arrow, json, jsonl, avro"
),
span: blamed,
}),
},
None => Err(ShellError::FileNotFoundCustom {
msg: "File without extension".into(),
span: file.span,
}),
}
.map(|value| PipelineData::Value(value, None))
}
fn from_parquet(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
if call.has_flag(engine_state, stack, "lazy")? {
let file: String = call.req(engine_state, stack, 0)?;
let args = ScanArgsParquet {
n_rows: None,
cache: true,
parallel: ParallelStrategy::Auto,
rechunk: false,
row_index: None,
low_memory: false,
cloud_options: None,
use_statistics: false,
hive_options: HiveOptions::default(),
};
let df: NuLazyFrame = LazyFrame::scan_parquet(file, args)
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
df.into_value(call.head)
} else {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let reader = ParquetReader::new(r);
let reader = match columns {
None => reader,
Some(columns) => reader.with_columns(Some(columns)),
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
}
fn from_avro(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let reader = AvroReader::new(r);
let reader = match columns {
None => reader,
Some(columns) => reader.with_columns(Some(columns)),
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Avro reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
fn from_ipc(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
if call.has_flag(engine_state, stack, "lazy")? {
let file: String = call.req(engine_state, stack, 0)?;
let args = ScanArgsIpc {
n_rows: None,
cache: true,
rechunk: false,
row_index: None,
memory_map: true,
cloud_options: None,
};
let df: NuLazyFrame = LazyFrame::scan_ipc(file, args)
.map_err(|e| ShellError::GenericError {
error: "IPC reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
df.into_value(call.head)
} else {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let reader = IpcReader::new(r);
let reader = match columns {
None => reader,
Some(columns) => reader.with_columns(Some(columns)),
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "IPC reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
}
fn from_json(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
let buf_reader = BufReader::new(file);
let reader = JsonReader::new(buf_reader);
let reader = match maybe_schema {
Some(schema) => reader.with_schema(schema.into()),
None => reader,
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Json reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
fn from_jsonl(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let buf_reader = BufReader::new(file);
let reader = JsonReader::new(buf_reader)
.with_json_format(JsonFormat::JsonLines)
.infer_schema_len(infer_schema);
let reader = match maybe_schema {
Some(schema) => reader.with_schema(schema.into()),
None => reader,
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Json lines reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
fn from_csv(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
let skip_rows: Option<usize> = call.get_flag(engine_state, stack, "skip-rows")?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
if call.has_flag(engine_state, stack, "lazy")? {
let file: String = call.req(engine_state, stack, 0)?;
let csv_reader = LazyCsvReader::new(file);
let csv_reader = match delimiter {
None => csv_reader,
Some(d) => {
if d.item.len() != 1 {
return Err(ShellError::GenericError {
error: "Incorrect delimiter".into(),
msg: "Delimiter has to be one character".into(),
span: Some(d.span),
help: None,
inner: vec![],
});
} else {
let delimiter = match d.item.chars().next() {
Some(d) => d as u8,
None => unreachable!(),
};
csv_reader.with_separator(delimiter)
}
}
};
let csv_reader = csv_reader.has_header(!no_header);
let csv_reader = match maybe_schema {
Some(schema) => csv_reader.with_schema(Some(schema.into())),
None => csv_reader,
};
let csv_reader = match infer_schema {
None => csv_reader,
Some(r) => csv_reader.with_infer_schema_length(Some(r)),
};
let csv_reader = match skip_rows {
None => csv_reader,
Some(r) => csv_reader.with_skip_rows(r),
};
let df: NuLazyFrame = csv_reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
df.into_value(call.head)
} else {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let csv_reader = CsvReader::from_path(&file.item)
.map_err(|e| ShellError::GenericError {
error: "Error creating CSV reader".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?
.with_encoding(CsvEncoding::LossyUtf8);
let csv_reader = match delimiter {
None => csv_reader,
Some(d) => {
if d.item.len() != 1 {
return Err(ShellError::GenericError {
error: "Incorrect delimiter".into(),
msg: "Delimiter has to be one character".into(),
span: Some(d.span),
help: None,
inner: vec![],
});
} else {
let delimiter = match d.item.chars().next() {
Some(d) => d as u8,
None => unreachable!(),
};
csv_reader.with_separator(delimiter)
}
}
};
let csv_reader = csv_reader.has_header(!no_header);
let csv_reader = match maybe_schema {
Some(schema) => csv_reader.with_schema(Some(schema.into())),
None => csv_reader,
};
let csv_reader = match infer_schema {
None => csv_reader,
Some(r) => csv_reader.infer_schema(Some(r)),
};
let csv_reader = match skip_rows {
None => csv_reader,
Some(r) => csv_reader.with_skip_rows(r),
};
let csv_reader = match columns {
None => csv_reader,
Some(columns) => csv_reader.with_columns(Some(columns)),
};
let df: NuDataFrame = csv_reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
}

104
crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs
View File

@ -1,104 +0,0 @@
use crate::dataframe::{
eager::SQLContext,
values::{Column, NuDataFrame, NuLazyFrame},
};
use nu_engine::command_prelude::*;
// attribution:
// sql_context.rs, and sql_expr.rs were copied from polars-sql. thank you.
// maybe we should just use the crate at some point but it's not published yet.
// https://github.com/pola-rs/polars/tree/master/polars-sql
#[derive(Clone)]
pub struct QueryDf;
impl Command for QueryDf {
fn name(&self) -> &str {
"dfr query"
}
fn usage(&self) -> &str {
"Query dataframe using SQL. Note: The dataframe is always named 'df' in your query's from clause."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("sql", SyntaxShape::String, "sql query")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn search_terms(&self) -> Vec<&str> {
vec!["dataframe", "sql", "search"]
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Query dataframe using SQL",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr query 'select a from df'",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let sql_query: String = call.req(engine_state, stack, 0)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut ctx = SQLContext::new();
ctx.register("df", &df.df);
let df_sql = ctx
.execute(&sql_query)
.map_err(|e| ShellError::GenericError {
error: "Dataframe Error".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let lazy = NuLazyFrame::new(false, df_sql);
let eager = lazy.collect(call.head)?;
let value = Value::custom(Box::new(eager), call.head);
Ok(PipelineData::Value(value, None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(QueryDf {})])
}
}

185
crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs
View File

@ -1,185 +0,0 @@
use crate::dataframe::{
utils::extract_strings,
values::{Column, NuDataFrame, NuLazyFrame},
};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct RenameDF;
impl Command for RenameDF {
fn name(&self) -> &str {
"dfr rename"
}
fn usage(&self) -> &str {
"Rename a dataframe column."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"columns",
SyntaxShape::Any,
"Column(s) to be renamed. A string or list of strings",
)
.required(
"new names",
SyntaxShape::Any,
"New names for the selected column(s). A string or list of strings",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe or lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Renames a series",
example: "[5 6 7 8] | dfr into-df | dfr rename '0' new_name",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"new_name".to_string(),
vec![
Value::test_int(5),
Value::test_int(6),
Value::test_int(7),
Value::test_int(8),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Renames a dataframe column",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename a a_new",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a_new".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Renames two dataframe columns",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename [a b] [a_new b_new]",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a_new".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b_new".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head)?;
if NuLazyFrame::can_downcast(&value) {
let df = NuLazyFrame::try_from_value(value)?;
command_lazy(engine_state, stack, call, df)
} else {
let df = NuDataFrame::try_from_value(value)?;
command_eager(engine_state, stack, call, df)
}
}
}
fn command_eager(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
mut df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let columns: Value = call.req(engine_state, stack, 0)?;
let columns = extract_strings(columns)?;
let new_names: Value = call.req(engine_state, stack, 1)?;
let new_names = extract_strings(new_names)?;
for (from, to) in columns.iter().zip(new_names.iter()) {
df.as_mut()
.rename(from, to)
.map_err(|e| ShellError::GenericError {
error: "Error renaming".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
}
Ok(PipelineData::Value(df.into_value(call.head), None))
}
fn command_lazy(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
lazy: NuLazyFrame,
) -> Result<PipelineData, ShellError> {
let columns: Value = call.req(engine_state, stack, 0)?;
let columns = extract_strings(columns)?;
let new_names: Value = call.req(engine_state, stack, 1)?;
let new_names = extract_strings(new_names)?;
if columns.len() != new_names.len() {
let value: Value = call.req(engine_state, stack, 1)?;
return Err(ShellError::IncompatibleParametersSingle {
msg: "New name list has different size to column list".into(),
span: value.span(),
});
}
let lazy = lazy.into_polars();
let lazy: NuLazyFrame = lazy.rename(&columns, &new_names).into();
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(RenameDF {})])
}
}

127
crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs
View File

@ -1,127 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::{prelude::NamedFrom, series::Series};
#[derive(Clone)]
pub struct SampleDF;
impl Command for SampleDF {
fn name(&self) -> &str {
"dfr sample"
}
fn usage(&self) -> &str {
"Create sample dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named(
"n-rows",
SyntaxShape::Int,
"number of rows to be taken from dataframe",
Some('n'),
)
.named(
"fraction",
SyntaxShape::Number,
"fraction of dataframe to be taken",
Some('f'),
)
.named(
"seed",
SyntaxShape::Number,
"seed for the selection",
Some('s'),
)
.switch("replace", "sample with replace", Some('e'))
.switch("shuffle", "shuffle sample", Some('u'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Sample rows from dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr sample --n-rows 1",
result: None, // No expected value because sampling is random
},
Example {
description: "Shows sample row using fraction and replace",
example:
"[[a b]; [1 2] [3 4] [5 6]] | dfr into-df | dfr sample --fraction 0.5 --replace",
result: None, // No expected value because sampling is random
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let rows: Option<Spanned<i64>> = call.get_flag(engine_state, stack, "n-rows")?;
let fraction: Option<Spanned<f64>> = call.get_flag(engine_state, stack, "fraction")?;
let seed: Option<u64> = call
.get_flag::<i64>(engine_state, stack, "seed")?
.map(|val| val as u64);
let replace: bool = call.has_flag(engine_state, stack, "replace")?;
let shuffle: bool = call.has_flag(engine_state, stack, "shuffle")?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
match (rows, fraction) {
(Some(rows), None) => df
.as_ref()
.sample_n(&Series::new("s", &[rows.item]), replace, shuffle, seed)
.map_err(|e| ShellError::GenericError {
error: "Error creating sample".into(),
msg: e.to_string(),
span: Some(rows.span),
help: None,
inner: vec![],
}),
(None, Some(frac)) => df
.as_ref()
.sample_frac(&Series::new("frac", &[frac.item]), replace, shuffle, seed)
.map_err(|e| ShellError::GenericError {
error: "Error creating sample".into(),
msg: e.to_string(),
span: Some(frac.span),
help: None,
inner: vec![],
}),
(Some(_), Some(_)) => Err(ShellError::GenericError {
error: "Incompatible flags".into(),
msg: "Only one selection criterion allowed".into(),
span: Some(call.head),
help: None,
inner: vec![],
}),
(None, None) => Err(ShellError::GenericError {
error: "No selection".into(),
msg: "No selection criterion was found".into(),
span: Some(call.head),
help: Some("Perhaps you want to use the flag -n or -f".into()),
inner: vec![],
}),
}
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}

112
crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs
View File

@ -1,112 +0,0 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct SchemaDF;
impl Command for SchemaDF {
fn name(&self) -> &str {
"dfr schema"
}
fn usage(&self) -> &str {
"Show schema for a dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.switch("datatype-list", "creates a lazy dataframe", Some('l'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Dataframe schema",
example: r#"[[a b]; [1 "foo"] [3 "bar"]] | dfr into-df | dfr schema"#,
result: Some(Value::record(
record! {
"a" => Value::string("i64", Span::test_data()),
"b" => Value::string("str", Span::test_data()),
},
Span::test_data(),
)),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
if call.has_flag(engine_state, stack, "datatype-list")? {
Ok(PipelineData::Value(datatype_list(Span::unknown()), None))
} else {
command(engine_state, stack, call, input)
}
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let schema = df.schema();
let value: Value = schema.into();
Ok(PipelineData::Value(value, None))
}
fn datatype_list(span: Span) -> Value {
let types: Vec<Value> = [
("null", ""),
("bool", ""),
("u8", ""),
("u16", ""),
("u32", ""),
("u64", ""),
("i8", ""),
("i16", ""),
("i32", ""),
("i64", ""),
("f32", ""),
("f64", ""),
("str", ""),
("binary", ""),
("date", ""),
("datetime<time_unit: (ms, us, ns) timezone (optional)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns. Timezone wildcard is *. Other Timezone examples: UTC, America/Los_Angeles."),
("duration<time_unit: (ms, us, ns)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns."),
("time", ""),
("object", ""),
("unknown", ""),
("list<dtype>", ""),
]
.iter()
.map(|(dtype, note)| {
Value::record(record! {
"dtype" => Value::string(*dtype, span),
"note" => Value::string(*note, span),
},
span)
})
.collect();
Value::list(types, span)
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(SchemaDF {})])
}
}

82
crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs
View File

@ -1,82 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ShapeDF;
impl Command for ShapeDF {
fn name(&self) -> &str {
"dfr shape"
}
fn usage(&self) -> &str {
"Shows column and row size for a dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Shows row and column shape",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr shape",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("rows".to_string(), vec![Value::test_int(2)]),
Column::new("columns".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let rows = Value::int(df.as_ref().height() as i64, call.head);
let cols = Value::int(df.as_ref().width() as i64, call.head);
let rows_col = Column::new("rows".to_string(), vec![rows]);
let cols_col = Column::new("columns".to_string(), vec![cols]);
NuDataFrame::try_from_columns(vec![rows_col, cols_col], None)
.map(|df| PipelineData::Value(df.into_value(call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ShapeDF {})])
}
}

84
crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs
View File

@ -1,84 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct SliceDF;
impl Command for SliceDF {
fn name(&self) -> &str {
"dfr slice"
}
fn usage(&self) -> &str {
"Creates new dataframe from a slice of rows."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("offset", SyntaxShape::Int, "start of slice")
.required("size", SyntaxShape::Int, "size of slice")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Create new dataframe from a slice of the rows",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr slice 0 1",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(1)]),
Column::new("b".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let offset: i64 = call.req(engine_state, stack, 0)?;
let size: usize = call.req(engine_state, stack, 1)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let res = df.as_ref().slice(offset, size);
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(SliceDF {})])
}
}

200
crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs
View File

@ -1,200 +0,0 @@
use polars::error::PolarsError;
use polars::prelude::{col, lit, DataType, Expr, LiteralValue, PolarsResult as Result, TimeUnit};
use sqlparser::ast::{
ArrayElemTypeDef, BinaryOperator as SQLBinaryOperator, DataType as SQLDataType,
Expr as SqlExpr, Function as SQLFunction, Value as SqlValue, WindowType,
};
fn map_sql_polars_datatype(data_type: &SQLDataType) -> Result<DataType> {
Ok(match data_type {
SQLDataType::Char(_)
| SQLDataType::Varchar(_)
| SQLDataType::Uuid
| SQLDataType::Clob(_)
| SQLDataType::Text
| SQLDataType::String(_) => DataType::String,
SQLDataType::Float(_) => DataType::Float32,
SQLDataType::Real => DataType::Float32,
SQLDataType::Double => DataType::Float64,
SQLDataType::TinyInt(_) => DataType::Int8,
SQLDataType::UnsignedTinyInt(_) => DataType::UInt8,
SQLDataType::SmallInt(_) => DataType::Int16,
SQLDataType::UnsignedSmallInt(_) => DataType::UInt16,
SQLDataType::Int(_) => DataType::Int32,
SQLDataType::UnsignedInt(_) => DataType::UInt32,
SQLDataType::BigInt(_) => DataType::Int64,
SQLDataType::UnsignedBigInt(_) => DataType::UInt64,
SQLDataType::Boolean => DataType::Boolean,
SQLDataType::Date => DataType::Date,
SQLDataType::Time(_, _) => DataType::Time,
SQLDataType::Timestamp(_, _) => DataType::Datetime(TimeUnit::Microseconds, None),
SQLDataType::Interval => DataType::Duration(TimeUnit::Microseconds),
SQLDataType::Array(array_type_def) => match array_type_def {
ArrayElemTypeDef::AngleBracket(inner_type)
| ArrayElemTypeDef::SquareBracket(inner_type) => {
DataType::List(Box::new(map_sql_polars_datatype(inner_type)?))
}
_ => {
return Err(PolarsError::ComputeError(
"SQL Datatype Array(None) was not supported in polars-sql yet!".into(),
))
}
},
_ => {
return Err(PolarsError::ComputeError(
format!("SQL Datatype {data_type:?} was not supported in polars-sql yet!").into(),
))
}
})
}
fn cast_(expr: Expr, data_type: &SQLDataType) -> Result<Expr> {
let polars_type = map_sql_polars_datatype(data_type)?;
Ok(expr.cast(polars_type))
}
fn binary_op_(left: Expr, right: Expr, op: &SQLBinaryOperator) -> Result<Expr> {
Ok(match op {
SQLBinaryOperator::Plus => left + right,
SQLBinaryOperator::Minus => left - right,
SQLBinaryOperator::Multiply => left * right,
SQLBinaryOperator::Divide => left / right,
SQLBinaryOperator::Modulo => left % right,
SQLBinaryOperator::StringConcat => {
left.cast(DataType::String) + right.cast(DataType::String)
}
SQLBinaryOperator::Gt => left.gt(right),
SQLBinaryOperator::Lt => left.lt(right),
SQLBinaryOperator::GtEq => left.gt_eq(right),
SQLBinaryOperator::LtEq => left.lt_eq(right),
SQLBinaryOperator::Eq => left.eq(right),
SQLBinaryOperator::NotEq => left.eq(right).not(),
SQLBinaryOperator::And => left.and(right),
SQLBinaryOperator::Or => left.or(right),
SQLBinaryOperator::Xor => left.xor(right),
_ => {
return Err(PolarsError::ComputeError(
format!("SQL Operator {op:?} was not supported in polars-sql yet!").into(),
))
}
})
}
fn literal_expr(value: &SqlValue) -> Result<Expr> {
Ok(match value {
SqlValue::Number(s, _) => {
// Check for existence of decimal separator dot
if s.contains('.') {
s.parse::<f64>().map(lit).map_err(|_| {
PolarsError::ComputeError(format!("Can't parse literal {s:?}").into())
})
} else {
s.parse::<i64>().map(lit).map_err(|_| {
PolarsError::ComputeError(format!("Can't parse literal {s:?}").into())
})
}?
}
SqlValue::SingleQuotedString(s) => lit(s.clone()),
SqlValue::NationalStringLiteral(s) => lit(s.clone()),
SqlValue::HexStringLiteral(s) => lit(s.clone()),
SqlValue::DoubleQuotedString(s) => lit(s.clone()),
SqlValue::Boolean(b) => lit(*b),
SqlValue::Null => Expr::Literal(LiteralValue::Null),
_ => {
return Err(PolarsError::ComputeError(
format!("Parsing SQL Value {value:?} was not supported in polars-sql yet!").into(),
))
}
})
}
pub fn parse_sql_expr(expr: &SqlExpr) -> Result<Expr> {
Ok(match expr {
SqlExpr::Identifier(e) => col(&e.value),
SqlExpr::BinaryOp { left, op, right } => {
let left = parse_sql_expr(left)?;
let right = parse_sql_expr(right)?;
binary_op_(left, right, op)?
}
SqlExpr::Function(sql_function) => parse_sql_function(sql_function)?,
SqlExpr::Cast {
expr,
data_type,
format: _,
} => cast_(parse_sql_expr(expr)?, data_type)?,
SqlExpr::Nested(expr) => parse_sql_expr(expr)?,
SqlExpr::Value(value) => literal_expr(value)?,
_ => {
return Err(PolarsError::ComputeError(
format!("Expression: {expr:?} was not supported in polars-sql yet!").into(),
))
}
})
}
fn apply_window_spec(expr: Expr, window_type: Option<&WindowType>) -> Result<Expr> {
Ok(match &window_type {
Some(wtype) => match wtype {
WindowType::WindowSpec(window_spec) => {
// Process for simple window specification, partition by first
let partition_by = window_spec
.partition_by
.iter()
.map(parse_sql_expr)
.collect::<Result<Vec<_>>>()?;
expr.over(partition_by)
// Order by and Row range may not be supported at the moment
}
// TODO: make NamedWindow work
WindowType::NamedWindow(_named) => {
return Err(PolarsError::ComputeError(
format!("Expression: {expr:?} was not supported in polars-sql yet!").into(),
))
}
},
None => expr,
})
}
fn parse_sql_function(sql_function: &SQLFunction) -> Result<Expr> {
use sqlparser::ast::{FunctionArg, FunctionArgExpr};
// Function name mostly do not have name space, so it mostly take the first args
let function_name = sql_function.name.0[0].value.to_ascii_lowercase();
let args = sql_function
.args
.iter()
.map(|arg| match arg {
FunctionArg::Named { arg, .. } => arg,
FunctionArg::Unnamed(arg) => arg,
})
.collect::<Vec<_>>();
Ok(
match (
function_name.as_str(),
args.as_slice(),
sql_function.distinct,
) {
("sum", [FunctionArgExpr::Expr(expr)], false) => {
apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.sum()
}
("count", [FunctionArgExpr::Expr(expr)], false) => {
apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.count()
}
("count", [FunctionArgExpr::Expr(expr)], true) => {
apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.n_unique()
}
// Special case for wildcard args to count function.
("count", [FunctionArgExpr::Wildcard], false) => lit(1i32).count(),
_ => {
return Err(PolarsError::ComputeError(
format!(
"Function {function_name:?} with args {args:?} was not supported in polars-sql yet!"
)
.into(),
))
}
},
)
}

279
crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs
View File

@ -1,279 +0,0 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::{
chunked_array::ChunkedArray,
prelude::{
AnyValue, DataFrame, DataType, Float64Type, IntoSeries, NewChunkedArray,
QuantileInterpolOptions, Series, StringType,
},
};
#[derive(Clone)]
pub struct Summary;
impl Command for Summary {
fn name(&self) -> &str {
"dfr summary"
}
fn usage(&self) -> &str {
"For a dataframe, produces descriptive statistics (summary statistics) for its numeric columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.category(Category::Custom("dataframe".into()))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.named(
"quantiles",
SyntaxShape::Table(vec![]),
"provide optional quantiles",
Some('q'),
)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "list dataframe descriptives",
example: "[[a b]; [1 1] [1 1]] | dfr into-df | dfr summary",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"descriptor".to_string(),
vec![
Value::test_string("count"),
Value::test_string("sum"),
Value::test_string("mean"),
Value::test_string("median"),
Value::test_string("std"),
Value::test_string("min"),
Value::test_string("25%"),
Value::test_string("50%"),
Value::test_string("75%"),
Value::test_string("max"),
],
),
Column::new(
"a (i64)".to_string(),
vec![
Value::test_float(2.0),
Value::test_float(2.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(0.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
],
),
Column::new(
"b (i64)".to_string(),
vec![
Value::test_float(2.0),
Value::test_float(2.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(0.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let quantiles: Option<Vec<Value>> = call.get_flag(engine_state, stack, "quantiles")?;
let quantiles = quantiles.map(|values| {
values
.iter()
.map(|value| {
let span = value.span();
match value {
Value::Float { val, .. } => {
if (&0.0..=&1.0).contains(&val) {
Ok(*val)
} else {
Err(ShellError::GenericError {
error: "Incorrect value for quantile".into(),
msg: "value should be between 0 and 1".into(),
span: Some(span),
help: None,
inner: vec![],
})
}
}
Value::Error { error, .. } => Err(*error.clone()),
_ => Err(ShellError::GenericError {
error: "Incorrect value for quantile".into(),
msg: "value should be a float".into(),
span: Some(span),
help: None,
inner: vec![],
}),
}
})
.collect::<Result<Vec<f64>, ShellError>>()
});
let quantiles = match quantiles {
Some(quantiles) => quantiles?,
None => vec![0.25, 0.50, 0.75],
};
let mut quantiles_labels = quantiles
.iter()
.map(|q| Some(format!("{}%", q * 100.0)))
.collect::<Vec<Option<String>>>();
let mut labels = vec![
Some("count".to_string()),
Some("sum".to_string()),
Some("mean".to_string()),
Some("median".to_string()),
Some("std".to_string()),
Some("min".to_string()),
];
labels.append(&mut quantiles_labels);
labels.push(Some("max".to_string()));
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let names = ChunkedArray::<StringType>::from_slice_options("descriptor", &labels).into_series();
let head = std::iter::once(names);
let tail = df
.as_ref()
.get_columns()
.iter()
.filter(|col| !matches!(col.dtype(), &DataType::Object("object", _)))
.map(|col| {
let count = col.len() as f64;
let sum = col.sum_as_series().ok().and_then(|series| {
series
.cast(&DataType::Float64)
.ok()
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
});
let mean = match col.mean_as_series().get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
};
let median = match col.median_as_series() {
Ok(v) => match v.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
},
_ => None,
};
let std = match col.std_as_series(0) {
Ok(v) => match v.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
},
_ => None,
};
let min = col.min_as_series().ok().and_then(|series| {
series
.cast(&DataType::Float64)
.ok()
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
});
let mut quantiles = quantiles
.clone()
.into_iter()
.map(|q| {
col.quantile_as_series(q, QuantileInterpolOptions::default())
.ok()
.and_then(|ca| ca.cast(&DataType::Float64).ok())
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
})
.collect::<Vec<Option<f64>>>();
let max = col.max_as_series().ok().and_then(|series| {
series
.cast(&DataType::Float64)
.ok()
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
});
let mut descriptors = vec![Some(count), sum, mean, median, std, min];
descriptors.append(&mut quantiles);
descriptors.push(max);
let name = format!("{} ({})", col.name(), col.dtype());
ChunkedArray::<Float64Type>::from_slice_options(&name, &descriptors).into_series()
});
let res = head.chain(tail).collect::<Vec<Series>>();
DataFrame::new(res)
.map_err(|e| ShellError::GenericError {
error: "Dataframe Error".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(Summary {})])
}
}

Some files were not shown because too many files have changed in this diff Show More