extern/nushell
1
0
Fork 1
mirror of https://github.com/nushell/nushell.git synced 2024-12-25 08:29:07 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into initial-docker-command-impl

Browse Source
This commit is contained in:
Jonathan Rothberg 2019-09-23 17:57:56 -07:00
commit a1f26d947d
12 changed files with 344 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.editorconfig
View File

@ -6,4 +6,9 @@ indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = false
end_of_line = lf
end_of_line = lf
[*.{yml,yaml}]
indent_size = 2
charset = utf-8
insert_final_newline = true

98
.github/workflows/docker-publish.yml vendored Normal file
View File

@ -0,0 +1,98 @@
name: Publish consumable Docker images
on:
push:
tags: ['*.*.*']
jobs:
compile:
runs-on: ubuntu-latest
strategy:
matrix:
arch:
- x86_64-unknown-linux-musl
- x86_64-unknown-linux-gnu
steps:
- uses: actions/checkout@v1
- run: cargo install cross
- name: compile for specific target
env: { arch: '${{ matrix.arch }}' }
run: |
cross build --target ${{ matrix.arch }} --release
# leave only the executable file
rm -rd target/${{ matrix.arch }}/release/{*/*,*.d,*.rlib,.fingerprint}
find . -empty -delete
- uses: actions/upload-artifact@master
with:
name: ${{ matrix.arch }}
path: target/${{ matrix.arch }}/release
docker:
name: Build and publish docker images
needs: compile
runs-on: ubuntu-latest
strategy:
matrix:
tag:
- alpine
- slim
- debian
- glibc-busybox
- musl-busybox
- musl-distroless
- glibc-distroless
- glibc
- musl
include:
- { tag: alpine, base-image: alpine, arch: x86_64-unknown-linux-musl, plugin: true }
- { tag: slim, base-image: 'debian:stable-slim', arch: x86_64-unknown-linux-gnu, plugin: true }
- { tag: debian, base-image: debian, arch: x86_64-unknown-linux-gnu, plugin: true }
- { tag: glibc-busybox, base-image: 'busybox:glibc', arch: x86_64-unknown-linux-gnu, use-patch: true }
- { tag: musl-busybox, base-image: 'busybox:musl', arch: x86_64-unknown-linux-musl, }
- { tag: musl-distroless, base-image: 'gcr.io/distroless/static', arch: x86_64-unknown-linux-musl, }
- { tag: glibc-distroless, base-image: 'gcr.io/distroless/cc', arch: x86_64-unknown-linux-gnu, use-patch: true }
- { tag: glibc, base-image: scratch, arch: x86_64-unknown-linux-gnu, }
- { tag: musl, base-image: scratch, arch: x86_64-unknown-linux-musl, }
steps:
- uses: actions/checkout@v1
- uses: actions/download-artifact@master
with: { name: '${{ matrix.arch }}', path: target/release }
- name: Build and publish exact version
run: |
REGISTRY=${REGISTRY,,}; export TAG=${GITHUB_REF##*/}-${{ matrix.tag }};
export NU_BINS=target/release/$( [ ${{ matrix.plugin }} = true ] && echo nu* || echo nu )
export PATCH=$([ ${{ matrix.use-patch }} = true ] && echo .${{ matrix.tag }} || echo '')
chmod +x $NU_BINS
echo ${{ secrets.DOCKER_REGISTRY }} | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
docker-compose --file docker/docker-compose.package.yml build
docker-compose --file docker/docker-compose.package.yml push # exact version
env:
BASE_IMAGE: ${{ matrix.base-image }}
REGISTRY: docker.pkg.github.com/${{ github.repository }}
#region semantics tagging
- name: Retag and push without suffixing version
run: |
VERSION=${GITHUB_REF##*/}
docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${{ matrix.tag }}
docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%%.*}-${{ matrix.tag }}
docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%.*}-${{ matrix.tag }}
docker push ${REGISTRY,,}/nu:${VERSION%.*}-${{ matrix.tag }} # latest patch
docker push ${REGISTRY,,}/nu:${VERSION%%.*}-${{ matrix.tag }} # latest features
docker push ${REGISTRY,,}/nu:${{ matrix.tag }} # latest version
env: { REGISTRY: 'docker.pkg.github.com/${{ github.repository }}' }
- name: Retag and push debian as latest
if: matrix.tag == 'debian'
run: |
VERSION=${GITHUB_REF##*/}
docker tag ${REGISTRY,,}/nu:${{ matrix.tag }} ${REGISTRY,,}/nu:latest
docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%.*}
docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION%%.*}
docker tag ${REGISTRY,,}/nu:${VERSION}-${{ matrix.tag }} ${REGISTRY,,}/nu:${VERSION}
docker push ${REGISTRY,,}/nu:${VERSION} # exact version
docker push ${REGISTRY,,}/nu:${VERSION%%.*} # latest features
docker push ${REGISTRY,,}/nu:${VERSION%.*} # latest patch
docker push ${REGISTRY,,}/nu:latest # latest version
env: { REGISTRY: 'docker.pkg.github.com/${{ github.repository }}' }
#endregion semantics tagging

69
README.md
View File

@ -7,7 +7,7 @@
A modern shell for the GitHub era
![Example of nushell](images/nushell-autocomplete4.gif "Example of nushell")
![Example of nushell](images/nushell-autocomplete.gif "Example of nushell")
# Status
@ -19,7 +19,7 @@ Nu comes with a set of built-in commands (listed below). If a command is unknown
There are a few good resources to learn about Nu. First, there is a [book](https://book.nushell.sh) about Nu, currently in progress. The book focuses on using Nu and its core concepts.
If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://github.com/nushell/contributor-handbook/tree/master/en) to help get started. There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
If you're a developer who would like to contribute to Nu, we're also working on a [book for developers](https://github.com/nushell/contributor-book/tree/master/en) to help get started. There are also [good first issues](https://github.com/nushell/nushell/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) to help you dive in.
We also have an active [discord](https://discord.gg/NtAbbGn) and [twitter](https://twitter.com/nu_shell) if you'd like to come chat with us.
@ -50,14 +50,9 @@ cargo +nightly install nu
You can also install Nu with all the bells and whistles:
```
cargo +nightly install nu --features raw-key,clipboard
cargo +nightly install nu --all-features
```
The following optional features are currently supported:
* **raw-key** - direct keyboard input, which creates a smoother experience in viewing text and binaries
* **clipboard** - integration with the native clipboard via the `clip` command
## Docker
If you want to pull a pre-built container, you can browse tags for the [nushell organization](https://quay.io/organization/nushell)
@ -220,55 +215,57 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
| ------------- | ------------- |
| cd path | Change to a new path |
| cp source path | Copy files |
| date (--utc) | Get the current datetime |
| fetch url | Fetch contents from a url and retrieve data as a table if possible |
| help | Display help information about commands |
| ls (path) | View the contents of the current or given path |
| mkdir path | Make directories, creates intermediary directories as required. |
| mv source target | Move files or directories. |
| date (--utc) | Get the current datetime |
| open filename | Load a file into a cell, convert to table if possible (avoid by appending '--raw') |
| post url body (--user <user>) (--password <password>) | Post content to a url and retrieve data as a table if possible |
| ps | View current processes |
| sys | View information about the current system |
| which filename | Finds a program file. |
| open filename | Load a file into a cell, convert to table if possible (avoid by appending '--raw') |
| fetch url | Fetch contents from a url and retrieve data as a table if possible |
| post url body (--user <user>) (--password <password>) | Post content to a url and retrieve data as a table if possible |
| rm {file or directory} | Remove a file, (for removing directory append '--recursive') |
| version | Display Nu version |
## Shell commands
| exit (--now) | Exit the current shell (or all shells) |
| enter (path) | Create a new shell and begin at this path |
| p | Go to previous shell |
| n | Go to next shell |
| shells | Display the list of current shells |
| help | Display help information about commands |
| version | Display Nu version |
## Filters on tables (structured data)
| command | description |
| ------------- | ------------- |
| pick ...columns | Down-select table to only these columns |
| reject ...columns | Remove the given columns from the table |
| get column-or-column-path | Open column and get data from the corresponding cells |
| sort-by ...columns | Sort by the given columns |
| where condition | Filter table to match the condition |
| inc (column-or-column-path) | Increment a value or version. Optionally use the column of a table |
| add column-or-column-path value | Add a new column to the table |
| embed column | Creates a new table of one column with the given name, and places the current table inside of it |
| sum | Sum a column of values |
| edit column-or-column-path value | Edit an existing column to have a new value |
| embed column | Creates a new table of one column with the given name, and places the current table inside of it |
| first amount | Show only the first number of rows |
| get column-or-column-path | Open column and get data from the corresponding cells |
| inc (column-or-column-path) | Increment a value or version. Optionally use the column of a table |
| last amount | Show only the last number of rows |
| nth row-number | Return only the selected row |
| pick ...columns | Down-select table to only these columns |
| pivot --header-row <headers> | Pivot the tables, making columns into rows and vice versa |
| reject ...columns | Remove the given columns from the table |
| reverse | Reverses the table. |
| skip amount | Skip a number of rows |
| skip-while condition | Skips rows while the condition matches. |
| first amount | Show only the first number of rows |
| last amount | Show only the last number of rows |
| nth row-number | Return only the selected row |
| pivot --header-row <headers> | Pivot the tables, making columns into rows and vice versa |
| sort-by ...columns | Sort by the given columns |
| str (column) | Apply string function. Optionally use the column of a table |
| sum | Sum a column of values |
| tags | Read the tags (metadata) for values |
| to-json | Convert table into .json text |
| to-toml | Convert table into .toml text |
| to-yaml | Convert table into .yaml text |
| to-bson | Convert table into .bson text |
| to-csv | Convert table into .csv text |
| to-bson | Convert table into .bson binary data |
| to-tsv | Convert table into .tsv text |
| to-csv | Convert table into .csv text |
| to-json | Convert table into .json text |
| to-sqlite | Convert table to sqlite .db binary data |
| to-toml | Convert table into .toml text |
| to-tsv | Convert table into .tsv text |
| to-url | Convert table to a urlencoded string |
| to-yaml | Convert table into .yaml text |
| where condition | Filter table to match the condition |
## Filters on text (unstructured data)
| command | description |
@ -280,6 +277,7 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
| from-sqlite | Parse binary data as sqlite .db and create table |
| from-toml | Parse text as .toml and create table |
| from-tsv | Parse text as .tsv and create table |
| from-url | Parse urlencoded string and create a table |
| from-xml | Parse text as .xml and create a table |
| from-yaml | Parse text as a .yaml/.yml and create a table |
| lines | Split single string into rows, one per line |
@ -293,12 +291,13 @@ Nu adheres closely to a set of goals that make up its design philosophy. As feat
| command | description |
| ------------- | ------------- |
| autoview | View the contents of the pipeline as a table or list |
| binaryview | Autoview of binary data |
| clip | Copy the contents of the pipeline to the copy/paste buffer |
| binaryview | Autoview of binary data (optional feature) |
| clip | Copy the contents of the pipeline to the copy/paste buffer (optional feature) |
| save filename | Save the contents of the pipeline to a file |
| table | View the contents of the pipeline as a table |
| textview | Autoview of text data |
| tree | View the contents of the pipeline as a tree |
| tree | View the contents of the pipeline as a tree (optional feature) |
# License
The project is made available under the MIT license. See "LICENSE" for more information.

7
docker/Package.Dockerfile Normal file
View File

@ -0,0 +1,7 @@
ARG base
FROM ${base}
ARG artifact
COPY ${artifact} /bin/
ENTRYPOINT ["/bin/nu"]

15
docker/Package.glibc-busybox.Dockerfile Normal file
View File

@ -0,0 +1,15 @@
ARG base
FROM debian:stable-slim AS patch
FROM ${base}
ARG artifact
COPY ${artifact} /bin/
COPY --from=patch \
/lib/x86_64-linux-gnu/libz.so.1 \
/lib/x86_64-linux-gnu/libdl.so.2 \
/lib/x86_64-linux-gnu/librt.so.1 \
/lib/x86_64-linux-gnu/libgcc_s.so.1 \
/lib/x86_64-linux-gnu/
ENTRYPOINT ["/bin/nu"]

12
docker/Package.glibc-distroless.Dockerfile Normal file
View File

@ -0,0 +1,12 @@
ARG base
FROM debian:stable-slim AS patch
FROM ${base}
ARG artifact
COPY ${artifact} /bin/
COPY --from=patch \
/lib/x86_64-linux-gnu/libz.so.1 \
/lib/x86_64-linux-gnu/
ENTRYPOINT ["/bin/nu"]

11
docker/docker-compose.package.yml Normal file
View File

@ -0,0 +1,11 @@
version: '3'
services:
nushell:
image: ${REGISTRY}/nu:${TAG}
build:
context: ..
dockerfile: docker/Package${PATCH}.Dockerfile
args:
base: ${BASE_IMAGE}
artifact: ${NU_BINS}

124
docs/docker.md Normal file
View File

@ -0,0 +1,124 @@
# Docker Guide
| tag | base image | plugins | package manager | libs & bins | size |
| ------------------ | -------------------- | ------- | --------------- | ---------------------------------------------------------------- | ----------- |
| `latest`, `debian` | `debian:latest` | yes | apt | **a lot**, including _glibc_ | ~(48+62) MB |
| `slim` | `debian:stable-slim` | yes | apt | all `nu:debian` image but exclude [this list][.slimify-excludes] | ~(26+62) MB |
| `alpine` | `alpine:latest` | yes | apk | all `nu:musl-busybox` image + libcrypto, libssl, libtls, libz | ~(3+61) MB |
| `musl-busybox` | `busybox:musl` | no | — | GNU utils + _musl_ | ~(1+16) MB |
| `glibc-busybox` | `busybox:glibc` | no | — | GNU utils + _glibc_ | ~(3+17) MB |
| `musl-distroless` | `distroless/static` | no | — | see [here][distroless/base] | ~(2+16) MB |
| `glibc-distroless` | `distroless/cc` | no | — | `distroless/static` with _glibc_ | ~(17+17) MB |
| `glibc` | `scratch` | no | — | **only `nu` binary-executable** which depend on glibc runtime | ~17 MB |
| `musl` | `scratch` | no | — | **only `nu` binary-executable** statically linked to musl | ~16 MB |
[.slimify-excludes]: https://github.com/debuerreotype/debuerreotype/blob/master/scripts/.slimify-excludes
[distroless/base]: https://github.com/GoogleContainerTools/distroless/blob/master/base/README.md
## Image Variants
### `nu:<version>`
This is the defacto image. If you are unsure about what your needs are, you probably want to use this one. It is designed to be used both as a throw away container (mount your source code and start the container to start your app), as well as the base to build other images off of.
<details><summary>example</summary>
Let say you create a plugin in Rust.
- create a Dockerfile in your root project
```dockerfile
FROM nu:0.2
COPY /target/debug/nu_plugin_cowsay /bin/
ENTRYPOINT ["nu"]
```
- build your project first then run it via docker
```console
cargo build
docker run -it .
```
</details>
### `nu:<version>-slim`
This image does not contain the common packages contained in the default tag and only contains the minimal packages needed to run `nu`. Unless you are working in an environment where only the `nu` image will be deployed and you have space constraints, it's highly recommended to use the alpine image if you aim for small image size. Only use this image if you really need **both** `glibc` and small image size.
### `nu:<version>-alpine`
This image is based on the popular [Alpine Linux project](http://alpinelinux.org/), available in [the alpine official image][alpine]. Alpine Linux is much smaller than most distribution base images (~5MB), and thus leads to much slimmer images in general.
This variant is highly recommended when final image size being as small as possible is desired. The main caveat to note is that it does use `musl` libc instead of `glibc` and friends, so certain software might run into issues depending on the depth of their libc requirements. However, most software doesn't have an issue with this, so this variant is usually a very safe choice. See [this Hacker News comment thread](https://news.ycombinator.com/item?id=10782897) for more discussion of the issues that might arise and some pro/con comparisons of using Alpine-based images.
To minimize image size, it's uncommon for additional related tools (such as `git` or `bash`) to be included in Alpine-based images. Using this image as a base, add the things you need in your own Dockerfile (see the [alpine image description][alpine] for examples of how to install packages if you are unfamiliar).
### `nu:<version>-<libc-variant>`
This image is based on [`scratch`](https://hub.docker.com/_/scratch) which doesn't create an extra layer. This variants can be handy in a project that uses multiple programming language as you need a lot of tools. By using this in [multi-stage build][], you can slim down the docker image that need to be pulled.
[multi-stage build]: https://docs.docker.com/develop/develop-images/multistage-build/
<details><summary>example</summary>
- using `glibc` variant
```dockerfile
FROM nu:0.2-glibc as shell
FROM node:slim
# Build your plugins
COPY --from=shell /bin/nu /bin/
# Something else
ENTRYPOINT ["nu"]
```
- using `musl` variant
```dockerfile
FROM nu:musl as shell
FROM go:alpine
# Build your plugins
COPY --from=shell /bin/nu /bin/
# Something else
ENTRYPOINT ["nu"]
```
</details>
### `nu:<version>-<libc-variant>-distroless`
This image is base on [Distroless](https://github.com/GoogleContainerTools/distroless) which usually to contain only your application and its runtime dependencies. This image do not contain package managers, shells or any other programs you would expect to find in a standard Linux distribution except for nushell itself. All distroless variant always contains:
- ca-certificates
- A /etc/passwd entry for a root user
- A /tmp directory
- tzdata
As for `glibc-distroless` variant, it **adds**:
- glibc
- libssl
- openssl
> Most likely you want to use this in CI/CD environment for plugins that can be statically compiled.
<details><summary>example</summary>
```dockerfile
FROM nu:musl-distroless
COPY target/x86_64-unknown-linux-musl/release/nu_plugin_* /bin/
ENTRYPOINT ["nu"]
```
</details>
### `nu:<version>-<libc-variant>-busybox`
This image is based on [Busybox](http://www.busybox.net/) which is a very good ingredient to craft space-efficient distributions. It combines tiny versions of many common UNIX utilities into a single small executable. It also provides replacements for most of the utilities you usually find in GNU fileutils, shellutils, etc. The utilities in BusyBox generally have fewer options than their full-featured GNU cousins; however, the options that are included provide the expected functionality and behave very much like their GNU counterparts. Basically, this image provides a fairly complete environment for any small or embedded system.
> Use this only if you need common utilities like `tar`, `awk`, and many more but don't want extra blob like nushell plugins and others.
<details><summary>example</summary>
```dockerfile
FROM nu:0.2-glibc-busybox
ADD https://github.com/user/repo/releases/download/latest/nu_plugin_cowsay.tar.gz /tmp/
RUN tar xzfv nu_plugin_cowsay.tar.gz -C /bin --strip=1 nu_plugin_cowsay
ENTRYPOINT ["nu"]
```
</details>
[musl]: http://www.musl-libc.org/
[alpine]: https://hub.docker.com/_/alpine/

BIN
images/nushell-autocomplete.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.8 MiB

37
src/cli.rs
View File

@ -72,25 +72,30 @@ fn load_plugin(path: &std::path::Path, context: &mut Context) -> Result<(), Shel
trace!("processing {:?}", params);
if params.is_filter {
let fname = fname.to_string();
let name = params.name.clone();
context.add_commands(vec![whole_stream_command(PluginCommand::new(
name, fname, params,
))]);
Ok(())
let name = params.name.clone();
let fname = fname.to_string();
if context.has_command(&name) {
trace!("plugin {:?} already loaded.", &name);
} else {
let fname = fname.to_string();
let name = params.name.clone();
context.add_commands(vec![whole_stream_command(PluginSink::new(
name, fname, params,
))]);
Ok(())
if params.is_filter {
context.add_commands(vec![whole_stream_command(
PluginCommand::new(name, fname, params),
)]);
} else {
context.add_commands(vec![whole_stream_command(PluginSink::new(
name, fname, params,
))]);
};
}
Ok(())
}
Err(e) => Err(e),
},
Err(e) => Err(ShellError::string(format!("Error: {:?}", e))),
Err(e) => {
trace!("incompatible plugin {:?}", input);
Err(ShellError::string(format!("Error: {:?}", e)))
}
}
}
Err(e) => Err(ShellError::string(format!("Error: {:?}", e))),
@ -202,7 +207,9 @@ fn load_plugins(context: &mut Context) -> Result<(), ShellError> {
if is_valid_name && is_executable {
trace!("Trying {:?}", bin.display());
load_plugin(&bin, context)?;
// we are ok if this plugin load fails
let _ = load_plugin(&bin, context);
}
}
}

18
src/errors.rs
View File

@ -12,20 +12,6 @@ pub enum Description {
Synthetic(String),
}
impl Description {
pub fn from(value: Tagged<impl Into<String>>) -> Description {
let value_tag = value.tag();
match value_tag {
Tag {
span: crate::data::meta::Span { start: 0, end: 0 },
..
} => Description::Synthetic(value.item.into()),
_ => Description::Source(Tagged::from_item(value.item.into(), value_tag)),
}
}
}
impl Description {
fn into_label(self) -> Result<Label<Tag>, String> {
match self {
@ -114,10 +100,6 @@ impl ShellError {
.start()
}
pub(crate) fn missing_property(subpath: Description, expr: Description) -> ShellError {
ProximateShellError::MissingProperty { subpath, expr }.start()
}
pub(crate) fn missing_value(tag: Option<Tag>, reason: impl Into<String>) -> ShellError {
ProximateShellError::MissingValue {
tag,

20
src/evaluate/evaluator.rs
View File

@ -1,5 +1,5 @@
use crate::data::base::Block;
use crate::errors::{ArgumentError, Description};
use crate::errors::ArgumentError;
use crate::parser::{
hir::{self, Expression, RawExpression},
CommandRegistry, Text,
@ -87,10 +87,20 @@ pub(crate) fn evaluate_baseline_expr(
match next {
None => {
return Err(ShellError::missing_property(
Description::from(item.tagged_type_name()),
Description::from(name.clone()),
))
let possibilities = item.data_descriptors();
let mut possible_matches: Vec<_> = possibilities
.iter()
.map(|x| (natural::distance::levenshtein_distance(x, &name), x))
.collect();
possible_matches.sort();
return Err(ShellError::labeled_error(
"Unknown column",
format!("did you mean '{}'?", possible_matches[0].1),
expr.tag(),
));
}
Some(next) => {
item = next.clone().item.tagged(expr.tag());