stdlib: make the library a standalone crate (#8770)

# Description
as we now have a prelude thanks to #8627, i'd like to work on the
structure of the library 😋

and i think the first step is to make it a true standalone crate 😏

this PR
- moves all the library from `crates/nu-utils/standard_library/` to
`crates/nu-std/`
- moves the `rust` loading code from `src/run.rs` to
`crates/nu-std/src/lib.rs`

crates/nu-std/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+authors = ["The Nushell Project Developers"]
+description = "The standard library of Nushell"
+repository = "https://github.com/nushell/nushell/tree/main/crates/nu-std"
+edition = "2021"
+license = "MIT"
+name = "nu-std"
+version = "0.78.1"
+
+[dependencies]
+miette = { version = "5.6.0", features = ["fancy-no-backtrace"] }
+nu-cli = { version = "0.78.1", path = "../nu-cli" }
+nu-parser = { version = "0.78.1", path = "../nu-parser" }
+nu-protocol = { version = "0.78.1", path = "../nu-protocol" }

crates/nu-std/README.md

@@ -0,0 +1,76 @@
+<h1 align="center">
+  Welcome to the standard library of `nushell`!
+  <img src="https://media.giphy.com/media/hvRJCLFzcasrR4ia7z/giphy.gif" width="28"></img>
+</h1>
+
+The standard library is a pure-`nushell` collection of commands to allow anyone to build
+complex applications using standardized tools gathered incrementally.
+
+In this library, you might find `rust`-like `assert` commands to write tests, tools to
+manipulate paths and strings, etc, etc, ...
+
+## :toolbox: use the standard library in the REPL or in scripts
+in order to "import" the standard library to either the interactive [*REPL*][REPL] of
+`nushell` or inside some `.nu` script, you might want to use the
+[`use`](https://nushell.sh/commands/docs/use.html) command!
+```bash
+use /path/to/standard_library/std.nu
+```
+
+> ### :mag: a concrete example
+> - my name is @amtoine and i use the `ghq` tool to manage `git` projects
+> > **Note**  
+> > `ghq` stores any repository inside `$env.GHQ_ROOT` under `<host>/<owner>/<repo>/`
+> - the path to my local fork of `nushell` is then defined as
+> ```bash
+> let-env NUSHELL_REPO = ($env.GHQ_ROOT | path join "github.com" "amtoine" "nushell")
+> ```
+> - and the full path to the standard library is defined as
+> ```bash
+> let-env STD_LIB = ($env.NUSHELL_REPO | path join "crates" "nu-utils" "standard_library")
+> ```
+> > see the content of `$env.STD_LIB` :yum:
+> > ```bash
+> > >_ ls $env.STD_LIB | get name | str replace $env.STD_LIB "" | str trim -l -c "/"
+> > ╭───┬───────────╮
+> > │ 0 │ README.md │
+> > │ 1 │ std.nu    │
+> > │ 2 │ tests.nu  │
+> > ╰───┴───────────╯
+> > ```
+> - finally we can `use` the standard library and have access to the commands it exposes :thumbsup:
+> ```bash
+> >_ use std.nu
+> >_ help std
+> Module: std
+>
+> Exported commands:
+>   assert (std assert), assert eq (std assert eq), assert ne (std assert ne), match (std match)
+>
+> This module does not export environment.
+> ```
+
+## :pencil2: contribute to the standard library
+- all the commands of the standard_library are located in [`std.nu`](std.nu)
+- the tests are located in files that have a name starting with "test_", e.g. [`test_std.nu`](test_std.nu)
+- a test runner, at [`tests.nu`](tests.nu), allows to run all the tests automatically
+
+### :wrench: add new commands
+- add new standard commands by appending to [`std.nu`](std.nu)
+- add associated tests to [`test_std.nu`](tests_std.nu) or preferably to `test_<submodule>.nu`.
+    - define a new exported (!) `test_<feature>` command
+    - import the `assert` functions you need at the top of the functions, e.g. `use std.nu "assert eq"`
+
+### :test_tube: run the tests
+the following call should return no errors
+```bash
+NU_LOG_LEVEL=DEBUG nu /path/to/standard_library/tests.nu
+```
+
+> #### :mag: a concrete example
+> with `STD_LIB` defined as in the example above
+> ```bash
+> NU_LOG_LEVEL=DEBUG nu ($env.STD_LIB | path join "tests.nu")
+> ```
+
+[REPL]: https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop

crates/nu-std/src/lib.rs

@@ -0,0 +1,88 @@
+use nu_cli::report_error;
+use nu_parser::{parse, parse_module_block};
+use nu_protocol::{engine::StateWorkingSet, Module, ShellError, Span};
+
+fn get_standard_library() -> &'static str {
+    include_str!("../std.nu")
+}
+
+fn load_prelude(working_set: &mut StateWorkingSet, prelude: Vec<(&str, &str)>, module: &Module) {
+    let mut decls = Vec::new();
+    let mut errs = Vec::new();
+    for (name, search_name) in prelude {
+        if let Some(id) = module.decls.get(&search_name.as_bytes().to_vec()) {
+            let decl = (name.as_bytes().to_vec(), id.to_owned());
+            decls.push(decl);
+        } else {
+            errs.push(ShellError::GenericError(
+                format!("could not load `{}` from `std`.", search_name),
+                String::new(),
+                None,
+                None,
+                Vec::new(),
+            ));
+        }
+    }
+
+    if !errs.is_empty() {
+        report_error(
+            working_set,
+            &ShellError::GenericError(
+                "Unable to load the prelude of the standard library.".into(),
+                String::new(),
+                None,
+                Some("this is a bug: please file an issue in the [issue tracker](https://github.com/nushell/nushell/issues/new/choose)".to_string()),
+                errs,
+            ),
+        );
+    }
+
+    working_set.use_decls(decls);
+}
+
+pub fn load_standard_library(
+    engine_state: &mut nu_protocol::engine::EngineState,
+) -> Result<(), miette::ErrReport> {
+    let delta = {
+        let name = "std".to_string();
+        let content = get_standard_library().as_bytes();
+
+        let mut working_set = StateWorkingSet::new(engine_state);
+
+        let start = working_set.next_span_start();
+        working_set.add_file(name.clone(), content);
+        let end = working_set.next_span_start();
+
+        let (_, module, comments) =
+            parse_module_block(&mut working_set, Span::new(start, end), name.as_bytes());
+
+        if let Some(err) = working_set.parse_errors.first() {
+            report_error(&working_set, err);
+        }
+
+        parse(&mut working_set, Some(&name), content, true);
+
+        if let Some(err) = working_set.parse_errors.first() {
+            report_error(&working_set, err);
+        }
+
+        let prelude = vec![
+            ("std help", "help"),
+            ("std help commands", "help commands"),
+            ("std help aliases", "help aliases"),
+            ("std help modules", "help modules"),
+            ("std help externs", "help externs"),
+            ("std help operators", "help operators"),
+        ];
+
+        load_prelude(&mut working_set, prelude, &module);
+
+        working_set.add_module(&name, module, comments);
+
+        working_set.render()
+    };
+
+    engine_state.merge_delta(delta)?;
+
+    Ok(())
+}

crates/nu-std/test_asserts.nu

@@ -0,0 +1,62 @@
+use std.nu *
+
+export def test_assert [] {
+    assert true
+    assert (1 + 2 == 3)
+    assert error { assert false }
+    assert error { assert (1 + 2 == 4) }
+}
+
+export def test_assert_equal [] {
+    assert equal (1 + 2) 3
+    assert equal (0.1 + 0.2 | into string | into decimal) 0.3 # 0.30000000000000004 == 0.3
+    assert error { assert equal 1 "foo" }
+    assert error { assert equal (1 + 2) 4 }
+}
+
+export def test_assert_not_equal [] {
+    assert not equal (1 + 2) 4
+    assert not equal 1 "foo"
+    assert not equal (1 + 2) "3"
+    assert error { assert not equal 1 1 }
+}
+
+export def test_assert_error [] {
+    let failing_code = {|| missing_code_to_run}
+    assert error $failing_code
+
+    let good_code = {|| }
+    let assert_error_raised = (try { do assert $good_code; false } catch { true })
+    assert $assert_error_raised "The assert error should raise an error if there is no error in the executed code."
+}
+
+export def test_assert_less [] {
+    assert less 1 2
+    assert error { assert less 1 1 }
+}
+
+export def test_assert_less_or_equal [] {
+    assert less or equal 1 2
+    assert less or equal 1 1
+    assert error { assert less or equal 1 0 }
+}
+
+export def test_assert_greater [] {
+    assert greater 2 1
+    assert error { assert greater 2 2 }
+}
+
+export def test_assert_greater_or_equal [] {
+    assert greater or equal 1 1
+    assert greater or equal 2 1
+    assert error { assert greater or equal 0 1 }
+}
+
+export def test_assert_length [] {
+    assert length [0, 0, 0]  3
+    assert error { assert length [0, 0] 3 }
+}
+
+export def test_assert_skip [] {
+    assert skip # This test case is skipped on purpose
+}

crates/nu-std/test_dirs.nu

@@ -0,0 +1,74 @@
+use std.nu "assert length"
+use std.nu "assert equal"
+
+def clean [path: path] {
+    cd $path
+    cd ..
+    rm -r $path
+}
+
+export def test_dirs_command [] {
+    # need some directories to play with
+    let base_path = ($nu.temp-path | path join $"test_dirs_(random uuid)")
+    let path_a = ($base_path | path join "a")
+    let path_b = ($base_path | path join "b")
+
+    try {
+        mkdir $base_path $path_a $path_b
+
+        cd $base_path
+        use std.nu "dirs next"
+        use std.nu "dirs prev"
+        use std.nu "dirs add"
+        use std.nu "dirs drop"
+        use std.nu "dirs show"
+
+        assert length $env.DIRS_LIST 1 "list is just pwd after initialization"
+        assert equal $base_path $env.DIRS_LIST.0 "list is just pwd after initialization"
+
+        dirs next
+        assert equal $base_path $env.DIRS_LIST.0 "next wraps at end of list"
+
+        dirs prev
+        assert equal $base_path $env.DIRS_LIST.0 "prev wraps at top of list"
+
+        dirs add $path_b $path_a
+        assert equal $path_b $env.PWD "add changes PWD to first added dir"
+        assert length $env.DIRS_LIST 3 "add in fact adds to list"
+        assert equal $path_a $env.DIRS_LIST.2 "add in fact adds to list"
+
+        dirs next 2
+        assert equal $base_path $env.PWD "next wraps at end of list"
+
+        dirs prev 1
+        assert equal $path_a $env.PWD "prev wraps at start of list"
+
+        dirs drop
+        assert length $env.DIRS_LIST 2 "drop removes from list"
+        assert equal $base_path $env.PWD "drop changes PWD to next in list (after dropped element)"
+
+        assert equal (dirs show) [[active path]; [true $base_path] [false $path_b]] "show table contains expected information"
+    } catch { |error|
+        clean $base_path
+
+        let error = (
+            $error
+            | get debug
+            | str replace "{" "("
+            | str replace "}" ")"
+            | parse 'GenericError("{msg}", "{text}", Some(Span ( start: {start}, end: {end} )), {rest})'
+            | reject rest
+            | get 0
+        )
+        error make {
+            msg: $error.msg
+            label: {
+                text: $error.text
+                start: ($error.start | into int)
+                end: ($error.end | into int)
+            }
+        }
+    }
+
+    try { clean $base_path }
+}

crates/nu-std/test_logger.nu

@@ -0,0 +1,46 @@
+use std.nu *
+
+def run [system_level, message_level] {
+    cd $env.FILE_PWD
+    do {
+        nu -c $'use std.nu; NU_LOG_LEVEL=($system_level) std log ($message_level) "test message"' 
+    } | complete | get -i stderr
+}
+def "assert no message" [system_level, message_level] {
+    let output = (run $system_level $message_level)
+    assert equal "" $output
+}
+
+def "assert message" [system_level, message_level, message_level_str] {
+    let output = (run $system_level $message_level)
+    assert str contains $output $message_level_str
+    assert str contains $output "test message"
+}
+
+export def test_critical [] {
+    assert no message 99 critical
+    assert message CRITICAL critical CRT
+}
+export def test_error [] {
+    assert no message CRITICAL error 
+    assert message ERROR error ERR
+}
+export def test_warning [] {
+    assert no message ERROR warning 
+    assert message WARNING warning WRN
+}
+export def test_info [] {
+    assert no message WARNING info 
+    assert message INFO info "INF" #INF has to be quoted, otherwise it is the `inf` float
+}
+export def test_debug [] {
+    assert no message INFO debug 
+    assert message DEBUG debug DBG
+}
+export def example [] {
+    log debug "Debug message"
+    log info "Info message"
+    log warning "Warning message"
+    log error "Error message"
+    log critical "Critical message"
+}

crates/nu-std/test_std.nu

@@ -0,0 +1,24 @@
+use std.nu
+
+export def test_path_add [] {
+    use std.nu "assert equal"
+
+    with-env [PATH []] {
+        assert equal $env.PATH []
+
+        std path add "/foo/"
+        assert equal $env.PATH ["/foo/"]
+
+        std path add "/bar/" "/baz/"
+        assert equal $env.PATH ["/bar/", "/baz/", "/foo/"]
+
+        let-env PATH = []
+
+        std path add "foo"
+        std path add "bar" "baz" --append
+        assert equal $env.PATH ["foo", "bar", "baz"]
+
+        assert equal (std path add "fooooo" --ret) ["fooooo", "foo", "bar", "baz"]
+        assert equal $env.PATH ["fooooo", "foo", "bar", "baz"]
+    }
+}

crates/nu-std/test_xml.nu

@@ -0,0 +1,30 @@
+use std.nu "xaccess"
+use std.nu "xupdate"
+use std.nu "xinsert"
+use std.nu "assert equal"
+
+export def test_xml_xaccess [] {
+    let sample_xml = ('<a><b><c a="b"></c></b><c></c><d><e>z</e><e>x</e></d></a>' | from xml)
+
+    assert equal ($sample_xml | xaccess [a]) [$sample_xml]
+    assert equal ($sample_xml | xaccess [*]) [$sample_xml]
+    assert equal ($sample_xml | xaccess [* d e]) [[tag, attributes, content]; [e, {}, [[tag, attributes, content]; [null, null, z]]], [e, {}, [[tag, attributes, content]; [null, null, x]]]]
+    assert equal ($sample_xml | xaccess [* d e 1]) [[tag, attributes, content]; [e, {}, [[tag, attributes, content]; [null, null, x]]]]
+    assert equal ($sample_xml | xaccess [* * * {|e| $e.attributes != {}}]) [[tag, attributes, content]; [c, {a: b}, []]]
+}
+
+export def test_xml_xupdate [] {
+    let sample_xml = ('<a><b><c a="b"></c></b><c></c><d><e>z</e><e>x</e></d></a>' | from xml)
+
+    assert equal ($sample_xml | xupdate [*] {|x| $x | update attributes {i: j}}) ('<a i="j"><b><c a="b"></c></b><c></c><d><e>z</e><e>x</e></d></a>' | from xml)
+    assert equal ($sample_xml | xupdate [* d e *] {|x| $x | update content 'nushell'}) ('<a><b><c a="b"></c></b><c></c><d><e>nushell</e><e>nushell</e></d></a>' | from xml)
+    assert equal ($sample_xml | xupdate [* * * {|e| $e.attributes != {}}] {|x| $x | update content ['xml']}) {tag: a, attributes: {}, content: [[tag, attributes, content]; [b, {}, [[tag, attributes, content]; [c, {a: b}, [xml]]]], [c, {}, []], [d, {}, [[tag, attributes, content]; [e, {}, [[tag, attributes, content]; [null, null, z]]], [e, {}, [[tag, attributes, content]; [null, null, x]]]]]]}
+}
+
+export def test_xml_xinsert [] {
+    let sample_xml = ('<a><b><c a="b"></c></b><c></c><d><e>z</e><e>x</e></d></a>' | from xml)
+
+    assert equal ($sample_xml | xinsert [a] {tag: b attributes:{} content: []}) ('<a><b><c a="b"></c></b><c></c><d><e>z</e><e>x</e></d><b></b></a>' | from xml)
+    assert equal ($sample_xml | xinsert [a d *] {tag: null attributes: null content: 'n'} | to xml) '<a><b><c a="b"></c></b><c></c><d><e>zn</e><e>xn</e></d></a>'
+    assert equal ($sample_xml | xinsert [a *] {tag: null attributes: null content: 'n'}) ('<a><b><c a="b"></c>n</b><c>n</c><d><e>z</e><e>x</e>n</d></a>' | from xml)
+}

crates/nu-std/tests.nu

@@ -0,0 +1,153 @@
+use std.nu *
+
+# show a test record in a pretty way
+#
+# `$in` must be a `record<file: string, module: string, name: string, pass: bool>`.
+#
+# the output would be like
+# - "<indentation> x <module> <test>" all in red if failed
+# - "<indentation> s <module> <test>" all in yellow if skipped
+# - "<indentation>   <module> <test>" all in green if passed
+def show-pretty-test [indent: int = 4] {
+    let test = $in
+
+    [
+        (" " * $indent)
+        (match $test.result {
+            "pass" => { ansi green },
+            "skip" => { ansi yellow },
+            _ => { ansi red }
+        })
+        (match $test.result {
+            "pass" => " ",
+            "skip" => "s",
+            _ => { char failed }
+        })
+        " "
+        $"($test.module) ($test.name)"
+        (ansi reset)
+    ] | str join
+}
+
+def throw-error [error: record] {
+    error make {
+        msg: $"(ansi red)($error.msg)(ansi reset)"
+        label: {
+            text: ($error.label)
+            start: $error.span.start
+            end: $error.span.end
+        }
+    }
+}
+
+# Test executor
+#
+# It executes exported "test_*" commands in "test_*" modules
+def main [
+    --path: path, # Path to look for tests. Default: directory of this file.
+    --module: string, # Module to run tests. Default: all test modules found.
+    --command: string, # Test command to run. Default: all test command found in the files.
+    --list, # list the selected tests without running them.
+] {
+    let module_search_pattern = ('**' | path join ({
+        stem: ($module | default "test_*")
+        extension: nu
+    } | path join))
+
+    if not ($path | is-empty) {
+        if not ($path | path exists) {
+            throw-error {
+                msg: "directory_not_found"
+                label: "no such directory"
+                span: (metadata $path | get span)
+            }
+        }
+    }
+
+    let path = ($path | default $env.FILE_PWD)
+
+    if not ($module | is-empty) {
+        try { ls ($path | path join $module_search_pattern) | null } catch {
+            throw-error {
+                msg: "module_not_found"
+                label: $"no such module in ($path)"
+                span: (metadata $module | get span)
+            }
+        }
+    }
+
+    let tests = (
+        ls ($path | path join $module_search_pattern)
+        | each {|row| {file: $row.name name: ($row.name | path parse | get stem)}}
+        | upsert test {|module|
+            nu -c $'use ($module.file) *; $nu.scope.commands | select name module_name | to nuon'
+            | from nuon
+            | where module_name == $module.name
+            | where ($it.name | str starts-with "test_")
+            | get name
+        }
+        | flatten
+        | rename file module name
+    )
+
+    let tests_to_run = (if not ($command | is-empty) {
+        $tests | where name == $command
+    } else if not ($module | is-empty) {
+        $tests | where module == $module
+    } else {
+        $tests
+    })
+
+    if $list {
+        return ($tests_to_run | select module name file)
+    }
+
+    if ($tests_to_run | is-empty) {
+        error make --unspanned {msg: "no test to run"}
+    }
+
+    let tests = (
+        $tests_to_run
+        | group-by module
+        | transpose name tests
+        | each {|module|
+            log info $"Running tests in ($module.name)"
+            $module.tests | each {|test|
+                log debug $"Running test ($test.name)"
+                nu -c $'
+                    use ($test.file) ($test.name)
+                    try {
+                        ($test.name)
+                    } catch { |err|
+                        if $err.msg == "ASSERT:SKIP" {
+                            exit 2
+                        } else {
+                            $err | get raw
+                        }
+                    }
+                '
+                let result = match $env.LAST_EXIT_CODE {
+                    0 => "pass",
+                    2 => "skip",
+                    _ => "fail",
+                }
+                if $result == "skip" {
+                    log warning $"Test case ($test.name) is skipped"
+                }
+                $test | merge ({result: $result})
+            }
+        }
+        | flatten
+    )
+
+    if not ($tests | where result == "fail" | is-empty) {
+        let text = ([
+            $"(ansi purple)some tests did not pass (char lparen)see complete errors above(char rparen):(ansi reset)"
+            ""
+            ($tests | each {|test| ($test | show-pretty-test 4)} | str join "\n")
+            ""
+        ] | str join "\n")
+
+        error make --unspanned { msg: $text }
+    }
+}