Move unit test runner to standard library (#8850)

Move test runner to standard library.
Originated from #8819 

# After Submitting

I'll update the documentation about testing:
http://www.nushell.sh/book/testing.html

---------

Co-authored-by: Mate Farkas <Mate.Farkas@oneidentity.com>

.github/workflows/ci.yml

@@ -116,7 +116,7 @@ jobs:
         run: cargo install --path . --locked --no-default-features
 
       - name: Standard library tests
-        run: nu crates/nu-std/tests/run.nu
+        run: nu -c 'use std; std run-tests --path crates/nu-std'
 
       - name: Setup Python
         uses: actions/setup-python@v4

crates/nu-std/README.md

@@ -31,7 +31,7 @@ use std
 ### :test_tube: run the tests
 the following call should return no errors
 ```bash
-NU_LOG_LEVEL=DEBUG cargo run -- crates/nu-std/tests.nu
+NU_LOG_LEVEL=DEBUG cargo run -- -c "use std; std run-tests --path crates/nu-std"
 ```
 
 > **Warning**  

crates/nu-std/lib/assert.nu

@@ -1,208 +0,0 @@
-# Universal assert command
-#
-# If the condition is not true, it generates an error.
-#
-# # Example
-#
-# ```nushell
-# >_ assert (3 == 3)
-# >_ assert (42 == 3)
-# Error:
-#   × Assertion failed: 
-#     ╭─[myscript.nu:11:1]
-#  11 │ assert (3 == 3)
-#  12 │ assert (42 == 3)
-#     ·         ───┬────
-#     ·            ╰── It is not true.
-#  13 │
-#     ╰────
-# ```
-#
-# The --error-label flag can be used if you want to create a custom assert command:
-# ```
-# def "assert even" [number: int] {
-#     assert ($number mod 2 == 0) --error-label {
-#         start: (metadata $number).span.start,
-#         end: (metadata $number).span.end,
-#         text: $"($number) is not an even number",
-#     }
-# }
-# ```
-export def main [
-    condition: bool, # Condition, which should be true 
-    message?: string, # Optional error message
-    --error-label: record # Label for `error make` if you want to create a custom assert
-] {
-    if $condition { return }
-    let span = (metadata $condition).span
-    error make {
-        msg: ($message | default "Assertion failed."),
-        label: ($error_label | default {
-            text: "It is not true.",
-            start: (metadata $condition).span.start,
-            end: (metadata $condition).span.end
-        })
-    }
-}
-
-# Assert that executing the code generates an error
-#
-# For more documentation see the assert command
-# 
-# # Examples
-#
-# > assert error {|| missing_command} # passes
-# > assert error {|| 12} # fails
-export def "assert error" [
-    code: closure,
-    message?: string
-] {
-    let error_raised = (try { do $code; false } catch { true })
-    main ($error_raised) $message --error-label {
-        start: (metadata $code).span.start
-        end: (metadata $code).span.end
-        text: $"There were no error during code execution: (view source $code)"
-    }
-}
-
-# Skip the current test case
-#
-# # Examples
-#
-# if $condition { assert skip }
-export def "assert skip" [] {
-    error make {msg: "ASSERT:SKIP"}
-}
-
-
-# Assert $left == $right
-#
-# For more documentation see the assert command
-#
-# # Examples
-# 
-# > assert equal 1 1 # passes
-# > assert equal (0.1 + 0.2) 0.3
-# > assert equal 1 2 # fails
-export def "assert equal" [left: any, right: any, message?: string] {
-    main ($left == $right) $message --error-label {
-        start: (metadata $left).span.start
-        end: (metadata $right).span.end
-        text: $"They are not equal. Left = ($left). Right = ($right)."
-    }
-}
-
-# Assert $left != $right
-#
-# For more documentation see the assert command
-#
-# # Examples
-#
-# > assert not equal 1 2 # passes
-# > assert not equal 1 "apple" # passes
-# > assert not equal 7 7 # fails
-export def "assert not equal" [left: any, right: any, message?: string] {
-    main ($left != $right) $message --error-label {
-        start: (metadata $left).span.start
-        end: (metadata $right).span.end
-        text: $"They both are ($left)."
-    }
-}
-
-# Assert $left <= $right
-#
-# For more documentation see the assert command
-#
-# # Examples
-#
-# > assert less or equal 1 2 # passes
-# > assert less or equal 1 1 # passes
-# > assert less or equal 1 0 # fails
-export def "assert less or equal" [left: any, right: any, message?: string] {
-    main ($left <= $right) $message --error-label {
-        start: (metadata $left).span.start
-        end: (metadata $right).span.end
-        text: $"Left: ($left), Right: ($right)"
-    }
-}
-
-# Assert $left < $right
-#
-# For more documentation see the assert command
-#
-# # Examples
-#
-# > assert less 1 2 # passes
-# > assert less 1 1 # fails
-export def "assert less" [left: any, right: any, message?: string] {
-    main ($left < $right) $message --error-label {
-        start: (metadata $left).span.start
-        end: (metadata $right).span.end
-        text: $"Left: ($left), Right: ($right)"
-    }
-}
-
-# Assert $left > $right
-#
-# For more documentation see the assert command
-#
-# # Examples
-#
-# > assert greater 2 1 # passes
-# > assert greater 2 2 # fails
-export def "assert greater" [left: any, right: any, message?: string] {
-    main ($left > $right) $message --error-label {
-        start: (metadata $left).span.start
-        end: (metadata $right).span.end
-        text: $"Left: ($left), Right: ($right)"
-    }
-}
-
-# Assert $left >= $right
-#
-# For more documentation see the assert command
-#
-# # Examples
-#
-# > assert greater or equal 2 1 # passes
-# > assert greater or equal 2 2 # passes
-# > assert greater or equal 1 2 # fails
-export def "assert greater or equal" [left: any, right: any, message?: string] {
-    main ($left >= $right) $message --error-label {
-        start: (metadata $left).span.start
-        end: (metadata $right).span.end
-        text: $"Left: ($left), Right: ($right)"
-    }
-}
-
-# Assert length of $left is $right
-#
-# For more documentation see the assert command
-#
-# # Examples
-#
-# > assert length [0, 0] 2 # passes
-# > assert length [0] 3 # fails
-export def "assert length" [left: list, right: int, message?: string] {
-    main (($left | length) == $right) $message --error-label {
-        start: (metadata $left).span.start
-        end: (metadata $right).span.end
-        text: $"Length of ($left) is ($left | length), not ($right)"
-    }
-}
-
-# Assert that ($left | str contains $right)
-#
-# For more documentation see the assert command
-#
-# # Examples
-#
-# > assert str contains "arst" "rs" # passes
-# > assert str contains "arst" "k" # fails
-export def "assert str contains" [left: string, right: string, message?: string] {
-    main ($left | str contains $right) $message --error-label {
-        start: (metadata $left).span.start
-        end: (metadata $right).span.end
-        text: $"'($left)' does not contain '($right)'."
-    }
-}

crates/nu-std/lib/mod.nu

@@ -1,12 +1,12 @@
 # std.nu, `used` to load all standard library components
 
-export use assert *
 export use dirs *
 export-env {
     use dirs *
 }
 export use help *
 export use log *
+export use testing *
 export use xml *
 
 # Add the given paths to the PATH.

crates/nu-std/lib/testing.nu

@@ -0,0 +1,388 @@
+##################################################################################
+#
+# Module testing
+#
+# Assert commands and test runner.
+#
+##################################################################################
+use log *
+
+# Universal assert command
+#
+# If the condition is not true, it generates an error.
+#
+# # Example
+#
+# ```nushell
+# >_ assert (3 == 3)
+# >_ assert (42 == 3)
+# Error:
+#   × Assertion failed: 
+#     ╭─[myscript.nu:11:1]
+#  11 │ assert (3 == 3)
+#  12 │ assert (42 == 3)
+#     ·         ───┬────
+#     ·            ╰── It is not true.
+#  13 │
+#     ╰────
+# ```
+#
+# The --error-label flag can be used if you want to create a custom assert command:
+# ```
+# def "assert even" [number: int] {
+#     assert ($number mod 2 == 0) --error-label {
+#         start: (metadata $number).span.start,
+#         end: (metadata $number).span.end,
+#         text: $"($number) is not an even number",
+#     }
+# }
+# ```
+export def assert [
+    condition: bool, # Condition, which should be true 
+    message?: string, # Optional error message
+    --error-label: record # Label for `error make` if you want to create a custom assert
+] {
+    if $condition { return }
+    let span = (metadata $condition).span
+    error make {
+        msg: ($message | default "Assertion failed."),
+        label: ($error_label | default {
+            text: "It is not true.",
+            start: (metadata $condition).span.start,
+            end: (metadata $condition).span.end
+        })
+    }
+}
+
+# Assert that executing the code generates an error
+#
+# For more documentation see the assert command
+# 
+# # Examples
+#
+# > assert error {|| missing_command} # passes
+# > assert error {|| 12} # fails
+export def "assert error" [
+    code: closure,
+    message?: string
+] {
+    let error_raised = (try { do $code; false } catch { true })
+    assert ($error_raised) $message --error-label {
+        start: (metadata $code).span.start
+        end: (metadata $code).span.end
+        text: $"There were no error during code execution: (view source $code)"
+    }
+}
+
+# Skip the current test case
+#
+# # Examples
+#
+# if $condition { assert skip }
+export def "assert skip" [] {
+    error make {msg: "ASSERT:SKIP"}
+}
+
+
+# Assert $left == $right
+#
+# For more documentation see the assert command
+#
+# # Examples
+# 
+# > assert equal 1 1 # passes
+# > assert equal (0.1 + 0.2) 0.3
+# > assert equal 1 2 # fails
+export def "assert equal" [left: any, right: any, message?: string] {
+    assert ($left == $right) $message --error-label {
+        start: (metadata $left).span.start
+        end: (metadata $right).span.end
+        text: $"They are not equal. Left = ($left). Right = ($right)."
+    }
+}
+
+# Assert $left != $right
+#
+# For more documentation see the assert command
+#
+# # Examples
+#
+# > assert not equal 1 2 # passes
+# > assert not equal 1 "apple" # passes
+# > assert not equal 7 7 # fails
+export def "assert not equal" [left: any, right: any, message?: string] {
+    assert ($left != $right) $message --error-label {
+        start: (metadata $left).span.start
+        end: (metadata $right).span.end
+        text: $"They both are ($left)."
+    }
+}
+
+# Assert $left <= $right
+#
+# For more documentation see the assert command
+#
+# # Examples
+#
+# > assert less or equal 1 2 # passes
+# > assert less or equal 1 1 # passes
+# > assert less or equal 1 0 # fails
+export def "assert less or equal" [left: any, right: any, message?: string] {
+    assert ($left <= $right) $message --error-label {
+        start: (metadata $left).span.start
+        end: (metadata $right).span.end
+        text: $"Left: ($left), Right: ($right)"
+    }
+}
+
+# Assert $left < $right
+#
+# For more documentation see the assert command
+#
+# # Examples
+#
+# > assert less 1 2 # passes
+# > assert less 1 1 # fails
+export def "assert less" [left: any, right: any, message?: string] {
+    assert ($left < $right) $message --error-label {
+        start: (metadata $left).span.start
+        end: (metadata $right).span.end
+        text: $"Left: ($left), Right: ($right)"
+    }
+}
+
+# Assert $left > $right
+#
+# For more documentation see the assert command
+#
+# # Examples
+#
+# > assert greater 2 1 # passes
+# > assert greater 2 2 # fails
+export def "assert greater" [left: any, right: any, message?: string] {
+    assert ($left > $right) $message --error-label {
+        start: (metadata $left).span.start
+        end: (metadata $right).span.end
+        text: $"Left: ($left), Right: ($right)"
+    }
+}
+
+# Assert $left >= $right
+#
+# For more documentation see the assert command
+#
+# # Examples
+#
+# > assert greater or equal 2 1 # passes
+# > assert greater or equal 2 2 # passes
+# > assert greater or equal 1 2 # fails
+export def "assert greater or equal" [left: any, right: any, message?: string] {
+    assert ($left >= $right) $message --error-label {
+        start: (metadata $left).span.start
+        end: (metadata $right).span.end
+        text: $"Left: ($left), Right: ($right)"
+    }
+}
+
+# Assert length of $left is $right
+#
+# For more documentation see the assert command
+#
+# # Examples
+#
+# > assert length [0, 0] 2 # passes
+# > assert length [0] 3 # fails
+export def "assert length" [left: list, right: int, message?: string] {
+    assert (($left | length) == $right) $message --error-label {
+        start: (metadata $left).span.start
+        end: (metadata $right).span.end
+        text: $"Length of ($left) is ($left | length), not ($right)"
+    }
+}
+
+# Assert that ($left | str contains $right)
+#
+# For more documentation see the assert command
+#
+# # Examples
+#
+# > assert str contains "arst" "rs" # passes
+# > assert str contains "arst" "k" # fails
+export def "assert str contains" [left: string, right: string, message?: string] {
+    assert ($left | str contains $right) $message --error-label {
+        start: (metadata $left).span.start
+        end: (metadata $right).span.end
+        text: $"'($left)' does not contain '($right)'."
+    }
+}
+
+# show a test record in a pretty way
+#
+# `$in` must be a `record<file: string, module: string, name: string, pass: bool>`.
+#
+# the output would be like
+# - "<indentation> x <module> <test>" all in red if failed
+# - "<indentation> s <module> <test>" all in yellow if skipped
+# - "<indentation>   <module> <test>" all in green if passed
+def show-pretty-test [indent: int = 4] {
+    let test = $in
+
+    [
+        (" " * $indent)
+        (match $test.result {
+            "pass" => { ansi green },
+            "skip" => { ansi yellow },
+            _ => { ansi red }
+        })
+        (match $test.result {
+            "pass" => " ",
+            "skip" => "s",
+            _ => { char failed }
+        })
+        " "
+        $"($test.module) ($test.name)"
+        (ansi reset)
+    ] | str join
+}
+
+def throw-error [error: record] {
+    error make {
+        msg: $"(ansi red)($error.msg)(ansi reset)"
+        label: {
+            text: ($error.label)
+            start: $error.span.start
+            end: $error.span.end
+        }
+    }
+}
+
+# Run Nushell tests
+#
+# It executes exported "test_*" commands in "test_*" modules
+export def 'run-tests' [
+    --path: path, # Path to look for tests. Default: current directory.
+    --module: string, # Module to run tests. Default: all test modules found.
+    --command: string, # Test command to run. Default: all test command found in the files.
+    --list, # list the selected tests without running them.
+] {
+    let module_search_pattern = ('**' | path join ({
+        stem: ($module | default "test_*")
+        extension: nu
+    } | path join))
+
+    let path = ($path | default $env.PWD)
+
+    if not ($path | path exists) {
+        throw-error {
+            msg: "directory_not_found"
+            label: "no such directory"
+            span: (metadata $path | get span)
+        }
+    }
+
+    if not ($module | is-empty) {
+        try { ls ($path | path join $module_search_pattern) | null } catch {
+            throw-error {
+                msg: "module_not_found"
+                label: $"no such module in ($path)"
+                span: (metadata $module | get span)
+            }
+        }
+    }
+
+    let tests = (
+        ls ($path | path join $module_search_pattern)
+        | each {|row| {file: $row.name name: ($row.name | path parse | get stem)}}
+        | upsert commands {|module|
+            ^$nu.current-exe -c $'use `($module.file)` *; $nu.scope.commands | select name module_name | to nuon'
+            | from nuon
+            | where module_name == $module.name
+            | get name
+        }
+        | upsert test {|module| $module.commands | where ($it | str starts-with "test_") }
+        | upsert setup {|module| "setup" in $module.commands }
+        | upsert teardown {|module| "teardown" in $module.commands }
+        | reject commands
+        | flatten
+        | rename file module name
+    )
+
+    let tests_to_run = (if not ($command | is-empty) {
+        $tests | where name == $command
+    } else if not ($module | is-empty) {
+        $tests | where module == $module
+    } else {
+        $tests
+    })
+
+    if $list {
+        return ($tests_to_run | select module name file)
+    }
+
+    if ($tests_to_run | is-empty) {
+        error make --unspanned {msg: "no test to run"}
+    }
+
+    let tests = (
+        $tests_to_run
+        | group-by module
+        | transpose name tests
+        | each {|module|
+            log info $"Running tests in ($module.name)"
+            $module.tests | each {|test|
+                log debug $"Running test ($test.name)"
+
+                let context_setup = if $test.setup {
+                    $"use `($test.file)` setup; let context = \(setup\)"
+                } else {
+                    "let context = {}"
+                }
+
+                let context_teardown = if $test.teardown {
+                    $"use `($test.file)` teardown; $context | teardown"
+                } else {
+                    ""
+                }
+
+                let nu_script = $'
+                    ($context_setup)
+                    use `($test.file)` ($test.name)
+                    try {
+                        $context | ($test.name)
+                        ($context_teardown)
+                    } catch { |err|
+                        ($context_teardown)
+                        if $err.msg == "ASSERT:SKIP" {
+                            exit 2
+                        } else {
+                            $err | get raw
+                        }
+                    }
+                '
+                ^$nu.current-exe -c $nu_script
+
+                let result = match $env.LAST_EXIT_CODE {
+                    0 => "pass",
+                    2 => "skip",
+                    _ => "fail",
+                }
+                if $result == "skip" {
+                    log warning $"Test case ($test.name) is skipped"
+                }
+                $test | merge ({result: $result})
+            }
+        }
+        | flatten
+    )
+
+    if not ($tests | where result == "fail" | is-empty) {
+        let text = ([
+            $"(ansi purple)some tests did not pass (char lparen)see complete errors above(char rparen):(ansi reset)"
+            ""
+            ($tests | each {|test| ($test | show-pretty-test 4)} | str join "\n")
+            ""
+        ] | str join "\n")
+
+        error make --unspanned { msg: $text }
+    }
+}

crates/nu-std/src/lib.rs

@@ -71,11 +71,11 @@ pub fn load_standard_library(
         let submodules = vec![
             // helper modules that could be used in other parts of the library
             ("log", include_str!("../lib/log.nu")),
-            ("assert", include_str!("../lib/assert.nu")),
 
             // the rest of the library
             ("dirs", include_str!("../lib/dirs.nu")),
             ("help", include_str!("../lib/help.nu")),
+            ("testing", include_str!("../lib/testing.nu")),
             ("xml", include_str!("../lib/xml.nu")),
         ];
 

crates/nu-std/tests/run.nu

@@ -1,174 +0,0 @@
-use std *
-
-# show a test record in a pretty way
-#
-# `$in` must be a `record<file: string, module: string, name: string, pass: bool>`.
-#
-# the output would be like
-# - "<indentation> x <module> <test>" all in red if failed
-# - "<indentation> s <module> <test>" all in yellow if skipped
-# - "<indentation>   <module> <test>" all in green if passed
-def show-pretty-test [indent: int = 4] {
-    let test = $in
-
-    [
-        (" " * $indent)
-        (match $test.result {
-            "pass" => { ansi green },
-            "skip" => { ansi yellow },
-            _ => { ansi red }
-        })
-        (match $test.result {
-            "pass" => " ",
-            "skip" => "s",
-            _ => { char failed }
-        })
-        " "
-        $"($test.module) ($test.name)"
-        (ansi reset)
-    ] | str join
-}
-
-def throw-error [error: record] {
-    error make {
-        msg: $"(ansi red)($error.msg)(ansi reset)"
-        label: {
-            text: ($error.label)
-            start: $error.span.start
-            end: $error.span.end
-        }
-    }
-}
-
-# Test executor
-#
-# It executes exported "test_*" commands in "test_*" modules
-def main [
-    --path: path, # Path to look for tests. Default: directory of this file.
-    --module: string, # Module to run tests. Default: all test modules found.
-    --command: string, # Test command to run. Default: all test command found in the files.
-    --list, # list the selected tests without running them.
-] {
-    let module_search_pattern = ('**' | path join ({
-        stem: ($module | default "test_*")
-        extension: nu
-    } | path join))
-
-    if not ($path | is-empty) {
-        if not ($path | path exists) {
-            throw-error {
-                msg: "directory_not_found"
-                label: "no such directory"
-                span: (metadata $path | get span)
-            }
-        }
-    }
-
-    let path = ($path | default $env.FILE_PWD)
-
-    if not ($module | is-empty) {
-        try { ls ($path | path join $module_search_pattern) | null } catch {
-            throw-error {
-                msg: "module_not_found"
-                label: $"no such module in ($path)"
-                span: (metadata $module | get span)
-            }
-        }
-    }
-
-    let tests = (
-        ls ($path | path join $module_search_pattern)
-        | each {|row| {file: $row.name name: ($row.name | path parse | get stem)}}
-        | upsert commands {|module|
-            nu -c $'use `($module.file)` *; $nu.scope.commands | select name module_name | to nuon'
-            | from nuon
-            | where module_name == $module.name
-            | get name
-        }
-        | upsert test {|module| $module.commands | where ($it | str starts-with "test_") }
-        | upsert setup {|module| "setup" in $module.commands }
-        | upsert teardown {|module| "teardown" in $module.commands }
-        | reject commands
-        | flatten
-        | rename file module name
-    )
-
-    let tests_to_run = (if not ($command | is-empty) {
-        $tests | where name == $command
-    } else if not ($module | is-empty) {
-        $tests | where module == $module
-    } else {
-        $tests
-    })
-
-    if $list {
-        return ($tests_to_run | select module name file)
-    }
-
-    if ($tests_to_run | is-empty) {
-        error make --unspanned {msg: "no test to run"}
-    }
-
-    let tests = (
-        $tests_to_run
-        | group-by module
-        | transpose name tests
-        | each {|module|
-            log info $"Running tests in ($module.name)"
-            $module.tests | each {|test|
-                log debug $"Running test ($test.name)"
-
-                let context_setup = if $test.setup {
-                    $"use `($test.file)` setup; let context = \(setup\)"
-                } else {
-                    "let context = {}"
-                }
-
-                let context_teardown = if $test.teardown {
-                    $"use `($test.file)` teardown; $context | teardown"
-                } else {
-                    ""
-                }
-
-                let nu_script = $'
-                    ($context_setup)
-                    use `($test.file)` ($test.name)
-                    try {
-                        $context | ($test.name)
-                        ($context_teardown)
-                    } catch { |err|
-                        ($context_teardown)
-                        if $err.msg == "ASSERT:SKIP" {
-                            exit 2
-                        } else {
-                            $err | get raw
-                        }
-                    }
-                '
-                nu -c $nu_script
-
-                let result = match $env.LAST_EXIT_CODE {
-                    0 => "pass",
-                    2 => "skip",
-                    _ => "fail",
-                }
-                if $result == "skip" {
-                    log warning $"Test case ($test.name) is skipped"
-                }
-                $test | merge ({result: $result})
-            }
-        }
-        | flatten
-    )
-
-    if not ($tests | where result == "fail" | is-empty) {
-        let text = ([
-            $"(ansi purple)some tests did not pass (char lparen)see complete errors above(char rparen):(ansi reset)"
-            ""
-            ($tests | each {|test| ($test | show-pretty-test 4)} | str join "\n")
-            ""
-        ] | str join "\n")
-
-        error make --unspanned { msg: $text }
-    }
-}

crates/nu-std/tests/test_logger.nu

@@ -1,9 +1,8 @@
 use std *
 
 def run [system_level, message_level] {
-    cd $env.FILE_PWD
     do {
-        nu -c $'use std; NU_LOG_LEVEL=($system_level) std log ($message_level) "test message"' 
+        ^$nu.current-exe -c $'use std; NU_LOG_LEVEL=($system_level) std log ($message_level) "test message"' 
     } | complete | get -i stderr
 }
 def "assert no message" [system_level, message_level] {

toolkit.nu

@@ -56,7 +56,7 @@ export def test [
 
 # run the tests for the standard library
 export def "test stdlib" [] {
-    cargo run -- crates/nu-std/tests/run.nu
+    cargo run -- -c "use std; std run-tests --path crates/nu-std"
 }
 
 # print the pipe input inside backticks, dimmed and italic, as a pretty command