Implement annotations support in test runner (#9406)

<!--
if this PR closes one or more issues, you can automatically link the PR
with
them by using one of the [*linking
keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword),
e.g.
- this PR should close #xxxx
- fixes #xxxx

you can also mention related issues, PRs or discussions!
-->

# Description
Test runner now uses annotations instead of magic function names to pick
up code to run. Additionally skipping tests is now done on annotation
level so skipping and unskipping a test no longer requires changes to
the test code

In order for a function to be picked up by the test runner it needs to
meet following criteria:
* Needs to be private (all exported functions are ignored)
* Needs to contain one of valid annotations (and only the annotation)
directly above the definition, all other comments are ignored

Following are considered valid annotations:
* \# test
* \# test-skip
* \# before-all
* \# before-each
* \# after-each
* \# after-all

# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A
clippy::needless_collect -A clippy::result_large_err` to check that
you're using the standard code style
- `cargo test --workspace` to check that all tests pass
- `cargo run -- crates/nu-std/tests/run.nu` to run the tests for the
standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->

crates/nu-std/std/assert.nu

@@ -120,16 +120,6 @@ export def error [
     }
 }
 
-# Skip the current test case
-#
-# # Examples
-#
-# if $condition { assert skip }
-export def skip [] {
-    error make {msg: "ASSERT:SKIP"}
-}
-
-
 # Assert $left == $right
 #
 # For more documentation see the assert command

crates/nu-std/std/testing.nu

@@ -1,6 +1,113 @@
 use log.nu
 
 
+# Here we store the map of annotations internal names and the annotation actually used during test creation
+# The reason we do that is to allow annotations to be easily renamed without modifying rest of the code
+# Functions with no annotations or with annotations not on the list are rejected during module evaluation
+# test and test-skip annotations may be used multiple times throughout the module as the function names are stored in a list
+# Other annotations should only be used once within a module file
+# If you find yourself in need of multiple before- or after- functions it's a sign your test suite probably needs redesign
+def valid-annotations [] {
+    {
+        "#[test]": "test",
+        "#[ignore]": "test-skip",
+        "#[before-each]": "before-each"
+        "#[before-all]": "before-all"
+        "#[after-each]": "after-each"
+        "#[after-all]": "after-all"
+    }
+}
+
+# Returns a table containing the list of function names together with their annotations (comments above the declaration)
+#
+# The way this works is we first generate an ast for a file and then extract all tokens containing keyword def with internalcall shape
+# Then for each token we extract the next token (the actual function name) and the previous token (usually closing bracket of previous function)
+# We then open the file as raw text and extract the last line of substring starting from the span end of previous token up to span start of the def token which contains the annotation and surrounding whitespaces
+# We take the last line only in order to not pick up unrelated comments from the source code
+def get-annotated [
+    file: path
+] {
+    let raw_file = (open $file)
+
+    let ast = (
+        ^$nu.current-exe --ide-ast $file
+        | from json
+        | enumerate
+        | flatten
+    )
+
+    $ast
+    | where content == def and shape == shape_internalcall
+    | insert function_name {|x|
+        $ast
+        | get ($x.index + 1)
+        | get content
+    }
+    | insert annotation {|x|
+        # index == 0 means the function is defined on the first lines of the file in which case there will be no preceding tokens
+        if $x.index > 0 {
+            let preceding_span = (
+                $ast
+                | get ($x.index - 1)
+                | get span
+            )
+            $raw_file
+            | str substring $preceding_span.end..$x.span.start
+            | lines
+            | where $it != ''
+            | last
+            | str trim
+            | str join (char nl)
+        } else {
+            ''
+        }
+    }
+    | select function_name annotation
+
+}
+
+# Takes table of function names and their annotations such as the one returned by get-annotated
+#
+# Returns a record where keys are internal names of valid annotations and values are corresponding function names
+# Annotations that allow multiple functions are of type list<string>
+# Other annotations are of type string
+# Result gets merged with the template record so that the output shape remains consistent regardless of the table content
+def create-test-record [] {
+    let input = $in
+
+    let template_record = {
+        before-each: '',
+        before-all: '',
+        after-each: '',
+        after-all: '',
+        test-skip: []
+    }
+
+    let test_record = (
+        $input
+        | where annotation in (valid-annotations|columns)
+        | update annotation {|x|
+            valid-annotations
+            | get $x.annotation
+        }
+        | group-by annotation
+        | transpose key value
+        | update value {|x|
+            $x.value.function_name
+            | if $x.key in ["test", "test-skip"] {
+                $in
+            } else {
+                get 0
+            }
+        }
+        | transpose -i -r -d
+    )
+
+    $template_record
+    | merge $test_record
+
+}
+
 def throw-error [error: record] {
     error make {
         msg: $"(ansi red)($error.msg)(ansi reset)"
@@ -41,17 +148,16 @@ def show-pretty-test [indent: int = 4] {
     ] | str join
 }
 
-def get-commands [
-    file: path
-] {
-    ^$nu.current-exe --ide-ast $file
-    | from json
-    | get content
-    | split list def
-    | skip 1
-    | each {get 0}
-}
-
+# Takes a test record and returns the execution result
+# Test is executed via following steps:
+# * Public function with random name is generated that runs specified test in try/catch block
+# * Module file is opened
+# * Random public function is appended to the end of the file
+# * Modified file is saved under random name
+# * Nu subprocess is spawned
+# * Inside subprocess the modified file is imported and random function called
+# * Output of the random function is serialized into nuon and returned to parent process
+# * Modified file is removed
 def run-test [
     test: record
 ] {
@@ -67,11 +173,7 @@ export def ($test_function_name) [] {
         ($test.after-each)
     } catch { |err|
         ($test.after-each)
-        if $err.msg == "ASSERT:SKIP" {
-            exit 2
-        } else {
-            $err | get raw
-        }
+        $err | get raw
     }
 }
 "
@@ -84,7 +186,6 @@ export def ($test_function_name) [] {
     let result = (
         ^$nu.current-exe -c $"use ($rendered_module_path) *; ($test_function_name)|to nuon"
         | complete
-
     )
 
     rm $rendered_module_path
@@ -92,16 +193,23 @@ export def ($test_function_name) [] {
     return $result
 }
 
+
+# Takes a module record and returns a table with following columns:
+#
+# * file   - path to file under test
+# * name   - name of the module under test
+# * test   - name of specific test
+# * result - test execution result
 def run-tests-for-module [
-    module: record
+    module: record<file: path name: string before-each: string after-each: string before-all: string after-all: string test: list test-skip: list>
 ] {
-    let global_context = if $module.before-all {
+    let global_context = if not ($module.before-all|is-empty) {
             log info $"Running before-all for module ($module.name)"
             run-test {
                 file: $module.file,
                 before-each: 'let context = {}',
                 after-each: '',
-                test: 'before-all'
+                test: $module.before-all
             }
             | if $in.exit_code == 0 {
                 $in.stdout
@@ -116,56 +224,86 @@ def run-tests-for-module [
             {}
     }
 
+    # since tests are skipped based on their annotation and never actually executed we can generate their list in advance
+    let skipped_tests = (
+        if not ($module.test-skip|is-empty) {
+            $module
+            | update test $module.test-skip
+            | reject test-skip
+            | flatten
+            | insert result 'skip'
+        } else {
+            []
+        }
+    )
+
     let tests = (
         $module
-        | flatten
-        | rename -c [tests test]
+        | reject test-skip
+        | flatten test
         | update before-each {|x|
-            if $module.before-each {
-                $"let context = \(($global_context)|merge \(before-each\)\)"
+            if not ($module.before-each|is-empty) {
+                $"let context = \(($global_context)|merge \(($module.before-each)\)\)"
             } else {
                 $"let context = ($global_context)"
             }
         }
         | update after-each {|x|
-            if $module.after-each {
-                '$context | after-each'
+            if not ($module.after-each|is-empty) {
+                $"$context | ($module.after-each)"
             } else {
                 ''
             }
         }
         | each {|test|
-            log info $"Running ($test.test) in module ($module.name) with context ($global_context)"
+            log info $"Running ($test.test) in module ($module.name)"
+            log debug $"Global context is ($global_context)"
+
             $test|insert result {|x|
                 run-test $test
-                | match $in.exit_code {
-                    0 => "pass",
-                    2 => "skip",
-                    _ => "fail",
+                | if $in.exit_code == 0 {
+                    'pass'
+                } else {
+                    'fail'
                 }
             }
         }
+        | append $skipped_tests
+        | select file name test result
     )
 
-    if $module.after-all {
+    if not ($module.after-all|is-empty) {
         log info $"Running after-all for module ($module.name)"
 
         run-test {
                 file: $module.file,
                 before-each: $"let context = ($global_context)",
                 after-each: '',
-                test: 'after-all'
+                test: $module.after-all
         }
     }
     return $tests
 }
 
+# Run tests for nushell code
+#
+# By default all detected tests are executed
+# Test list can be filtered out by specifying either path to search for, name of the module to run tests for or specific test name
+# In order for a function to be recognized as a test by the test runner it needs to be annotated with # test
+# Following annotations are supported by the test runner:
+# * test        - test case to be executed during test run
+# * test-skip   - test case to be skipped during test run
+# * before-all  - function to run at the beginning of test run. Returns a global context record that is piped into every test function
+# * before-each - function to run before every test case. Returns a per-test context record that is merged with global context and piped into test functions
+# * after-each  - function to run after every test case. Receives the context record just like the test cases
+# * after-all   - function to run after all test cases have been executed. Receives the global context record
 export def run-tests [
-    --path: path, # Path to look for tests. Default: current directory.
-    --module: string, # Test module to run. Default: all test modules found.
-    --test: string, # Individual test to run. Default: all test command found in the files.
-    --list, # list the selected tests without running them.
+    --path: path,      # Path to look for tests. Default: current directory.
+    --module: string,  # Test module to run. Default: all test modules found.
+    --test: string,    # Individual test to run. Default: all test command found in the files.
+    --list,            # list the selected tests without running them.
 ] {
+
     let module_search_pattern = ('**' | path join ({
         stem: ($module | default "*")
         extension: nu
@@ -197,25 +335,24 @@ export def run-tests [
     let modules = (
         ls ($path | path join $module_search_pattern)
         | each {|row| {file: $row.name name: ($row.name | path parse | get stem)}}
-        | upsert commands {|module|
-            get-commands $module.file
+        | insert commands {|module|
+            get-annotated $module.file
         }
-        | upsert tests {|module| $module.commands|where $it starts-with "test_"}
-        | filter {|x| ($x.tests|length) > 0}
-        | filter {|x| if ($test|is-empty) {true} else {$test in $x.tests}}
+        | filter {|x| ($x.commands|length) > 0 and ($x.commands.annotation|any {|y| $y in (valid-annotations|columns)})} # if file has no private functions at all or no functions annotated with a valid annotation we drop it
+        | upsert commands {|module|
+            $module.commands
+            | create-test-record
+        }
+        | flatten
+        | filter {|x| ($x.test|length) > 0}
+        | filter {|x| if ($test|is-empty) {true} else {$test in $x.test}}
         | filter {|x| if ($module|is-empty) {true} else {$module == $x.name}}
-        | upsert before-each {|module| "before-each" in $module.commands}
-        | upsert before-all {|module| "before-all" in $module.commands}
-        | upsert after-each {|module| "after-each" in $module.commands}
-        | upsert after-all {|module| "after-all" in $module.commands}
-        | reject commands
-        | rename file name tests
-        | update tests {|x|
-            if ($test|is-empty) {
-                $x.tests
+        | update test {|x|
+            $x.test
+            | if ($test|is-empty) {
+                $in
             } else {
-                $x.tests
-                | where $it == $test
+                where $it == $test
             }
         }
     )
@@ -233,9 +370,8 @@ export def run-tests [
             run-tests-for-module $module
         }
         | flatten
-        | select name test result
     )
-    if not ($results | where result == "fail" | is-empty) {
+    if ($results | any {|x| $x.result == fail}) {
         let text = ([
             $"(ansi purple)some tests did not pass (char lparen)see complete errors below(char rparen):(ansi reset)"
             ""
@@ -245,5 +381,4 @@ export def run-tests [
 
         error make --unspanned { msg: $text }
     }
-
 }

crates/nu-std/tests/logger_tests/test_basic_commands.nu

@@ -42,47 +42,57 @@ def "assert message short" [
     assert str contains $output "test message"
 }
 
-def test_critical [] {
+#[test]
+def critical [] {
     assert no message 99 critical
     assert message CRITICAL critical CRT
 }
 
-def test_critical_short [] {
+#[test]
+def critical_short [] {
     assert message short CRITICAL critical C
 }
 
-def test_error [] {
+#[test]
+def error [] {
     assert no message CRITICAL error
     assert message ERROR error ERR
 }
 
-def test_error_short [] {
+#[test]
+def error_short [] {
     assert message short ERROR error E
 }
 
-def test_warning [] {
+#[test]
+def warning [] {
     assert no message ERROR warning
     assert message WARNING warning WRN
 }
 
-def test_warning_short [] {
+#[test]
+def warning_short [] {
     assert message short WARNING warning W
 }
 
-def test_info [] {
+#[test]
+def info [] {
     assert no message WARNING info
     assert message INFO info "INF" # INF has to be quoted, otherwise it is the `inf` float
 }
 
-def test_info_short [] {
+#[test]
+def info_short [] {
     assert message short INFO info I
 }
 
-def test_debug [] {
+#[test]
+def debug [] {
     assert no message INFO debug
     assert message DEBUG debug DBG
 }
 
-def test_debug_short [] {
+#[test]
+def debug_short [] {
     assert message short DEBUG debug D
 }

crates/nu-std/tests/logger_tests/test_log_custom.nu

@@ -22,20 +22,23 @@ def run-command [
     }  | complete | get --ignore-errors stderr
 }
 
-def test_errors_during_deduction [] {
+#[test]
+def errors_during_deduction [] {
     assert str contains (run-command "DEBUG" "msg" "%MSG%" 25) "Cannot deduce log level prefix for given log level"
     assert str contains (run-command "DEBUG" "msg" "%MSG%" 25 --ansi (ansi red)) "Cannot deduce log level prefix for given log level"
     assert str contains (run-command "DEBUG" "msg" "%MSG%" 25 --level-prefix "abc") "Cannot deduce ansi for given log level"
 }
 
-def test_valid_calls [] {
+#[test]
+def valid_calls [] {
     assert equal (run-command "DEBUG" "msg" "%MSG%" 25 --level-prefix "abc" --ansi (ansi default) | str trim --right) "msg"
     assert equal (run-command "DEBUG" "msg" "%LEVEL% %MSG%" 20 | str trim --right) $"($env.LOG_PREFIX.INFO) msg"
     assert equal (run-command "DEBUG" "msg" "%LEVEL% %MSG%" --level-prefix "abc" 20 | str trim --right) "abc msg"
     assert equal (run-command "INFO" "msg" "%ANSI_START%%LEVEL% %MSG%%ANSI_STOP%" $env.LOG_LEVEL.CRITICAL | str trim --right) $"($env.LOG_ANSI.CRITICAL)CRT msg(ansi reset)"
 }
 
-def test_log_level_handling [] {
+#[test]
+def log_level_handling [] {
     assert equal (run-command "DEBUG" "msg" "%LEVEL% %MSG%" 20 | str trim --right) $"($env.LOG_PREFIX.INFO) msg"
     assert equal (run-command "WARNING" "msg" "%LEVEL% %MSG%" 20 | str trim --right) ""
 }

crates/nu-std/tests/logger_tests/test_log_format_flag.nu

@@ -39,7 +39,8 @@ def "assert formatted" [
     assert equal ($output | str trim --right) (format-message $message $format $prefix $ansi)
 }
 
-def "test_format_flag" [] {
+#[test]
+def format_flag [] {
     assert formatted "test" "25 %MSG% %ANSI_START% %LEVEL%%ANSI_STOP%" critical
     assert formatted "test" "25 %MSG% %ANSI_START% %LEVEL%%ANSI_STOP%" error
     assert formatted "test" "25 %MSG% %ANSI_START% %LEVEL%%ANSI_STOP%" warning

crates/nu-std/tests/logger_tests/test_logger_env.nu

@@ -1,6 +1,7 @@
 use std *
 
-def test_env_log_ansi [] {
+#[test]
+def env_log_ansi [] {
     assert equal $env.LOG_ANSI.CRITICAL (ansi red_bold)
     assert equal $env.LOG_ANSI.ERROR (ansi red)
     assert equal $env.LOG_ANSI.WARNING (ansi yellow)
@@ -8,7 +9,8 @@ def test_env_log_ansi [] {
     assert equal $env.LOG_ANSI.DEBUG (ansi default_dimmed)
 }
 
-def test_env_log_level [] {
+#[test]
+def env_log_level [] {
     assert equal $env.LOG_LEVEL.CRITICAL 50
     assert equal $env.LOG_LEVEL.ERROR 40
     assert equal $env.LOG_LEVEL.WARNING 30
@@ -16,7 +18,8 @@ def test_env_log_level [] {
     assert equal $env.LOG_LEVEL.DEBUG 10
 }
 
-def test_env_log_prefix [] {
+#[test]
+def env_log_prefix [] {
     assert equal $env.LOG_PREFIX.CRITICAL "CRT"
     assert equal $env.LOG_PREFIX.ERROR "ERR"
     assert equal $env.LOG_PREFIX.WARNING "WRN"
@@ -24,7 +27,8 @@ def test_env_log_prefix [] {
     assert equal $env.LOG_PREFIX.DEBUG "DBG"
 }
 
-def test_env_log_short_prefix [] {
+#[test]
+def env_log_short_prefix [] {
     assert equal $env.LOG_SHORT_PREFIX.CRITICAL "C"
     assert equal $env.LOG_SHORT_PREFIX.ERROR "E"
     assert equal $env.LOG_SHORT_PREFIX.WARNING "W"
@@ -32,6 +36,7 @@ def test_env_log_short_prefix [] {
     assert equal $env.LOG_SHORT_PREFIX.DEBUG "D"
 }
 
-def test_env_log_format [] {
+#[test]
+def env_log_format [] {
     assert equal $env.LOG_FORMAT $"%ANSI_START%%DATE%|%LEVEL%|(ansi u)%MSG%%ANSI_STOP%"
 }

crates/nu-std/tests/test_asserts.nu

@@ -1,34 +1,39 @@
 use std *
 
-def test_assert [] {
+#[test]
+def assert_basic [] {
     assert true
     assert (1 + 2 == 3)
     assert error { assert false }
     assert error { assert (1 + 2 == 4) }
 }
 
-def test_assert_not [] {
+#[test]
+def assert_not [] {
     assert not false
     assert not (1 + 2 == 4)
     assert error { assert not true }
     assert error { assert not (1 + 2 == 3) }
 }
 
-def test_assert_equal [] {
+#[test]
+def assert_equal [] {
     assert equal (1 + 2) 3
     assert equal (0.1 + 0.2 | into string | into decimal) 0.3 # 0.30000000000000004 == 0.3
     assert error { assert equal 1 "foo" }
     assert error { assert equal (1 + 2) 4 }
 }
 
-def test_assert_not_equal [] {
+#[test]
+def assert_not_equal [] {
     assert not equal (1 + 2) 4
     assert not equal 1 "foo"
     assert not equal (1 + 2) "3"
     assert error { assert not equal 1 1 }
 }
 
-def test_assert_error [] {
+#[test]
+def assert_error [] {
     let failing_code = {|| missing_code_to_run}
     assert error $failing_code
 
@@ -37,33 +42,39 @@ def test_assert_error [] {
     assert $assert_error_raised "The assert error should raise an error if there is no error in the executed code."
 }
 
-def test_assert_less [] {
+#[test]
+def assert_less [] {
     assert less 1 2
     assert error { assert less 1 1 }
 }
 
-def test_assert_less_or_equal [] {
+#[test]
+def assert_less_or_equal [] {
     assert less or equal 1 2
     assert less or equal 1 1
     assert error { assert less or equal 1 0 }
 }
 
-def test_assert_greater [] {
+#[test]
+def assert_greater [] {
     assert greater 2 1
     assert error { assert greater 2 2 }
 }
 
-def test_assert_greater_or_equal [] {
+#[test]
+def assert_greater_or_equal [] {
     assert greater or equal 1 1
     assert greater or equal 2 1
     assert error { assert greater or equal 0 1 }
 }
 
-def test_assert_length [] {
+#[test]
+def assert_length [] {
     assert length [0, 0, 0]  3
     assert error { assert length [0, 0] 3 }
 }
 
-def test_assert_skip [] {
-    assert skip # This test case is skipped on purpose
+#[ignore]
+def assert_skip [] {
+    assert true # This test case is skipped on purpose
 }

crates/nu-std/tests/test_dirs.nu

@@ -6,6 +6,7 @@ use std log
 # Each 'use' for that module in the test script will execute the def-env block.
 # PWD at the time of the `use` will be what the export def-env block will see.
 
+#[before-each]
 def before-each [] {
     # need some directories to play with
     let base_path = ($nu.temp-path | path join $"test_dirs_(random uuid)")
@@ -17,6 +18,7 @@ def before-each [] {
     {base_path: $base_path, path_a: $path_a, path_b: $path_b}
 }
 
+#[after-each]
 def after-each [] {
     let base_path = $in.base_path
     cd $base_path
@@ -28,13 +30,15 @@ def cur_dir_check [expect_dir, scenario] {
     log debug $"check dir ($expect_dir), scenario ($scenario)"
     assert equal $expect_dir $env.PWD $"expected not PWD after ($scenario)"
 }
+
 def cur_ring_check [expect_dir:string, expect_position: int scenario:string] {
     log debug $"check ring ($expect_dir), position ($expect_position) scenario ($scenario)"
     assert ($expect_position < ($env.DIRS_LIST | length)) $"ring big enough after ($scenario)"
     assert equal $expect_position $env.DIRS_POSITION $"position in ring after ($scenario)"
 }
 
-def test_dirs_command [] {
+#[test]
+def dirs_command [] {
     # careful with order of these statements!
     # must capture value of $in before executing `use`s
     let $c = $in
@@ -84,7 +88,8 @@ def test_dirs_command [] {
     assert equal $env.PWD $c.base_path "drop changes PWD (regression test for #9449)"
 }
 
-def test_dirs_next [] {
+#[test]
+def dirs_next [] {
     # must capture value of $in before executing `use`s
     let $c = $in
     # must set PWD *before* doing `use` that will run the def-env block in dirs module.
@@ -105,7 +110,8 @@ def test_dirs_next [] {
 
 }
 
-def test_dirs_cd [] {
+#[test]
+def dirs_cd [] {
     # must capture value of $in before executing `use`s
     let $c = $in
     # must set PWD *before* doing `use` that will run the def-env block in dirs module.

crates/nu-std/tests/test_iter.nu

@@ -1,6 +1,7 @@
 use std *
 
-def test_iter_find [] {
+#[test]
+def iter_find [] {
     let hastack1 = [1 2 3 4 5 6 7]
     let hastack2 = [nushell rust shell iter std]
     let hastack3 = [nu 69 2023-04-20 "std"]
@@ -18,7 +19,8 @@ def test_iter_find [] {
     assert equal $res null
 }
 
-def test_iter_intersperse [] {
+#[test]
+def iter_intersperse [] {
     let res = ([1 2 3 4] | iter intersperse 0)
     assert equal $res [1 0 2 0 3 0 4]
 
@@ -38,7 +40,8 @@ def test_iter_intersperse [] {
     assert equal $res [4]
 }
 
-def test_iter_scan [] {
+#[test]
+def iter_scan [] {
     let scanned = ([1 2 3] | iter scan 0 {|x, y| $x + $y} -n)
     assert equal $scanned [1, 3, 6]
 
@@ -49,7 +52,8 @@ def test_iter_scan [] {
     assert equal $scanned ["a" "ab" "abc" "abcd"]
 }
 
-def test_iter_filter_map [] {
+#[test]
+def iter_filter_map [] {
     let res = ([2 5 "4" 7] | iter filter-map {|it| $it ** 2})
     assert equal $res [4 25 49]
 
@@ -60,7 +64,8 @@ def test_iter_filter_map [] {
     assert equal $res [3 42 69]
 }
 
-def test_iter_find_index [] {
+#[test]
+def iter_find_index [] {
     let res = (
          ["iter", "abc", "shell", "around", "nushell", "std"]
          | iter find-index {|x| $x starts-with 's'}
@@ -75,7 +80,8 @@ def test_iter_find_index [] {
     assert equal $res 0
 }
 
-def test_iter_zip_with [] {
+#[test]
+def iter_zip_with [] {
     let res = (
         [1 2 3] | iter zip-with [2 3 4] {|a, b| $a + $b }
     )
@@ -101,7 +107,8 @@ def test_iter_zip_with [] {
     ]
 }
 
-def test_iter_flat_map [] {
+#[test]
+def iter_flat_map [] {
     let res = (
         [[1 2 3] [2 3 4] [5 6 7]] | iter flat-map {|it| $it | math sum}
     )
@@ -111,7 +118,8 @@ def test_iter_flat_map [] {
     assert equal $res [11 22 33]
 }
 
-export def test_iter_zip_into_record [] {
+#[test]
+def iter_zip_into_record [] {
     let headers = [name repo position]
     let values = [rust github 1]
 

crates/nu-std/tests/test_setup_teardown.nu

@@ -1,26 +1,30 @@
 use std log
 use std assert
 
+#[before-each]
 def before-each [] {
     log debug "Setup is running"
     {msg: "This is the context"}
 }
 
+#[after-each]
 def after-each [] {
     log debug $"Teardown is running. Context: ($in)"
 }
 
-def test_assert_pass [] {
+#[test]
+def assert_pass [] {
     log debug $"Assert is running. Context: ($in)"
 }
 
-def test_assert_skip [] {
+#[ignore]
+def assert_skip [] {
     log debug $"Assert is running. Context: ($in)"
-    assert skip
 }
 
-def test_assert_fail_skipped_by_default [] {
-    assert skip # Comment this line if you want to see what happens if a test fails
+#[ignore]
+def assert_fail_skipped_by_default [] {
+    # Change test-skip to test if you want to see what happens if a test fails
     log debug $"Assert is running. Context: ($in)"
     assert false
 }

crates/nu-std/tests/test_std.nu

@@ -1,6 +1,7 @@
 use std
 
-def test_path_add [] {
+#[test]
+def path_add [] {
     use std assert
 
     let path_name = if "PATH" in $env { "PATH" } else { "Path" }
@@ -33,6 +34,7 @@ def test_path_add [] {
     }
 }
 
-def test_banner [] {
+#[test]
+def banner [] {
     std assert ((std banner | lines | length) == 15)
 }

crates/nu-std/tests/test_xml.nu

@@ -3,6 +3,7 @@ use std xml xupdate
 use std xml xinsert
 use std assert
 
+#[before-each]
 def before-each [] {
     {sample_xml: ('
         <a>
@@ -18,7 +19,8 @@ def before-each [] {
     }
 }
 
-def test_xml_xaccess [] {
+#[test]
+def xml_xaccess [] {
     let sample_xml = $in.sample_xml
 
     assert equal ($sample_xml | xaccess [a]) [$sample_xml]
@@ -28,7 +30,8 @@ def test_xml_xaccess [] {
     assert equal ($sample_xml | xaccess [* * * {|e| $e.attributes != {}}]) [[tag, attributes, content]; [c, {a: b}, []]]
 }
 
-def test_xml_xupdate [] {
+#[test]
+def xml_xupdate [] {
     let sample_xml = $in.sample_xml
 
     assert equal ($sample_xml | xupdate [*] {|x| $x | update attributes {i: j}}) ('<a i="j"><b><c a="b"></c></b><c></c><d><e>z</e><e>x</e></d></a>' | from xml)
@@ -36,7 +39,8 @@ def test_xml_xupdate [] {
     assert equal ($sample_xml | xupdate [* * * {|e| $e.attributes != {}}] {|x| $x | update content ['xml']}) {tag: a, attributes: {}, content: [[tag, attributes, content]; [b, {}, [[tag, attributes, content]; [c, {a: b}, [xml]]]], [c, {}, []], [d, {}, [[tag, attributes, content]; [e, {}, [[tag, attributes, content]; [null, null, z]]], [e, {}, [[tag, attributes, content]; [null, null, x]]]]]]}
 }
 
-def test_xml_xinsert [] {
+#[test]
+def xml_xinsert [] {
     let sample_xml = $in.sample_xml
 
     assert equal ($sample_xml | xinsert [a] {tag: b attributes:{} content: []}) ('<a><b><c a="b"></c></b><c></c><d><e>z</e><e>x</e></d><b></b></a>' | from xml)

toolkit.nu

@@ -87,8 +87,10 @@ export def test [
 }
 
 # run the tests for the standard library
-export def "test stdlib" [] {
-    cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"
+export def "test stdlib" [
+    --extra-args: string = ''
+] {
+    cargo run -- -c $"use std testing; testing run-tests --path crates/nu-std ($extra_args)"
 }
 
 # print the pipe input inside backticks, dimmed and italic, as a pretty command