std: refactor test-runner to no longer require tests to be exported (#9355)

# Description
Test runner now performs following actions in order to run tests:
* Module file is opened
* Public function with random name is added to the source code, this
function calls user-specified private function
* Modified module file is saved under random name in $nu.temp-path
* Modified module file is imported in subprocess, injected function is
called by the test runner
# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->
* Test functions no longer need to be exported
* test functions no longer need to reside in separate test_ files
* setup and teardown renamed to before-each and after-each respectively
* before-all and after-all functions added that run before all tests in
given module. This matches the behavior of test runners used by other
languages such as JUnit/TestNG or Mocha
# Tests + Formatting

# After Submitting

---------

Co-authored-by: Kamil <skelly37@protonmail.com>
Co-authored-by: amtoine <stevan.antoine@gmail.com>

crates/nu-std/tests/logger_tests/test_basic_commands.nu

@@ -42,47 +42,47 @@ def "assert message short" [
     assert str contains $output "test message"
 }
 
-export def test_critical [] {
+def test_critical [] {
     assert no message 99 critical
     assert message CRITICAL critical CRT
 }
 
-export def test_critical_short [] {
+def test_critical_short [] {
     assert message short CRITICAL critical C
 }
 
-export def test_error [] {
-    assert no message CRITICAL error 
+def test_error [] {
+    assert no message CRITICAL error
     assert message ERROR error ERR
 }
 
-export def test_error_short [] {
+def test_error_short [] {
     assert message short ERROR error E
 }
 
-export def test_warning [] {
-    assert no message ERROR warning 
+def test_warning [] {
+    assert no message ERROR warning
     assert message WARNING warning WRN
 }
 
-export def test_warning_short [] {
+def test_warning_short [] {
     assert message short WARNING warning W
 }
 
-export def test_info [] {
-    assert no message WARNING info 
+def test_info [] {
+    assert no message WARNING info
     assert message INFO info "INF" # INF has to be quoted, otherwise it is the `inf` float
 }
 
-export def test_info_short [] {
+def test_info_short [] {
     assert message short INFO info I
 }
 
-export def test_debug [] {
-    assert no message INFO debug 
+def test_debug [] {
+    assert no message INFO debug
     assert message DEBUG debug DBG
 }
 
-export def test_debug_short [] {
+def test_debug_short [] {
     assert message short DEBUG debug D
-}
+}

crates/nu-std/tests/logger_tests/test_log_custom.nu

@@ -22,20 +22,20 @@ def run-command [
     }  | complete | get --ignore-errors stderr
 }
 
-export def test_errors_during_deduction [] {
+def test_errors_during_deduction [] {
     assert str contains (run-command "DEBUG" "msg" "%MSG%" 25) "Cannot deduce log level prefix for given log level"
     assert str contains (run-command "DEBUG" "msg" "%MSG%" 25 --ansi (ansi red)) "Cannot deduce log level prefix for given log level"
     assert str contains (run-command "DEBUG" "msg" "%MSG%" 25 --level-prefix "abc") "Cannot deduce ansi for given log level"
 }
 
-export def test_valid_calls [] {
+def test_valid_calls [] {
     assert equal (run-command "DEBUG" "msg" "%MSG%" 25 --level-prefix "abc" --ansi (ansi default) | str trim --right) "msg"
     assert equal (run-command "DEBUG" "msg" "%LEVEL% %MSG%" 20 | str trim --right) $"($env.LOG_PREFIX.INFO) msg"
     assert equal (run-command "DEBUG" "msg" "%LEVEL% %MSG%" --level-prefix "abc" 20 | str trim --right) "abc msg"
     assert equal (run-command "INFO" "msg" "%ANSI_START%%LEVEL% %MSG%%ANSI_STOP%" $env.LOG_LEVEL.CRITICAL | str trim --right) $"($env.LOG_ANSI.CRITICAL)CRT msg(ansi reset)"
 }
 
-export def test_log_level_handling [] {
+def test_log_level_handling [] {
     assert equal (run-command "DEBUG" "msg" "%LEVEL% %MSG%" 20 | str trim --right) $"($env.LOG_PREFIX.INFO) msg"
     assert equal (run-command "WARNING" "msg" "%LEVEL% %MSG%" 20 | str trim --right) ""
-}
+}

crates/nu-std/tests/logger_tests/test_log_format_flag.nu

@@ -39,7 +39,7 @@ def "assert formatted" [
     assert equal ($output | str trim --right) (format-message $message $format $prefix $ansi)
 }
 
-export def "test_format_flag" [] {
+def "test_format_flag" [] {
     assert formatted "test" "25 %MSG% %ANSI_START% %LEVEL%%ANSI_STOP%" critical
     assert formatted "test" "25 %MSG% %ANSI_START% %LEVEL%%ANSI_STOP%" error
     assert formatted "test" "25 %MSG% %ANSI_START% %LEVEL%%ANSI_STOP%" warning
@@ -50,4 +50,4 @@ export def "test_format_flag" [] {
     assert formatted --short "test" "TEST %ANSI_START% %MSG%%ANSI_STOP%" warning
     assert formatted --short "test" "TEST %ANSI_START% %MSG%%ANSI_STOP%" info
     assert formatted --short "test" "TEST %ANSI_START% %MSG%%ANSI_STOP%" debug
-}
+}

crates/nu-std/tests/logger_tests/test_logger_env.nu

@@ -1,6 +1,6 @@
 use std *
 
-export def "test_env_log_ansi" [] {
+def test_env_log_ansi [] {
     assert equal $env.LOG_ANSI.CRITICAL (ansi red_bold)
     assert equal $env.LOG_ANSI.ERROR (ansi red)
     assert equal $env.LOG_ANSI.WARNING (ansi yellow)
@@ -8,7 +8,7 @@ export def "test_env_log_ansi" [] {
     assert equal $env.LOG_ANSI.DEBUG (ansi default_dimmed)
 }
 
-export def "test_env_log_level" [] {
+def test_env_log_level [] {
     assert equal $env.LOG_LEVEL.CRITICAL 50
     assert equal $env.LOG_LEVEL.ERROR 40
     assert equal $env.LOG_LEVEL.WARNING 30
@@ -16,7 +16,7 @@ export def "test_env_log_level" [] {
     assert equal $env.LOG_LEVEL.DEBUG 10
 }
 
-export def "test_env_log_prefix" [] {
+def test_env_log_prefix [] {
     assert equal $env.LOG_PREFIX.CRITICAL "CRT"
     assert equal $env.LOG_PREFIX.ERROR "ERR"
     assert equal $env.LOG_PREFIX.WARNING "WRN"
@@ -24,7 +24,7 @@ export def "test_env_log_prefix" [] {
     assert equal $env.LOG_PREFIX.DEBUG "DBG"
 }
 
-export def "test_env_log_short_prefix" [] {
+def test_env_log_short_prefix [] {
     assert equal $env.LOG_SHORT_PREFIX.CRITICAL "C"
     assert equal $env.LOG_SHORT_PREFIX.ERROR "E"
     assert equal $env.LOG_SHORT_PREFIX.WARNING "W"
@@ -32,6 +32,6 @@ export def "test_env_log_short_prefix" [] {
     assert equal $env.LOG_SHORT_PREFIX.DEBUG "D"
 }
 
-export def "test_env_log_format" [] {
+def test_env_log_format [] {
     assert equal $env.LOG_FORMAT $"%ANSI_START%%DATE%|%LEVEL%|(ansi u)%MSG%%ANSI_STOP%"
-}
+}

crates/nu-std/tests/test_asserts.nu

@@ -1,34 +1,34 @@
 use std *
 
-export def test_assert [] {
+def test_assert [] {
     assert true
     assert (1 + 2 == 3)
     assert error { assert false }
     assert error { assert (1 + 2 == 4) }
 }
 
-export def test_assert_not [] {
+def test_assert_not [] {
     assert not false
     assert not (1 + 2 == 4)
     assert error { assert not true }
     assert error { assert not (1 + 2 == 3) }
 }
 
-export def test_assert_equal [] {
+def test_assert_equal [] {
     assert equal (1 + 2) 3
     assert equal (0.1 + 0.2 | into string | into decimal) 0.3 # 0.30000000000000004 == 0.3
     assert error { assert equal 1 "foo" }
     assert error { assert equal (1 + 2) 4 }
 }
 
-export def test_assert_not_equal [] {
+def test_assert_not_equal [] {
     assert not equal (1 + 2) 4
     assert not equal 1 "foo"
     assert not equal (1 + 2) "3"
     assert error { assert not equal 1 1 }
 }
 
-export def test_assert_error [] {
+def test_assert_error [] {
     let failing_code = {|| missing_code_to_run}
     assert error $failing_code
 
@@ -37,33 +37,33 @@ export def test_assert_error [] {
     assert $assert_error_raised "The assert error should raise an error if there is no error in the executed code."
 }
 
-export def test_assert_less [] {
+def test_assert_less [] {
     assert less 1 2
     assert error { assert less 1 1 }
 }
 
-export def test_assert_less_or_equal [] {
+def test_assert_less_or_equal [] {
     assert less or equal 1 2
     assert less or equal 1 1
     assert error { assert less or equal 1 0 }
 }
 
-export def test_assert_greater [] {
+def test_assert_greater [] {
     assert greater 2 1
     assert error { assert greater 2 2 }
 }
 
-export def test_assert_greater_or_equal [] {
+def test_assert_greater_or_equal [] {
     assert greater or equal 1 1
     assert greater or equal 2 1
     assert error { assert greater or equal 0 1 }
 }
 
-export def test_assert_length [] {
+def test_assert_length [] {
     assert length [0, 0, 0]  3
     assert error { assert length [0, 0] 3 }
 }
 
-export def test_assert_skip [] {
+def test_assert_skip [] {
     assert skip # This test case is skipped on purpose
 }

crates/nu-std/tests/test_dirs.nu

@@ -3,10 +3,10 @@ use std assert
 use std log
 
 # A couple of nuances to understand when testing module that exports environment:
-# Each 'use' for that module in the test script will execute the export def-env block.
+# Each 'use' for that module in the test script will execute the def-env block.
 # PWD at the time of the `use` will be what the export def-env block will see.
 
-export def setup [] {
+def before-each [] {
     # need some directories to play with
     let base_path = ($nu.temp-path | path join $"test_dirs_(random uuid)")
     let path_a = ($base_path | path join "a")
@@ -17,7 +17,7 @@ export def setup [] {
     {base_path: $base_path, path_a:$path_a, path_b: $path_b}
 }
 
-export def teardown [] {
+def after-each [] {
     let base_path = $in.base_path
     cd $base_path
     cd ..
@@ -34,16 +34,16 @@ def cur_ring_check [expect_dir:string, expect_position: int scenario:string] {
     assert equal $expect_position $env.DIRS_POSITION $"position in ring after ($scenario)"
 }
 
-export def test_dirs_command [] {
+def test_dirs_command [] {
     # careful with order of these statements!
     # must capture value of $in before executing `use`s
     let $c = $in
 
-    # must set PWD *before* doing `use` that will run the export def-env block in dirs module.
+    # must set PWD *before* doing `use` that will run the def-env block in dirs module.
     cd $c.base_path
 
     # must execute these uses for the UOT commands *after* the test and *not* just put them at top of test module.
-    # the export def-env gets messed up
+    # the def-env gets messed up
     use std dirs
 
     assert equal [$c.base_path] $env.DIRS_LIST "list is just pwd after initialization"
@@ -74,10 +74,10 @@ export def test_dirs_command [] {
     assert equal (dirs show) [[active path]; [false $c.base_path] [true $c.path_b]] "show table contains expected information"
 }
 
-export def test_dirs_next [] {
+def test_dirs_next [] {
     # must capture value of $in before executing `use`s
     let $c = $in
-    # must set PWD *before* doing `use` that will run the export def-env block in dirs module.
+    # must set PWD *before* doing `use` that will run the def-env block in dirs module.
     cd $c.base_path
     assert equal $env.PWD $c.base_path "test setup"
 
@@ -95,10 +95,10 @@ export def test_dirs_next [] {
 
 }
 
-export def test_dirs_cd [] {
+def test_dirs_cd [] {
     # must capture value of $in before executing `use`s
     let $c = $in
-    # must set PWD *before* doing `use` that will run the export def-env block in dirs module.
+    # must set PWD *before* doing `use` that will run the def-env block in dirs module.
     cd $c.base_path
 
     use std dirs

crates/nu-std/tests/test_iter.nu

@@ -1,6 +1,6 @@
 use std *
 
-export def test_iter_find [] {
+def test_iter_find [] {
     let hastack1 = [1 2 3 4 5 6 7]
     let hastack2 = [nushell rust shell iter std]
     let hastack3 = [nu 69 2023-04-20 "std"]
@@ -18,7 +18,7 @@ export def test_iter_find [] {
     assert equal $res null
 }
 
-export def test_iter_intersperse [] {
+def test_iter_intersperse [] {
     let res = ([1 2 3 4] | iter intersperse 0)
     assert equal $res [1 0 2 0 3 0 4]
 
@@ -38,7 +38,7 @@ export def test_iter_intersperse [] {
     assert equal $res [4]
 }
 
-export def test_iter_scan [] {
+def test_iter_scan [] {
     let scanned = ([1 2 3] | iter scan 0 {|x, y| $x + $y} -n)
     assert equal $scanned [1, 3, 6]
 
@@ -49,18 +49,18 @@ export def test_iter_scan [] {
     assert equal $scanned ["a" "ab" "abc" "abcd"]
 }
 
-export def test_iter_filter_map [] {
+def test_iter_filter_map [] {
     let res = ([2 5 "4" 7] | iter filter-map {|it| $it ** 2})
     assert equal $res [4 25 49]
 
     let res = (
-        ["3" "42" "69" "n" "x" ""] 
+        ["3" "42" "69" "n" "x" ""]
         | iter filter-map {|it| $it | into int}
         )
     assert equal $res [3 42 69]
 }
 
-export def test_iter_find_index [] {
+def test_iter_find_index [] {
     let res = (
          ["iter", "abc", "shell", "around", "nushell", "std"]
          | iter find-index {|x| $x starts-with 's'}
@@ -75,7 +75,7 @@ export def test_iter_find_index [] {
     assert equal $res 0
 }
 
-export def test_iter_zip_with [] {
+def test_iter_zip_with [] {
     let res = (
         [1 2 3] | iter zip-with [2 3 4] {|a, b| $a + $b }
     )
@@ -98,10 +98,10 @@ export def test_iter_zip_with [] {
         [name    repo    position];
         [rust    github  1]
         [haskell gitlab  2]
-    ] 
+    ]
 }
 
-export def test_iter_flat_map [] {
+def test_iter_flat_map [] {
     let res = (
         [[1 2 3] [2 3 4] [5 6 7]] | iter flat-map {|it| $it | math sum}
     )

crates/nu-std/tests/test_setup_teardown.nu

@@ -1,30 +1,30 @@
 use std log
 use std assert
 
-export def setup [] {
+def before-each [] {
     log debug "Setup is running"
     {msg: "This is the context"}
 }
 
-export def teardown [] {
+def after-each [] {
     log debug $"Teardown is running. Context: ($in)"
 }
 
-export def test_assert_pass [] {
+def test_assert_pass [] {
     log debug $"Assert is running. Context: ($in)"
 }
 
-export def test_assert_skip [] {
+def test_assert_skip [] {
     log debug $"Assert is running. Context: ($in)"
     assert skip
 }
 
-export def test_assert_fail_skipped_by_default [] {
+def test_assert_fail_skipped_by_default [] {
     assert skip # Comment this line if you want to see what happens if a test fails
     log debug $"Assert is running. Context: ($in)"
     assert false
 }
 
-export def unrelated [] {
+def unrelated [] {
     log error "This should not run"
 }

crates/nu-std/tests/test_std.nu

@@ -1,13 +1,13 @@
 use std
 
-export def test_path_add [] {
+def test_path_add [] {
     use std assert
 
     let path_name = if "PATH" in $env { "PATH" } else { "Path" }
 
     with-env [$path_name []] {
         def get_path [] { $env | get $path_name }
-    
+
         assert equal (get_path) []
 
         std path add "/foo/"
@@ -27,14 +27,14 @@ export def test_path_add [] {
 
         let-env $path_name = []
         let target_paths = {linux: "foo", windows: "bar", macos: "baz"}
-        
+
         std path add $target_paths
         assert equal (get_path) [($target_paths | get $nu.os-info.name)]
 
-        
+
     }
 }
 
-export def test_banner [] {
+def test_banner [] {
     std assert ((std banner | lines | length) == 15)
 }

crates/nu-std/tests/test_xml.nu

@@ -3,7 +3,7 @@ use std xml xupdate
 use std xml xinsert
 use std assert
 
-export def setup [] {
+def before-each [] {
     {sample_xml: ('
         <a>
             <b>
@@ -18,7 +18,7 @@ export def setup [] {
     }
 }
 
-export def test_xml_xaccess [] {
+def test_xml_xaccess [] {
     let sample_xml = $in.sample_xml
 
     assert equal ($sample_xml | xaccess [a]) [$sample_xml]
@@ -28,7 +28,7 @@ export def test_xml_xaccess [] {
     assert equal ($sample_xml | xaccess [* * * {|e| $e.attributes != {}}]) [[tag, attributes, content]; [c, {a: b}, []]]
 }
 
-export def test_xml_xupdate [] {
+def test_xml_xupdate [] {
     let sample_xml = $in.sample_xml
 
     assert equal ($sample_xml | xupdate [*] {|x| $x | update attributes {i: j}}) ('<a i="j"><b><c a="b"></c></b><c></c><d><e>z</e><e>x</e></d></a>' | from xml)
@@ -36,7 +36,7 @@ export def test_xml_xupdate [] {
     assert equal ($sample_xml | xupdate [* * * {|e| $e.attributes != {}}] {|x| $x | update content ['xml']}) {tag: a, attributes: {}, content: [[tag, attributes, content]; [b, {}, [[tag, attributes, content]; [c, {a: b}, [xml]]]], [c, {}, []], [d, {}, [[tag, attributes, content]; [e, {}, [[tag, attributes, content]; [null, null, z]]], [e, {}, [[tag, attributes, content]; [null, null, x]]]]]]}
 }
 
-export def test_xml_xinsert [] {
+def test_xml_xinsert [] {
     let sample_xml = $in.sample_xml
 
     assert equal ($sample_xml | xinsert [a] {tag: b attributes:{} content: []}) ('<a><b><c a="b"></c></b><c></c><d><e>z</e><e>x</e></d><b></b></a>' | from xml)