From 440a0e960a37719cd05c5080f81dc80b4d56d8b7 Mon Sep 17 00:00:00 2001
From: Yethal <26117918+Yethal@users.noreply.github.com>
Date: Fri, 7 Jul 2023 09:20:36 +0200
Subject: [PATCH] test-runner: Performance improvements + regex match for test
 include/exclude (#9622)

<!--
if this PR closes one or more issues, you can automatically link the PR
with
them by using one of the [*linking
keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword),
e.g.
- this PR should close #xxxx
- fixes #xxxx

you can also mention related issues, PRs or discussions!
-->

# Description
<!--
Thank you for improving Nushell. Please, check our [contributing
guide](../CONTRIBUTING.md) and talk to the core team before making major
changes.

Description of your pull request goes here. **Provide examples and/or
screenshots** if your changes affect the user experience.
-->
Test-runner performance improved by:
* Not loading user config or stdlib during ide parsing
* Not loading user config during test execution
* Running tests in parallel instead of serially
On my machine `toolkit test stdlib` execution time went from 38s to 15s
(with all code precompiled)

Use regex match for test include/exclude and module exclude to allow for
multiple tests/modules to be excluded.
# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A
clippy::needless_collect -A clippy::result_large_err` to check that
you're using the standard code style
- `cargo test --workspace` to check that all tests pass
- `cargo run -- crates/nu-std/tests/run.nu` to run the tests for the
standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->
---
 crates/nu-std/std/testing.nu | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/crates/nu-std/std/testing.nu b/crates/nu-std/std/testing.nu
index 1fee82adf..26fe17c55 100644
--- a/crates/nu-std/std/testing.nu
+++ b/crates/nu-std/std/testing.nu
@@ -30,7 +30,7 @@ def get-annotated [
     let raw_file = (open $file)
 
     let ast = (
-        ^$nu.current-exe --ide-ast $file
+        ^$nu.current-exe --no-config-file --no-std-lib --ide-ast $file
         | from json
         | enumerate
         | flatten
@@ -184,7 +184,7 @@ export def ($test_function_name) [] {
     | save $rendered_module_path
 
     let result = (
-        ^$nu.current-exe -c $"use ($rendered_module_path) *; ($test_function_name)|to nuon"
+        ^$nu.current-exe --no-config-file -c $"use ($rendered_module_path) *; ($test_function_name)|to nuon"
         | complete
     )
 
@@ -255,7 +255,7 @@ def run-tests-for-module [
                 ''
             }
         }
-        | each {|test|
+        | par-each {|test|
             log info $"Running ($test.test) in module ($module.name)"
             log debug $"Global context is ($global_context)"
 
@@ -300,9 +300,9 @@ def run-tests-for-module [
 export def run-tests [
     --path: path,             # Path to look for tests. Default: current directory.
     --module: string,         # Test module to run. Default: all test modules found.
-    --test: string,           # Individual test to run. Default: all test command found in the files.
-    --exclude: string,        # Individual test to exclude. Default: no tests are excluded
-    --exclude-module: string, # Test module to exclude. Default: No modules are excluded
+    --test: string,           # Pattern to use to include tests. Default: all tests found in the files.
+    --exclude: string,        # Pattern to use to exclude tests. Default: no tests are excluded
+    --exclude-module: string, # Pattern to use to exclude test modules. Default: No modules are excluded
     --list,                   # list the selected tests without running them.
 ] {
 
@@ -336,7 +336,7 @@ export def run-tests [
 
     let modules = (
         ls ($path | path join $module_search_pattern)
-        | each {|row| {file: $row.name name: ($row.name | path parse | get stem)}}
+        | par-each {|row| {file: $row.name name: ($row.name | path parse | get stem)}}
         | insert commands {|module|
             get-annotated $module.file
         }
@@ -347,13 +347,13 @@ export def run-tests [
         }
         | flatten
         | filter {|x| ($x.test|length) > 0}
-        | filter {|x| if ($exclude_module|is-empty) {true} else {$exclude_module != $x.name}}
-        | filter {|x| if ($test|is-empty) {true} else {$test in $x.test}}
+        | filter {|x| if ($exclude_module|is-empty) {true} else {$x.name !~ $exclude_module}}
+        | filter {|x| if ($test|is-empty) {true} else {$x.test|any {|y| $y =~ $test}}}
         | filter {|x| if ($module|is-empty) {true} else {$module == $x.name}}
         | update test {|x|
             $x.test
-            | filter {|y| if ($test|is-empty) {true} else {$y == $test}}
-            | filter {|y| if ($exclude|is-empty) {true} else {$y != $exclude}}
+            | filter {|y| if ($test|is-empty) {true} else {$y =~ $test}}
+            | filter {|y| if ($exclude|is-empty) {true} else {$y !~ $exclude}}
         }
     )
     if $list {
@@ -366,7 +366,7 @@ export def run-tests [
 
     let results = (
         $modules
-        | each {|module|
+        | par-each {|module|
             run-tests-for-module $module
         }
         | flatten
@@ -375,7 +375,7 @@ export def run-tests [
         let text = ([
             $"(ansi purple)some tests did not pass (char lparen)see complete errors below(char rparen):(ansi reset)"
             ""
-            ($results | each {|test| ($test | show-pretty-test 4)} | str join "\n")
+            ($results | par-each {|test| ($test | show-pretty-test 4)} | str join "\n")
             ""
         ] | str join "\n")