From 5ac3ad61c42c9c97952a5df11d79e69ffcde73db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20=C5=BD=C3=A1dn=C3=ADk?= <kubouch@gmail.com>
Date: Sun, 9 Jun 2024 14:03:52 +0300
Subject: [PATCH] Extend functionality of tango benchmark helpers (#13107)

<!--
if this PR closes one or more issues, you can automatically link the PR
with
them by using one of the [*linking
keywords*](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword),
e.g.
- this PR should close #xxxx
- fixes #xxxx

you can also mention related issues, PRs or discussions!
-->

# Description
<!--
Thank you for improving Nushell. Please, check our [contributing
guide](../CONTRIBUTING.md) and talk to the core team before making major
changes.

Description of your pull request goes here. **Provide examples and/or
screenshots** if your changes affect the user experience.
-->

Refactors the tango helpers in the toolkit and makes them more flexible
(e.g., being able to benchmark any branch against any branch, not just
current and main).

# User-Facing Changes
<!-- List of all changes that impact the user experience here. This
helps us keep track of breaking changes. -->

# Tests + Formatting
<!--
Don't forget to add tests that cover your changes.

Make sure you've run and fixed any issues with these commands:

- `cargo fmt --all -- --check` to check standard code formatting (`cargo
fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to
check that you're using the standard code style
- `cargo test --workspace` to check that all tests pass (on Windows make
sure to [enable developer
mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
- `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the
tests for the standard library

> **Note**
> from `nushell` you can also use the `toolkit` as follows
> ```bash
> use toolkit.nu # or use an `env_change` hook to activate it
automatically
> toolkit check pr
> ```
-->

# After Submitting
<!-- If your PR had any user-facing changes, update [the
documentation](https://github.com/nushell/nushell.github.io) after the
PR is merged, if necessary. This will help us keep the docs up to date.
-->
---
 toolkit.nu | 78 ++++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 61 insertions(+), 17 deletions(-)

diff --git a/toolkit.nu b/toolkit.nu
index 6f19900fdb..0ef56d1077 100644
--- a/toolkit.nu
+++ b/toolkit.nu
@@ -491,29 +491,73 @@ export def cov [] {
     print $"Coverage generation took ($end - $start)."
 }
 
-
-# Benchmark the current branch against the main branch
+# Benchmark a target revision (default: current branch) against a reference revision (default: main branch)
+#
+# Results are saved in a `./tango` directory
 # Ensure you have `cargo-export` installed to generate separate artifacts for each branch.
-export def "benchmark-current-branch-with-main" [] {
-    let main = "main"
-    let current_branch = (git branch --show-current)
+export def benchmark-compare [
+    target?: string     # which branch to compare (default: current branch)
+    reference?: string  # the reference to compare against (default: main branch)
+] {
+    let reference = $reference | default "main"
+    let current = git branch --show-current
+    let target = $target | default $current
 
-    cargo export $"target/($current_branch)" -- bench
-    git checkout $main
-    cargo export $"target/($main)" -- bench
-    git checkout $current_branch
-    ^$"./target/($current_branch)/benchmarks" compare $"./target/($main)/benchmarks" -o -s 50
+    print $'-- Benchmarking ($target) against ($reference)'
+
+    let export_dir = $env.PWD | path join "tango"
+    let ref_bin_dir = $export_dir | path join bin $reference
+    let tgt_bin_dir = $export_dir | path join bin $target
+
+    # benchmark the target revision
+    print $'-- Running benchmarks for ($target)'
+    git checkout $target
+    cargo export $tgt_bin_dir -- bench
+
+    # benchmark the comparison reference revision
+    print $'-- Running benchmarks for ($reference)'
+    git checkout $reference
+    cargo export $ref_bin_dir -- bench
+
+    # return back to the whatever revision before benchmarking
+    print '-- Done'
+    git checkout $current
+
+    # report results
+    let reference_bin = $ref_bin_dir | path join benchmarks
+    let target_bin = $tgt_bin_dir | path join benchmarks
+    ^$target_bin compare $reference_bin -o -s 50 --dump ($export_dir | path join samples)
 }
 
-# Benchmark the current branch and logs the result in `./target/samples`
+# Benchmark the current branch and logs the result in `./tango/samples`
+#
+# Results are saved in a `./tango` directory
 # Ensure you have `cargo-export` installed to generate separate artifacts for each branch.
-export def "benchmark-and-log-result" [] {
-    let current_branch = (git branch --show-current)
-    let current_dir = "./" | path expand
-    let res_path = $"($current_dir)/target/samples"
+export def benchmark-log [
+    target?: string     # which branch to compare (default: current branch)
+] {
+    let current = git branch --show-current
+    let target = $target | default $current
+    print $'-- Benchmarking ($target)'
 
-    cargo export $"target/($current_branch)" -- bench
-    ^$"./target/($current_branch)/benchmarks" compare -o -s 50 --dump $res_path
+    let export_dir = $env.PWD | path join "tango"
+    let bin_dir = ($export_dir | path join bin $target)
+
+    # benchmark the target revision
+    if $target != $current {
+        git checkout $target
+    }
+    cargo export $bin_dir -- bench
+
+    # return back to the whatever revision before benchmarking
+    print '-- Done'
+    if $target != $current {
+        git checkout $current
+    }
+
+    # report results
+    let bench_bin = ($bin_dir | path join benchmarks)
+    ^$bench_bin compare -o -s 50 --dump ($export_dir | path join samples)
 }
 
 # Build all Windows archives and MSIs for release manually