diff --git a/Makefile b/Makefile index f654501..de3f582 100644 --- a/Makefile +++ b/Makefile @@ -144,8 +144,9 @@ format: ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img +ZREPL_PLATFORMTEST_ARGS := platformtest: # do not track dependency on platformtest-bin to allow build of platformtest outside of test VM - "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" -poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" -imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" + "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" -poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" -imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" $(ZREPL_PLATFORMTEST_ARGS) ##################### NOARCH ##################### .PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean diff --git a/platformtest/harness/harness.go b/platformtest/harness/harness.go index 368f181..703b159 100644 --- a/platformtest/harness/harness.go +++ b/platformtest/harness/harness.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "time" "github.com/fatih/color" @@ -25,6 +26,9 @@ var boldGreen = color.New(color.Bold, color.FgHiGreen) var args struct { createArgs platformtest.ZpoolCreateArgs stopAndKeepPoolOnFail bool + + run string + runRE *regexp.Regexp } func main() { @@ -42,8 +46,11 @@ func doMain() error { flag.Int64Var(&args.createArgs.ImageSize, "imagesize", 100*(1<<20), "") flag.StringVar(&args.createArgs.Mountpoint, "mountpoint", "none", "") flag.BoolVar(&args.stopAndKeepPoolOnFail, "failure.stop-and-keep-pool", false, "if a test case fails, stop test execution and keep pool as it was when the test failed") + flag.StringVar(&args.run, "run", "", "") flag.Parse() + args.runRE = regexp.MustCompile(args.run) + outlets := logger.NewOutlets() outlet, level, err := logging.ParseOutlet(config.LoggingOutletEnum{Ret: &config.StdoutLoggingOutlet{ LoggingOutletCommon: config.LoggingOutletCommon{ @@ -64,9 +71,21 @@ func doMain() error { ctx := platformtest.WithLogger(context.Background(), logger) ex := platformtest.NewEx(logger) - for _, c := range tests.Cases { + type invocation struct { + runFunc tests.Case + result *testCaseResult + } - bold.Printf("BEGIN TEST CASE %s\n", c) + invocations := make([]*invocation, 0, len(tests.Cases)) + for _, c := range tests.Cases { + if args.runRE.MatchString(c.String()) { + invocations = append(invocations, &invocation{runFunc: c}) + } + } + + for _, inv := range invocations { + + bold.Printf("BEGIN TEST CASE %s\n", inv.runFunc.String()) pool, err := platformtest.CreateOrReplaceZpool(ctx, ex, args.createArgs) if err != nil { @@ -78,7 +97,8 @@ func doMain() error { RootDataset: filepath.Join(pool.Name(), "rootds"), } - res := runTestCase(ctx, ex, c) + res := runTestCase(ctx, ex, inv.runFunc) + inv.result = res if res.failed { fmt.Printf("%+v\n", res.failedStack) // print with stack trace } @@ -98,11 +118,47 @@ func doMain() error { bold.Printf("TEST SKIPPED\n") } else if res.succeeded { boldGreen.Printf("TEST PASSED\n") + } else { + panic("unreachable") } fmt.Println() } + var summary struct { + succ, fail, skip []*invocation + } + for _, inv := range invocations { + var bucket *[]*invocation + if inv.result.failed { + bucket = &summary.fail + } else if inv.result.skipped { + bucket = &summary.skip + } else if inv.result.succeeded { + bucket = &summary.succ + } else { + panic("unreachable") + } + *bucket = append(*bucket, inv) + } + printBucket := func(bucketName string, c *color.Color, bucket []*invocation) { + c.Printf("%s:", bucketName) + if len(bucket) == 0 { + fmt.Printf(" []\n") + return + } + fmt.Printf("\n") + for _, inv := range bucket { + fmt.Printf(" %s\n", inv.runFunc.String()) + } + } + printBucket("PASSING TESTS", boldGreen, summary.succ) + printBucket("SKIPPED TESTS", bold, summary.skip) + printBucket("FAILED TESTS", boldRed, summary.fail) + + if len(summary.fail) > 0 { + return errors.New("at least one test failed") + } return nil } @@ -113,7 +169,7 @@ type testCaseResult struct { failedStack error // has stack inside, valid if failed=true } -func runTestCase(ctx *platformtest.Context, ex platformtest.Execer, c tests.Case) testCaseResult { +func runTestCase(ctx *platformtest.Context, ex platformtest.Execer, c tests.Case) *testCaseResult { // run case var paniced = false @@ -133,14 +189,14 @@ func runTestCase(ctx *platformtest.Context, ex platformtest.Execer, c tests.Case if paniced { switch panicValue { case platformtest.SkipNowSentinel: - return testCaseResult{skipped: true} + return &testCaseResult{skipped: true} case platformtest.FailNowSentinel: - return testCaseResult{failed: true, failedStack: panicStack} + return &testCaseResult{failed: true, failedStack: panicStack} default: - return testCaseResult{failed: true, failedStack: panicStack} + return &testCaseResult{failed: true, failedStack: panicStack} } } else { - return testCaseResult{succeeded: true} + return &testCaseResult{succeeded: true} } }