fstests: re-arrange backend integration tests so they can be retried

Before this change backend integration tests depended on each other,
so tests could not be retried.

After this change we nest tests to ensure that tests are provided with
the starting state they expect.

Tell the integration test runner that it can retry backend tests also.

This also includes bin/test_independence.go which runs each test
individually for a backend to prove that they are independent.

backend/drive/drive_internal_test.go

@@ -243,10 +243,19 @@ func (f *Fs) InternalTestDocumentLink(t *testing.T) {
 }
 
 func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("DocumentImport", f.InternalTestDocumentImport)
-	t.Run("DocumentUpdate", f.InternalTestDocumentUpdate)
-	t.Run("DocumentExport", f.InternalTestDocumentExport)
-	t.Run("DocumentLink", f.InternalTestDocumentLink)
+	// These tests all depend on each other so run them as nested tests
+	t.Run("DocumentImport", func(t *testing.T) {
+		f.InternalTestDocumentImport(t)
+		t.Run("DocumentUpdate", func(t *testing.T) {
+			f.InternalTestDocumentUpdate(t)
+			t.Run("DocumentExport", func(t *testing.T) {
+				f.InternalTestDocumentExport(t)
+				t.Run("DocumentLink", func(t *testing.T) {
+					f.InternalTestDocumentLink(t)
+				})
+			})
+		})
+	})
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

bin/test_independence.go

@@ -0,0 +1,59 @@
+// +build ignore
+
+// Test that the tests in the suite passed in are independent
+
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"os/exec"
+	"regexp"
+)
+
+var matchLine = regexp.MustCompile(`(?m)^=== RUN\s*(TestIntegration/\S*)\s*$`)
+
+// run the test pass in and grep out the test names
+func findTests(packageToTest string) (tests []string) {
+	cmd := exec.Command("go", "test", "-v", packageToTest)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		_, _ = os.Stderr.Write(out)
+		log.Fatal(err)
+	}
+	results := matchLine.FindAllSubmatch(out, -1)
+	if results == nil {
+		log.Fatal("No tests found")
+	}
+	for _, line := range results {
+		tests = append(tests, string(line[1]))
+	}
+	return tests
+}
+
+// run the test passed in with the -run passed in
+func runTest(packageToTest string, testName string) {
+	cmd := exec.Command("go", "test", "-v", packageToTest, "-run", "^"+testName+"$")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Printf("%s FAILED ------------------", testName)
+		_, _ = os.Stderr.Write(out)
+		log.Printf("%s FAILED ------------------", testName)
+	} else {
+		log.Printf("%s OK", testName)
+	}
+}
+func main() {
+	flag.Parse()
+	args := flag.Args()
+	if len(args) != 1 {
+		log.Fatalf("Syntax: %s <test_to_run>", os.Args[0])
+	}
+	packageToTest := args[0]
+	testNames := findTests(packageToTest)
+	// fmt.Printf("%s\n", testNames)
+	for _, testName := range testNames {
+		runTest(packageToTest, testName)
+	}
+}

fstest/test_all/config.yaml

@@ -1,7 +1,6 @@
 tests:
  - path: backend
    addbackend: true
-   noretries:  true
    nobinary:   true
  - path: fs/operations
    subdir:   true