fstests: re-arrange backend integration tests so they can be retried

Before this change backend integration tests depended on each other,
so tests could not be retried.

After this change we nest tests to ensure that tests are provided with
the starting state they expect.

Tell the integration test runner that it can retry backend tests also.

This also includes bin/test_independence.go which runs each test
individually for a backend to prove that they are independent.

backend/drive/drive_internal_test.go

@@ -243,10 +243,19 @@ func (f *Fs) InternalTestDocumentLink(t *testing.T) {
 }
 
 func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("DocumentImport", f.InternalTestDocumentImport)
+	// These tests all depend on each other so run them as nested tests
-	t.Run("DocumentUpdate", f.InternalTestDocumentUpdate)
+	t.Run("DocumentImport", func(t *testing.T) {
-	t.Run("DocumentExport", f.InternalTestDocumentExport)
+		f.InternalTestDocumentImport(t)
-	t.Run("DocumentLink", f.InternalTestDocumentLink)
+		t.Run("DocumentUpdate", func(t *testing.T) {
+			f.InternalTestDocumentUpdate(t)
+			t.Run("DocumentExport", func(t *testing.T) {
+				f.InternalTestDocumentExport(t)
+				t.Run("DocumentLink", func(t *testing.T) {
+					f.InternalTestDocumentLink(t)
+				})
+			})
+		})
+	})
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

bin/test_independence.go

@@ -0,0 +1,59 @@
+// +build ignore
+
+// Test that the tests in the suite passed in are independent
+
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"os/exec"
+	"regexp"
+)
+
+var matchLine = regexp.MustCompile(`(?m)^=== RUN\s*(TestIntegration/\S*)\s*$`)
+
+// run the test pass in and grep out the test names
+func findTests(packageToTest string) (tests []string) {
+	cmd := exec.Command("go", "test", "-v", packageToTest)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		_, _ = os.Stderr.Write(out)
+		log.Fatal(err)
+	}
+	results := matchLine.FindAllSubmatch(out, -1)
+	if results == nil {
+		log.Fatal("No tests found")
+	}
+	for _, line := range results {
+		tests = append(tests, string(line[1]))
+	}
+	return tests
+}
+
+// run the test passed in with the -run passed in
+func runTest(packageToTest string, testName string) {
+	cmd := exec.Command("go", "test", "-v", packageToTest, "-run", "^"+testName+"$")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Printf("%s FAILED ------------------", testName)
+		_, _ = os.Stderr.Write(out)
+		log.Printf("%s FAILED ------------------", testName)
+	} else {
+		log.Printf("%s OK", testName)
+	}
+}
+func main() {
+	flag.Parse()
+	args := flag.Args()
+	if len(args) != 1 {
+		log.Fatalf("Syntax: %s <test_to_run>", os.Args[0])
+	}
+	packageToTest := args[0]
+	testNames := findTests(packageToTest)
+	// fmt.Printf("%s\n", testNames)
+	for _, testName := range testNames {
+		runTest(packageToTest, testName)
+	}
+}

fstest/fstests/fstests.go

@@ -1,5 +1,9 @@
 // Package fstests provides generic integration tests for the Fs and
-// Object interfaces
+// Object interfaces.
+//
+// These tests are concerned with the basic functionality of a
+// backend.  The tests in fs/sync and fs/operations tests more
+// cornercases that these tests don't.
 package fstests
 
 import (
@@ -245,8 +249,13 @@ type Opt struct {
 	ChunkedUpload            ChunkedUploadConfig
 }
 
-// Run runs the basic integration tests for a remote using the remote
+// Run runs the basic integration tests for a remote using the options passed in.
-// name passed in and the nil object
+//
+// They are structured in a heirachical way so that dependencies for the tests can be created.
+//
+// For example some tests require the directory to be created - these
+// are inside the "FsMkdir" test.  Some tests require some tests files
+// - these are inside the "FsPutFiles" test.
 func Run(t *testing.T, opt *Opt) {
 	var (
 		remote        fs.Fs
@@ -264,24 +273,9 @@ func Run(t *testing.T, opt *Opt) {
 			WinPath: `hello_ sausage/êé/Hello, 世界/ _ ' @ _ _ & _ + ≠/z.txt`,
 		}
 		isLocalRemote bool
+		purged        bool // whether the dir has been purged or not
 	)
 
-	// Make the Fs we are testing with, initialising the global variables
-	// subRemoteName - name of the remote after the TestRemote:
-	// subRemoteLeaf - a subdirectory to use under that
-	// remote - the result of  fs.NewFs(TestRemote:subRemoteName)
-	newFs := func(t *testing.T) {
-		var err error
-		subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(remoteName)
-		require.NoError(t, err)
-		remote, err = fs.NewFs(subRemoteName)
-		if err == fs.ErrorNotFoundInConfigFile {
-			t.Logf("Didn't find %q in config file - skipping tests", remoteName)
-			return
-		}
-		require.NoError(t, err, fmt.Sprintf("unexpected error: %v", err))
-	}
-
 	// Skip the test if the remote isn't configured
 	skipIfNotOk := func(t *testing.T) {
 		if remote == nil {
@@ -312,79 +306,6 @@ func Run(t *testing.T, opt *Opt) {
 		}
 	}
 
-	// TestInit tests basic intitialisation
-	t.Run("TestInit", func(t *testing.T) {
-		var err error
-
-		// Remove bad characters from Windows file name if set
-		if opt.SkipBadWindowsCharacters {
-			t.Logf("Removing bad windows characters from test file")
-			file2.Path = fstest.WinPath(file2.Path)
-		}
-
-		fstest.Initialise()
-
-		// Set extra config if supplied
-		for _, item := range opt.ExtraConfig {
-			config.FileSet(item.Name, item.Key, item.Value)
-		}
-		if *fstest.RemoteName != "" {
-			remoteName = *fstest.RemoteName
-		}
-		t.Logf("Using remote %q", remoteName)
-		if remoteName == "" {
-			remoteName, err = fstest.LocalRemote()
-			require.NoError(t, err)
-			isLocalRemote = true
-		}
-
-		newFs(t)
-
-		skipIfNotOk(t)
-
-		err = remote.Mkdir("")
-		require.NoError(t, err)
-		fstest.CheckListing(t, remote, []fstest.Item{})
-	})
-
-	// TestFsString tests the String method
-	t.Run("TestFsString", func(t *testing.T) {
-		skipIfNotOk(t)
-		str := remote.String()
-		require.NotEqual(t, "", str)
-	})
-
-	// TestFsName tests the Name method
-	t.Run("TestFsName", func(t *testing.T) {
-		skipIfNotOk(t)
-		got := remote.Name()
-		want := remoteName
-		if isLocalRemote {
-			want = "local:"
-		}
-		require.Equal(t, want, got+":")
-	})
-
-	// TestFsRoot tests the Root method
-	t.Run("TestFsRoot", func(t *testing.T) {
-		skipIfNotOk(t)
-		name := remote.Name() + ":"
-		root := remote.Root()
-		if isLocalRemote {
-			// only check last path element on local
-			require.Equal(t, filepath.Base(subRemoteName), filepath.Base(root))
-		} else {
-			require.Equal(t, subRemoteName, name+root)
-		}
-	})
-
-	// TestFsRmdirEmpty tests deleting an empty directory
-	t.Run("TestFsRmdirEmpty", func(t *testing.T) {
-		skipIfNotOk(t)
-		err := remote.Rmdir("")
-		require.NoError(t, err)
-	})
-
 	// Return true if f (or any of the things it wraps) is bucket
 	// based but not at the root.
 	isBucketBasedButNotRoot := func(f fs.Fs) bool {
@@ -398,8 +319,48 @@ func Run(t *testing.T, opt *Opt) {
 		return f.Features().BucketBased && strings.Contains(strings.Trim(f.Root(), "/"), "/")
 	}
 
+	// Remove bad characters from Windows file name if set
+	if opt.SkipBadWindowsCharacters {
+		t.Logf("Removing bad windows characters from test file")
+		file2.Path = fstest.WinPath(file2.Path)
+	}
+
+	// Initialise the remote
+	fstest.Initialise()
+
+	// Set extra config if supplied
+	for _, item := range opt.ExtraConfig {
+		config.FileSet(item.Name, item.Key, item.Value)
+	}
+	if *fstest.RemoteName != "" {
+		remoteName = *fstest.RemoteName
+	}
+	t.Logf("Using remote %q", remoteName)
+	var err error
+	if remoteName == "" {
+		remoteName, err = fstest.LocalRemote()
+		require.NoError(t, err)
+		isLocalRemote = true
+	}
+
+	// Make the Fs we are testing with, initialising the local variables
+	// subRemoteName - name of the remote after the TestRemote:
+	// subRemoteLeaf - a subdirectory to use under that
+	// remote - the result of  fs.NewFs(TestRemote:subRemoteName)
+	subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(remoteName)
+	require.NoError(t, err)
+	remote, err = fs.NewFs(subRemoteName)
+	if err == fs.ErrorNotFoundInConfigFile {
+		t.Logf("Didn't find %q in config file - skipping tests", remoteName)
+		return
+	}
+	require.NoError(t, err, fmt.Sprintf("unexpected error: %v", err))
+
+	// Skip the rest if it failed
+	skipIfNotOk(t)
+
 	// TestFsRmdirNotFound tests deleting a non existent directory
-	t.Run("TestFsRmdirNotFound", func(t *testing.T) {
+	t.Run("FsRmdirNotFound", func(t *testing.T) {
 		skipIfNotOk(t)
 		if isBucketBasedButNotRoot(remote) {
 			t.Skip("Skipping test as non root bucket based remote")
@@ -408,14 +369,54 @@ func Run(t *testing.T, opt *Opt) {
 		assert.Error(t, err, "Expecting error on Rmdir non existent")
 	})
 
-	// TestFsMkdir tests making a directory
+	// Make the directory
-	t.Run("TestFsMkdir", func(t *testing.T) {
+	err = remote.Mkdir("")
-		skipIfNotOk(t)
+	require.NoError(t, err)
+	fstest.CheckListing(t, remote, []fstest.Item{})
 
-		// Use a new directory here.  This is for the container based
+	// TestFsString tests the String method
-		// remotes which take time to create and destroy a container
+	t.Run("FsString", func(t *testing.T) {
-		// (eg azure blob)
+		skipIfNotOk(t)
-		newFs(t)
+		str := remote.String()
+		require.NotEqual(t, "", str)
+	})
+
+	// TestFsName tests the Name method
+	t.Run("FsName", func(t *testing.T) {
+		skipIfNotOk(t)
+		got := remote.Name()
+		want := remoteName
+		if isLocalRemote {
+			want = "local:"
+		}
+		require.Equal(t, want, got+":")
+	})
+
+	// TestFsRoot tests the Root method
+	t.Run("FsRoot", func(t *testing.T) {
+		skipIfNotOk(t)
+		name := remote.Name() + ":"
+		root := remote.Root()
+		if isLocalRemote {
+			// only check last path element on local
+			require.Equal(t, filepath.Base(subRemoteName), filepath.Base(root))
+		} else {
+			require.Equal(t, subRemoteName, name+root)
+		}
+	})
+
+	// TestFsRmdirEmpty tests deleting an empty directory
+	t.Run("FsRmdirEmpty", func(t *testing.T) {
+		skipIfNotOk(t)
+		err := remote.Rmdir("")
+		require.NoError(t, err)
+	})
+
+	// TestFsMkdir tests making a directory
+	//
+	// Tests that require the directory to be made are within this
+	t.Run("FsMkdir", func(t *testing.T) {
+		skipIfNotOk(t)
 
 		err := remote.Mkdir("")
 		require.NoError(t, err)
@@ -423,10 +424,9 @@ func Run(t *testing.T, opt *Opt) {
 
 		err = remote.Mkdir("")
 		require.NoError(t, err)
-	})
 
 		// TestFsMkdirRmdirSubdir tests making and removing a sub directory
-	t.Run("TestFsMkdirRmdirSubdir", func(t *testing.T) {
+		t.Run("FsMkdirRmdirSubdir", func(t *testing.T) {
 			skipIfNotOk(t)
 			dir := "dir/subdir"
 			err := operations.Mkdir(remote, dir)
@@ -443,7 +443,7 @@ func Run(t *testing.T, opt *Opt) {
 		})
 
 		// TestFsListEmpty tests listing an empty directory
-	t.Run("TestFsListEmpty", func(t *testing.T) {
+		t.Run("FsListEmpty", func(t *testing.T) {
 			skipIfNotOk(t)
 			fstest.CheckListing(t, remote, []fstest.Item{})
 		})
@@ -456,10 +456,10 @@ func Run(t *testing.T, opt *Opt) {
 			assert.Equal(t, []string{}, objsToNames(objs))
 			assert.Equal(t, []string{}, dirsToNames(dirs))
 		}
-	t.Run("TestFsListDirEmpty", TestFsListDirEmpty)
+		t.Run("FsListDirEmpty", TestFsListDirEmpty)
 
 		// TestFsListRDirEmpty tests listing the directories from an empty directory using ListR
-	t.Run("TestFsListRDirEmpty", func(t *testing.T) {
+		t.Run("FsListRDirEmpty", func(t *testing.T) {
 			defer skipIfNotListR(t)()
 			TestFsListDirEmpty(t)
 		})
@@ -477,16 +477,16 @@ func Run(t *testing.T, opt *Opt) {
 				assert.Equal(t, fs.ErrorDirNotFound, err)
 			}
 		}
-	t.Run("TestFsListDirNotFound", TestFsListDirNotFound)
+		t.Run("FsListDirNotFound", TestFsListDirNotFound)
 
 		// TestFsListRDirNotFound tests listing the directories from an empty directory using ListR
-	t.Run("TestFsListRDirNotFound", func(t *testing.T) {
+		t.Run("FsListRDirNotFound", func(t *testing.T) {
 			defer skipIfNotListR(t)()
 			TestFsListDirNotFound(t)
 		})
 
 		// TestFsNewObjectNotFound tests not finding a object
-	t.Run("TestFsNewObjectNotFound", func(t *testing.T) {
+		t.Run("FsNewObjectNotFound", func(t *testing.T) {
 			skipIfNotOk(t)
 			// Object in an existing directory
 			o, err := remote.NewObject("potato")
@@ -498,19 +498,13 @@ func Run(t *testing.T, opt *Opt) {
 			assert.Equal(t, fs.ErrorObjectNotFound, err)
 		})
 
-	// TestFsPutFile1 tests putting a file
-	t.Run("TestFsPutFile1", func(t *testing.T) {
-		skipIfNotOk(t)
-		file1Contents, _ = testPut(t, remote, &file1)
-	})
-
 		// TestFsPutError tests uploading a file where there is an error
 		//
 		// It makes sure that aborting a file half way through does not create
 		// a file on the remote.
 		//
 		// go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutError)$'
-	t.Run("TestFsPutError", func(t *testing.T) {
+		t.Run("FsPutError", func(t *testing.T) {
 			skipIfNotOk(t)
 
 			const N = 5 * 1024
@@ -530,20 +524,7 @@ func Run(t *testing.T, opt *Opt) {
 			assert.Equal(t, fs.ErrorObjectNotFound, err)
 		})
 
-	// TestFsPutFile2 tests putting a file into a subdirectory
+		t.Run("FsPutChunked", func(t *testing.T) {
-	t.Run("TestFsPutFile2", func(t *testing.T) {
-		skipIfNotOk(t)
-		/* file2Contents = */ testPut(t, remote, &file2)
-	})
-
-	// TestFsUpdateFile1 tests updating file1 with new contents
-	t.Run("TestFsUpdateFile1", func(t *testing.T) {
-		skipIfNotOk(t)
-		file1Contents, _ = testPut(t, remote, &file1)
-		// Note that the next test will check there are no duplicated file names
-	})
-
-	t.Run("TestFsPutChunked", func(t *testing.T) {
 			skipIfNotOk(t)
 
 			setUploadChunkSizer, _ := remote.(SetUploadChunkSizer)
@@ -646,6 +627,106 @@ func Run(t *testing.T, opt *Opt) {
 			}
 		})
 
+		// TestFsChangeNotify tests that changes are properly
+		// propagated
+		//
+		// go test -v -remote TestDrive: -run '^Test(Setup|Init|FsChangeNotify)$' -verbose
+		t.Run("FsChangeNotify", func(t *testing.T) {
+			skipIfNotOk(t)
+
+			// Check have ChangeNotify
+			doChangeNotify := remote.Features().ChangeNotify
+			if doChangeNotify == nil {
+				t.Skip("FS has no ChangeNotify interface")
+			}
+
+			err := operations.Mkdir(remote, "dir")
+			require.NoError(t, err)
+
+			pollInterval := make(chan time.Duration)
+			dirChanges := map[string]struct{}{}
+			objChanges := map[string]struct{}{}
+			doChangeNotify(func(x string, e fs.EntryType) {
+				fs.Debugf(nil, "doChangeNotify(%q, %+v)", x, e)
+				if strings.HasPrefix(x, file1.Path[:5]) || strings.HasPrefix(x, file2.Path[:5]) {
+					fs.Debugf(nil, "Ignoring notify for file1 or file2: %q, %v", x, e)
+					return
+				}
+				if e == fs.EntryDirectory {
+					dirChanges[x] = struct{}{}
+				} else if e == fs.EntryObject {
+					objChanges[x] = struct{}{}
+				}
+			}, pollInterval)
+			defer func() { close(pollInterval) }()
+			pollInterval <- time.Second
+
+			var dirs []string
+			for _, idx := range []int{1, 3, 2} {
+				dir := fmt.Sprintf("dir/subdir%d", idx)
+				err = operations.Mkdir(remote, dir)
+				require.NoError(t, err)
+				dirs = append(dirs, dir)
+			}
+
+			var objs []fs.Object
+			for _, idx := range []int{2, 4, 3} {
+				file := fstest.Item{
+					ModTime: time.Now(),
+					Path:    fmt.Sprintf("dir/file%d", idx),
+				}
+				_, o := testPut(t, remote, &file)
+				objs = append(objs, o)
+			}
+
+			// Looks for each item in wants in changes -
+			// if they are all found it returns true
+			contains := func(changes map[string]struct{}, wants []string) bool {
+				for _, want := range wants {
+					_, ok := changes[want]
+					if !ok {
+						return false
+					}
+				}
+				return true
+			}
+
+			// Wait a little while for the changes to come in
+			wantDirChanges := []string{"dir/subdir1", "dir/subdir3", "dir/subdir2"}
+			wantObjChanges := []string{"dir/file2", "dir/file4", "dir/file3"}
+			ok := false
+			for tries := 1; tries < 10; tries++ {
+				ok = contains(dirChanges, wantDirChanges) && contains(objChanges, wantObjChanges)
+				if ok {
+					break
+				}
+				t.Logf("Try %d/10 waiting for dirChanges and objChanges", tries)
+				time.Sleep(3 * time.Second)
+			}
+			if !ok {
+				t.Errorf("%+v does not contain %+v or \n%+v does not contain %+v", dirChanges, wantDirChanges, objChanges, wantObjChanges)
+			}
+
+			// tidy up afterwards
+			for _, o := range objs {
+				assert.NoError(t, o.Remove())
+			}
+			dirs = append(dirs, "dir")
+			for _, dir := range dirs {
+				assert.NoError(t, remote.Rmdir(dir))
+			}
+		})
+
+		// TestFsPut files writes file1, file2 and tests an update
+		//
+		// Tests that require file1, file2 are within this
+		t.Run("FsPutFiles", func(t *testing.T) {
+			skipIfNotOk(t)
+			file1Contents, _ = testPut(t, remote, &file1)
+			/* file2Contents = */ testPut(t, remote, &file2)
+			file1Contents, _ = testPut(t, remote, &file1)
+			// Note that the next test will check there are no duplicated file names
+
 			// TestFsListDirFile2 tests the files are correctly uploaded by doing
 			// Depth 1 directory listings
 			TestFsListDirFile2 := func(t *testing.T) {
@@ -689,11 +770,11 @@ func Run(t *testing.T, opt *Opt) {
 					list(dir, expectedDirNames, expectedObjNames)
 				}
 			}
-	t.Run("TestFsListDirFile2", TestFsListDirFile2)
+			t.Run("FsListDirFile2", TestFsListDirFile2)
 
 			// TestFsListRDirFile2 tests the files are correctly uploaded by doing
 			// Depth 1 directory listings using ListR
-	t.Run("TestFsListRDirFile2", func(t *testing.T) {
+			t.Run("FsListRDirFile2", func(t *testing.T) {
 				defer skipIfNotListR(t)()
 				TestFsListDirFile2(t)
 			})
@@ -707,10 +788,10 @@ func Run(t *testing.T, opt *Opt) {
 				require.NoError(t, err)
 				assert.Contains(t, dirsToNames(dirs), subRemoteLeaf, "Remote leaf not found")
 			}
-	t.Run("TestFsListDirRoot", TestFsListDirRoot)
+			t.Run("FsListDirRoot", TestFsListDirRoot)
 
 			// TestFsListRDirRoot tests that DirList works in the root using ListR
-	t.Run("TestFsListRDirRoot", func(t *testing.T) {
+			t.Run("FsListRDirRoot", func(t *testing.T) {
 				defer skipIfNotListR(t)()
 				TestFsListDirRoot(t)
 			})
@@ -736,10 +817,10 @@ func Run(t *testing.T, opt *Opt) {
 				assert.Equal(t, fileName, objs[0].Remote())
 				require.Len(t, dirs, 0)
 			}
-	t.Run("TestFsListSubdir", TestFsListSubdir)
+			t.Run("FsListSubdir", TestFsListSubdir)
 
 			// TestFsListRSubdir tests List works for a subdirectory using ListR
-	t.Run("TestFsListRSubdir", func(t *testing.T) {
+			t.Run("FsListRSubdir", func(t *testing.T) {
 				defer skipIfNotListR(t)()
 				TestFsListSubdir(t)
 			})
@@ -755,35 +836,35 @@ func Run(t *testing.T, opt *Opt) {
 				assert.Equal(t, []string{file1.Path}, objsToNames(objs))
 				assert.Equal(t, []string{`hello_ sausage`, `hello_ sausage/êé`}, dirsToNames(dirs))
 			}
-	t.Run("TestFsListLevel2", TestFsListLevel2)
+			t.Run("FsListLevel2", TestFsListLevel2)
 
 			// TestFsListRLevel2 tests List works for 2 levels using ListR
-	t.Run("TestFsListRLevel2", func(t *testing.T) {
+			t.Run("FsListRLevel2", func(t *testing.T) {
 				defer skipIfNotListR(t)()
 				TestFsListLevel2(t)
 			})
 
 			// TestFsListFile1 tests file present
-	t.Run("TestFsListFile1", func(t *testing.T) {
+			t.Run("FsListFile1", func(t *testing.T) {
 				skipIfNotOk(t)
 				fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
 			})
 
 			// TestFsNewObject tests NewObject
-	t.Run("TestFsNewObject", func(t *testing.T) {
+			t.Run("FsNewObject", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				file1.Check(t, obj, remote.Precision())
 			})
 
 			// TestFsListFile1and2 tests two files present
-	t.Run("TestFsListFile1and2", func(t *testing.T) {
+			t.Run("FsListFile1and2", func(t *testing.T) {
 				skipIfNotOk(t)
 				fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
 			})
 
 			// TestFsNewObjectDir tests NewObject on a directory which should produce an error
-	t.Run("TestFsNewObjectDir", func(t *testing.T) {
+			t.Run("FsNewObjectDir", func(t *testing.T) {
 				skipIfNotOk(t)
 				dir := path.Dir(file2.Path)
 				obj, err := remote.NewObject(dir)
@@ -792,7 +873,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestFsCopy tests Copy
-	t.Run("TestFsCopy", func(t *testing.T) {
+			t.Run("FsCopy", func(t *testing.T) {
 				skipIfNotOk(t)
 
 				// Check have Copy
@@ -826,7 +907,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestFsMove tests Move
-	t.Run("TestFsMove", func(t *testing.T) {
+			t.Run("FsMove", func(t *testing.T) {
 				skipIfNotOk(t)
 
 				// Check have Move
@@ -898,7 +979,7 @@ func Run(t *testing.T, opt *Opt) {
 			// TestFsDirMove tests DirMove
 			//
 			// go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|FsDirMove)$
-	t.Run("TestFsDirMove", func(t *testing.T) {
+			t.Run("FsDirMove", func(t *testing.T) {
 				skipIfNotOk(t)
 
 				// Check have DirMove
@@ -957,7 +1038,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestFsRmdirFull tests removing a non empty directory
-	t.Run("TestFsRmdirFull", func(t *testing.T) {
+			t.Run("FsRmdirFull", func(t *testing.T) {
 				skipIfNotOk(t)
 				if isBucketBasedButNotRoot(remote) {
 					t.Skip("Skipping test as non root bucket based remote")
@@ -967,7 +1048,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestFsPrecision tests the Precision of the Fs
-	t.Run("TestFsPrecision", func(t *testing.T) {
+			t.Run("FsPrecision", func(t *testing.T) {
 				skipIfNotOk(t)
 				precision := remote.Precision()
 				if precision == fs.ModTimeNotSupported {
@@ -979,86 +1060,8 @@ func Run(t *testing.T, opt *Opt) {
 				// FIXME check expected precision
 			})
 
-	// TestFsChangeNotify tests that changes are properly
-	// propagated
-	//
-	// go test -v -remote TestDrive: -run '^Test(Setup|Init|FsChangeNotify)$' -verbose
-	t.Run("TestFsChangeNotify", func(t *testing.T) {
-		skipIfNotOk(t)
-
-		// Check have ChangeNotify
-		doChangeNotify := remote.Features().ChangeNotify
-		if doChangeNotify == nil {
-			t.Skip("FS has no ChangeNotify interface")
-		}
-
-		err := operations.Mkdir(remote, "dir")
-		require.NoError(t, err)
-
-		pollInterval := make(chan time.Duration)
-		dirChanges := []string{}
-		objChanges := []string{}
-		doChangeNotify(func(x string, e fs.EntryType) {
-			fs.Debugf(nil, "doChangeNotify(%q, %+v)", x, e)
-			if strings.HasPrefix(x, file1.Path[:5]) || strings.HasPrefix(x, file2.Path[:5]) {
-				fs.Debugf(nil, "Ignoring notify for file1 or file2: %q, %v", x, e)
-				return
-			}
-			if e == fs.EntryDirectory {
-				if x != "dir" {
-					// ignore the base directory creation which we sometimes
-					// catch and sometimes don't
-					dirChanges = append(dirChanges, x)
-				}
-			} else if e == fs.EntryObject {
-				objChanges = append(objChanges, x)
-			}
-		}, pollInterval)
-		defer func() { close(pollInterval) }()
-		pollInterval <- time.Second
-
-		var dirs []string
-		for _, idx := range []int{1, 3, 2} {
-			dir := fmt.Sprintf("dir/subdir%d", idx)
-			err = operations.Mkdir(remote, dir)
-			require.NoError(t, err)
-			dirs = append(dirs, dir)
-		}
-
-		var objs []fs.Object
-		for _, idx := range []int{2, 4, 3} {
-			file := fstest.Item{
-				ModTime: time.Now(),
-				Path:    fmt.Sprintf("dir/file%d", idx),
-			}
-			_, o := testPut(t, remote, &file)
-			objs = append(objs, o)
-		}
-
-		// Wait a little while for the changes to come in
-		for tries := 1; tries < 10; tries++ {
-			if len(dirChanges) == 3 && len(objChanges) == 3 {
-				break
-			}
-			t.Logf("Try %d/10 waiting for dirChanges and objChanges", tries)
-			time.Sleep(3 * time.Second)
-		}
-
-		assert.Equal(t, []string{"dir/subdir1", "dir/subdir3", "dir/subdir2"}, dirChanges)
-		assert.Equal(t, []string{"dir/file2", "dir/file4", "dir/file3"}, objChanges)
-
-		// tidy up afterwards
-		for _, o := range objs {
-			assert.NoError(t, o.Remove())
-		}
-		dirs = append(dirs, "dir")
-		for _, dir := range dirs {
-			assert.NoError(t, remote.Rmdir(dir))
-		}
-	})
-
 			// TestObjectString tests the Object String method
-	t.Run("TestObjectString", func(t *testing.T) {
+			t.Run("ObjectString", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				assert.Equal(t, file1.Path, obj.String())
@@ -1068,7 +1071,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestObjectFs tests the object can be found
-	t.Run("TestObjectFs", func(t *testing.T) {
+			t.Run("ObjectFs", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				// If this is set we don't do the direct comparison of
@@ -1087,14 +1090,14 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestObjectRemote tests the Remote is correct
-	t.Run("TestObjectRemote", func(t *testing.T) {
+			t.Run("ObjectRemote", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				assert.Equal(t, file1.Path, obj.Remote())
 			})
 
 			// TestObjectHashes checks all the hashes the object supports
-	t.Run("TestObjectHashes", func(t *testing.T) {
+			t.Run("ObjectHashes", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				file1.CheckHashes(t, obj)
@@ -1106,10 +1109,10 @@ func Run(t *testing.T, opt *Opt) {
 				obj := findObject(t, remote, file1.Path)
 				file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision())
 			}
-	t.Run("TestObjectModTime", TestObjectModTime)
+			t.Run("ObjectModTime", TestObjectModTime)
 
 			// TestObjectMimeType tests the MimeType of the object is correct
-	t.Run("TestObjectMimeType", func(t *testing.T) {
+			t.Run("ObjectMimeType", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				do, ok := obj.(fs.MimeTyper)
@@ -1125,7 +1128,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestObjectSetModTime tests that SetModTime works
-	t.Run("TestObjectSetModTime", func(t *testing.T) {
+			t.Run("ObjectSetModTime", func(t *testing.T) {
 				skipIfNotOk(t)
 				newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z")
 				obj := findObject(t, remote, file1.Path)
@@ -1142,21 +1145,21 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestObjectSize tests that Size works
-	t.Run("TestObjectSize", func(t *testing.T) {
+			t.Run("ObjectSize", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				assert.Equal(t, file1.Size, obj.Size())
 			})
 
 			// TestObjectOpen tests that Open works
-	t.Run("TestObjectOpen", func(t *testing.T) {
+			t.Run("ObjectOpen", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				assert.Equal(t, file1Contents, readObject(t, obj, -1), "contents of file1 differ")
 			})
 
 			// TestObjectOpenSeek tests that Open works with SeekOption
-	t.Run("TestObjectOpenSeek", func(t *testing.T) {
+			t.Run("ObjectOpenSeek", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				assert.Equal(t, file1Contents[50:], readObject(t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek")
@@ -1165,7 +1168,7 @@ func Run(t *testing.T, opt *Opt) {
 			// TestObjectOpenRange tests that Open works with RangeOption
 			//
 			// go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|ObjectOpenRange)$'
-	t.Run("TestObjectOpenRange", func(t *testing.T) {
+			t.Run("ObjectOpenRange", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				for _, test := range []struct {
@@ -1186,14 +1189,14 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestObjectPartialRead tests that reading only part of the object does the correct thing
-	t.Run("TestObjectPartialRead", func(t *testing.T) {
+			t.Run("ObjectPartialRead", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				assert.Equal(t, file1Contents[:50], readObject(t, obj, 50), "contents of file1 differ after limited read")
 			})
 
 			// TestObjectUpdate tests that Update works
-	t.Run("TestObjectUpdate", func(t *testing.T) {
+			t.Run("ObjectUpdate", func(t *testing.T) {
 				skipIfNotOk(t)
 				contents := fstest.RandomString(200)
 				buf := bytes.NewBufferString(contents)
@@ -1220,7 +1223,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestObjectStorable tests that Storable works
-	t.Run("TestObjectStorable", func(t *testing.T) {
+			t.Run("ObjectStorable", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				require.NotNil(t, !obj.Storable(), "Expecting object to be storable")
@@ -1228,7 +1231,7 @@ func Run(t *testing.T, opt *Opt) {
 
 			// TestFsIsFile tests that an error is returned along with a valid fs
 			// which points to the parent directory.
-	t.Run("TestFsIsFile", func(t *testing.T) {
+			t.Run("FsIsFile", func(t *testing.T) {
 				skipIfNotOk(t)
 				remoteName := subRemoteName + "/" + file2.Path
 				file2Copy := file2
@@ -1241,7 +1244,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestFsIsFileNotFound tests that an error is not returned if no object is found
-	t.Run("TestFsIsFileNotFound", func(t *testing.T) {
+			t.Run("FsIsFileNotFound", func(t *testing.T) {
 				skipIfNotOk(t)
 				remoteName := subRemoteName + "/not found.txt"
 				fileRemote, err := fs.NewFs(remoteName)
@@ -1251,7 +1254,7 @@ func Run(t *testing.T, opt *Opt) {
 
 			// TestPublicLink tests creation of sharable, public links
 			// go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|PublicLink)$'
-	t.Run("TestPublicLink", func(t *testing.T) {
+			t.Run("PublicLink", func(t *testing.T) {
 				skipIfNotOk(t)
 
 				doPublicLink := remote.Features().PublicLink
@@ -1307,7 +1310,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestSetTier tests SetTier and GetTier functionality
-	t.Run("TestSetTier", func(t *testing.T) {
+			t.Run("SetTier", func(t *testing.T) {
 				skipIfNotSetTier(t)
 				obj := findObject(t, remote, file1.Path)
 				setter, ok := obj.(fs.SetTierer)
@@ -1328,7 +1331,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestObjectRemove tests Remove
-	t.Run("TestObjectRemove", func(t *testing.T) {
+			t.Run("ObjectRemove", func(t *testing.T) {
 				skipIfNotOk(t)
 				obj := findObject(t, remote, file1.Path)
 				err := obj.Remove()
@@ -1338,7 +1341,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestFsPutStream tests uploading files when size is not known in advance
-	t.Run("TestFsPutStream", func(t *testing.T) {
+			t.Run("FsPutStream", func(t *testing.T) {
 				skipIfNotOk(t)
 				if remote.Features().PutStream == nil {
 					t.Skip("FS has no PutStream interface")
@@ -1376,7 +1379,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestAbout tests the About optional interface
-	t.Run("TestObjectAbout", func(t *testing.T) {
+			t.Run("ObjectAbout", func(t *testing.T) {
 				skipIfNotOk(t)
 
 				// Check have About
@@ -1393,7 +1396,7 @@ func Run(t *testing.T, opt *Opt) {
 			})
 
 			// TestInternal calls InternalTest() on the Fs
-	t.Run("TestInternal", func(t *testing.T) {
+			t.Run("Internal", func(t *testing.T) {
 				skipIfNotOk(t)
 				if it, ok := remote.(InternalTester); ok {
 					it.InternalTest(t)
@@ -1402,27 +1405,32 @@ func Run(t *testing.T, opt *Opt) {
 				}
 			})
 
-	// TestObjectPurge tests Purge
+		})
-	t.Run("TestObjectPurge", func(t *testing.T) {
-		skipIfNotOk(t)
 
-		err := operations.Purge(remote, "")
+		// Purge the folder
+		err = operations.Purge(remote, "")
 		require.NoError(t, err)
+		purged = true
 		fstest.CheckListing(t, remote, []fstest.Item{})
 
+		// Check purging again if not bucket based
 		if !isBucketBasedButNotRoot(remote) {
 			err = operations.Purge(remote, "")
 			assert.Error(t, err, "Expecting error after on second purge")
 		}
+
 	})
 
-	// TestFinalise tidies up after the previous tests
+	// Check directory is purged
-	t.Run("TestFinalise", func(t *testing.T) {
+	if !purged {
-		skipIfNotOk(t)
+		_ = operations.Purge(remote, "")
+	}
+
+	// Remove the local directory so we don't clutter up /tmp
 	if strings.HasPrefix(remoteName, "/") {
+		t.Log("remoteName", remoteName)
 		// Remove temp directory
 		err := os.Remove(remoteName)
 		require.NoError(t, err)
 	}
-	})
 }

fstest/test_all/config.yaml

@@ -1,7 +1,6 @@
 tests:
  - path: backend
    addbackend: true
-   noretries:  true
    nobinary:   true
  - path: fs/operations
    subdir:   true