fstests: add integration tests for Directory Metadata and ModTime

fs/operations/operations_test.go

@@ -1639,3 +1639,112 @@ func TestTouchDir(t *testing.T) {
 		r.CheckRemoteItems(t, file1, file2, file3)
 	}
 }
+
+var testMetadata = fs.Metadata{
+	// System metadata supported by all backends
+	"mtime": t1.Format(time.RFC3339Nano),
+	// User metadata
+	"potato": "jersey",
+}
+
+func TestMkdirMetadata(t *testing.T) {
+	const name = "dir with metadata"
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	ci.Metadata = true
+	r := fstest.NewRun(t)
+	features := r.Fremote.Features()
+	if features.MkdirMetadata == nil {
+		t.Skip("Skipping test as remote does not support MkdirMetadata")
+	}
+
+	newDst, err := operations.MkdirMetadata(ctx, r.Fremote, name, testMetadata)
+	require.NoError(t, err)
+	require.NotNil(t, newDst)
+
+	require.True(t, features.ReadDirMetadata, "Expecting ReadDirMetadata to be supported if MkdirMetadata is supported")
+
+	// Check the returned directory and one read from the listing
+	fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
+	fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), testMetadata)
+}
+
+func TestMkdirModTime(t *testing.T) {
+	const name = "directory with modtime"
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil {
+		t.Skip("Skipping test as remote does not support DirSetModTime or MkdirMetadata")
+	}
+	newDst, err := operations.MkdirModTime(ctx, r.Fremote, name, t2)
+	require.NoError(t, err)
+
+	// Check the returned directory and one read from the listing
+	fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2)
+	fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
+}
+
+func TestCopyDirMetadata(t *testing.T) {
+	const nameNonExistent = "non existent directory"
+	const nameExistent = "existing directory"
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	ci.Metadata = true
+	r := fstest.NewRun(t)
+	if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil {
+		t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata")
+	}
+
+	// Create a source local directory with metadata
+	newSrc, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata)
+	require.NoError(t, err)
+	require.NotNil(t, newSrc)
+
+	// First try with the directory not existing
+	newDst, err := operations.CopyDirMetadata(ctx, r.Fremote, nil, nameNonExistent, newSrc)
+	require.NoError(t, err)
+	require.NotNil(t, newDst)
+
+	// Check the returned directory and one read from the listing
+	fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
+	fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameNonExistent), testMetadata)
+
+	// Then try with the directory existing
+	require.NoError(t, r.Fremote.Rmdir(ctx, nameNonExistent))
+	require.NoError(t, r.Fremote.Mkdir(ctx, nameExistent))
+	existingDir := fstest.NewDirectory(ctx, t, r.Fremote, nameExistent)
+
+	newDst, err = operations.CopyDirMetadata(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", newSrc)
+	require.NoError(t, err)
+	require.NotNil(t, newDst)
+
+	// Check the returned directory and one read from the listing
+	fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
+	fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameExistent), testMetadata)
+}
+
+func TestSetDirModTime(t *testing.T) {
+	const name = "set modtime on existing directory"
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	if r.Fremote.Features().DirSetModTime == nil && !r.Fremote.Features().WriteDirSetModTime {
+		t.Skip("Skipping test as remote does not support DirSetModTime or WriteDirSetModTime")
+	}
+
+	// First try with the directory not existing - should return an error
+	newDst, err := operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2)
+	require.Error(t, err)
+	require.Nil(t, newDst)
+
+	// Then try with the directory existing
+	require.NoError(t, r.Fremote.Mkdir(ctx, name))
+	existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name)
+
+	newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2)
+	require.NoError(t, err)
+	require.NotNil(t, newDst)
+
+	// Check the returned directory and one read from the listing
+	fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2)
+	fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
+}

fstest/fstest.go

@@ -519,3 +519,71 @@ func Purge(f fs.Fs) {
 		log.Printf("purge failed: %v", err)
 	}
 }
+
+// NewDirectory finds the directory with remote in f
+//
+// One day this will be an rclone primitive
+func NewDirectory(ctx context.Context, t *testing.T, f fs.Fs, remote string) fs.Directory {
+	var err error
+	var dir fs.Directory
+	sleepTime := 1 * time.Second
+	root := path.Dir(remote)
+	if root == "." {
+		root = ""
+	}
+	for i := 1; i <= *ListRetries; i++ {
+		var entries fs.DirEntries
+		entries, err = f.List(ctx, root)
+		if err != nil {
+			continue
+		}
+		for _, entry := range entries {
+			var ok bool
+			dir, ok = entry.(fs.Directory)
+			if ok && dir.Remote() == remote {
+				return dir
+			}
+		}
+		err = fmt.Errorf("directory %q not found in %q", remote, root)
+		t.Logf("Sleeping for %v for findDir eventual consistency: %d/%d (%v)", sleepTime, i, *ListRetries, err)
+		time.Sleep(sleepTime)
+		sleepTime = (sleepTime * 3) / 2
+	}
+	require.NoError(t, err)
+	return dir
+}
+
+// CheckEntryMetadata checks the metadata on the directory
+//
+// This checks a limited set of metadata on the directory
+func CheckEntryMetadata(ctx context.Context, t *testing.T, f fs.Fs, entry fs.DirEntry, wantMeta fs.Metadata) {
+	features := f.Features()
+	do, ok := entry.(fs.Metadataer)
+	require.True(t, ok, "Didn't find expected Metadata() method on %T", entry)
+	gotMeta, err := do.Metadata(ctx)
+	require.NoError(t, err)
+
+	for k, v := range wantMeta {
+		switch k {
+		case "mtime", "atime", "btime", "ctime":
+			// Check the system time Metadata
+			wantT, err := time.Parse(time.RFC3339, v)
+			require.NoError(t, err)
+			gotT, err := time.Parse(time.RFC3339, gotMeta[k])
+			require.NoError(t, err)
+			AssertTimeEqualWithPrecision(t, entry.Remote(), wantT, gotT, f.Precision())
+		default:
+			// Check the User metadata if we can
+			_, isDir := entry.(fs.Directory)
+			if (isDir && features.UserDirMetadata) || (!isDir && features.UserMetadata) {
+				assert.Equal(t, v, gotMeta[k])
+			}
+		}
+	}
+}
+
+// CheckDirModTime checks the modtime on the directory
+func CheckDirModTime(ctx context.Context, t *testing.T, f fs.Fs, dir fs.Directory, wantT time.Time) {
+	gotT := dir.ModTime(ctx)
+	AssertTimeEqualWithPrecision(t, dir.Remote(), wantT, gotT, f.Precision())
+}

fstest/fstests/fstests.go

@@ -307,19 +307,21 @@ type ExtraConfigItem struct{ Name, Key, Value string }
 
 // Opt is options for Run
 type Opt struct {
-	RemoteName                   string
+	RemoteName                      string
-	NilObject                    fs.Object
+	NilObject                       fs.Object
-	ExtraConfig                  []ExtraConfigItem
+	ExtraConfig                     []ExtraConfigItem
-	SkipBadWindowsCharacters     bool     // skips unusable characters for windows if set
+	SkipBadWindowsCharacters        bool     // skips unusable characters for windows if set
-	SkipFsMatch                  bool     // if set skip exact matching of Fs value
+	SkipFsMatch                     bool     // if set skip exact matching of Fs value
-	TiersToTest                  []string // List of tiers which can be tested in setTier test
+	TiersToTest                     []string // List of tiers which can be tested in setTier test
-	ChunkedUpload                ChunkedUploadConfig
+	ChunkedUpload                   ChunkedUploadConfig
-	UnimplementableFsMethods     []string // List of methods which can't be implemented in this wrapping Fs
+	UnimplementableFsMethods        []string // List of Fs methods which can't be implemented in this wrapping Fs
-	UnimplementableObjectMethods []string // List of methods which can't be implemented in this wrapping Fs
+	UnimplementableObjectMethods    []string // List of Object methods which can't be implemented in this wrapping Fs
-	SkipFsCheckWrap              bool     // if set skip FsCheckWrap
+	UnimplementableDirectoryMethods []string // List of Directory methods which can't be implemented in this wrapping Fs
-	SkipObjectCheckWrap          bool     // if set skip ObjectCheckWrap
+	SkipFsCheckWrap                 bool     // if set skip FsCheckWrap
-	SkipInvalidUTF8              bool     // if set skip invalid UTF-8 checks
+	SkipObjectCheckWrap             bool     // if set skip ObjectCheckWrap
-	QuickTestOK                  bool     // if set, run this test with make quicktest
+	SkipDirectoryCheckWrap          bool     // if set skip DirectoryCheckWrap
+	SkipInvalidUTF8                 bool     // if set skip invalid UTF-8 checks
+	QuickTestOK                     bool     // if set, run this test with make quicktest
 }
 
 // returns true if x is found in ss
@@ -1513,8 +1515,8 @@ func Run(t *testing.T, opt *Opt) {
 					}
 				}
 				if !features.ReadMetadata {
-					if metadata != nil {
+					if metadata != nil && !features.Overlay {
-						require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata")
+						require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata: %#v", metadata)
 					}
 				} else if features.WriteMetadata {
 					require.NotNil(t, metadata)
@@ -2301,6 +2303,190 @@ func Run(t *testing.T, opt *Opt) {
 			// somehow confused about root and absolute root.
 		})
 
+		// FsDirSetModTime tests setting the mod time on a directory if possible
+		t.Run("FsDirSetModTime", func(t *testing.T) {
+			const name = "dir-mod-time"
+			do := f.Features().DirSetModTime
+			if do == nil {
+				t.Skip("FS has no DirSetModTime interface")
+			}
+
+			// Set ModTime on non existing directory should return error
+			t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
+			err := do(ctx, name, t1)
+			require.Error(t, err)
+
+			// Make the directory and try again
+			err = f.Mkdir(ctx, name)
+			require.NoError(t, err)
+			err = do(ctx, name, t1)
+			require.NoError(t, err)
+
+			// Check the modtime got set properly
+			dir := fstest.NewDirectory(ctx, t, f, name)
+			fstest.CheckDirModTime(ctx, t, f, dir, t1)
+
+			// Tidy up
+			err = f.Rmdir(ctx, name)
+			require.NoError(t, err)
+		})
+
+		var testMetadata = fs.Metadata{
+			// System metadata supported by all backends
+			"mtime": "2001-02-03T04:05:06.499999999Z",
+			// User metadata
+			"potato": "jersey",
+		}
+		var testMetadata2 = fs.Metadata{
+			// System metadata supported by all backends
+			"mtime": "2002-02-03T04:05:06.499999999Z",
+			// User metadata
+			"potato": "king edwards",
+		}
+
+		// FsMkdirMetadata tests creating a directory with metadata if possible
+		t.Run("FsMkdirMetadata", func(t *testing.T) {
+			ctx, ci := fs.AddConfig(ctx)
+			ci.Metadata = true
+			const name = "dir-metadata"
+			do := f.Features().MkdirMetadata
+			if do == nil {
+				t.Skip("FS has no MkdirMetadata interface")
+			}
+			assert.True(t, f.Features().WriteDirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata")
+
+			// Create the directory from fresh
+			dir, err := do(ctx, name, testMetadata)
+			require.NoError(t, err)
+			require.NotNil(t, dir)
+
+			// Check the returned directory and one read from the listing
+			fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata)
+			fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata)
+
+			// Now update the metadata on the existing directory
+			t.Run("Update", func(t *testing.T) {
+				dir, err := do(ctx, name, testMetadata2)
+				require.NoError(t, err)
+				require.NotNil(t, dir)
+
+				// Check the returned directory and one read from the listing
+				fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata2)
+				// The TestUnionPolicy2 has randomness in it so it sets metadata on
+				// one directory but can read a different one from the listing.
+				if f.Name() != "TestUnionPolicy2" {
+					fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata2)
+				}
+			})
+
+			// Now test the Directory methods
+			t.Run("CheckDirectory", func(t *testing.T) {
+				_, ok := dir.(fs.Object)
+				assert.False(t, ok, "Directory must not type assert to Object")
+				_, ok = dir.(fs.ObjectInfo)
+				assert.False(t, ok, "Directory must not type assert to ObjectInfo")
+			})
+
+			// Tidy up
+			err = f.Rmdir(ctx, name)
+			require.NoError(t, err)
+		})
+
+		// FsDirectory checks methods on the directory object
+		t.Run("FsDirectory", func(t *testing.T) {
+			ctx, ci := fs.AddConfig(ctx)
+			ci.Metadata = true
+			const name = "dir-methods"
+			features := f.Features()
+
+			if !features.CanHaveEmptyDirectories {
+				t.Skip("Can't test if can't have empty directories")
+			}
+			if !features.ReadDirMetadata &&
+				!features.WriteDirMetadata &&
+				!features.WriteDirSetModTime &&
+				!features.UserDirMetadata &&
+				!features.Overlay &&
+				features.UnWrap == nil {
+				t.Skip("FS has no Directory methods and doesn't Wrap")
+			}
+
+			// Create a directory to start with
+			err := f.Mkdir(ctx, name)
+			require.NoError(t, err)
+
+			// Get the directory object
+			dir := fstest.NewDirectory(ctx, t, f, name)
+			_, ok := dir.(fs.Object)
+			assert.False(t, ok, "Directory must not type assert to Object")
+			_, ok = dir.(fs.ObjectInfo)
+			assert.False(t, ok, "Directory must not type assert to ObjectInfo")
+
+			// Now test the directory methods
+			t.Run("ReadDirMetadata", func(t *testing.T) {
+				if !features.ReadDirMetadata {
+					t.Skip("Directories don't support ReadDirMetadata")
+				}
+				if f.Name() == "TestUnionPolicy3" {
+					t.Skipf("Test unreliable on %q", f.Name())
+				}
+				fstest.CheckEntryMetadata(ctx, t, f, dir, fs.Metadata{
+					"mtime": dir.ModTime(ctx).Format(time.RFC3339Nano),
+				})
+			})
+
+			t.Run("WriteDirMetadata", func(t *testing.T) {
+				if !features.WriteDirMetadata {
+					t.Skip("Directories don't support WriteDirMetadata")
+				}
+				assert.NotNil(t, features.MkdirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata")
+				do, ok := dir.(fs.SetMetadataer)
+				require.True(t, ok, "Expected to find SetMetadata method on Directory")
+				err := do.SetMetadata(ctx, testMetadata)
+				require.NoError(t, err)
+
+				fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata)
+				fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata)
+			})
+
+			t.Run("WriteDirSetModTime", func(t *testing.T) {
+				if !features.WriteDirSetModTime {
+					t.Skip("Directories don't support WriteDirSetModTime")
+				}
+				assert.NotNil(t, features.DirSetModTime, "Backends must support Directory.SetModTime and Fs.DirSetModTime")
+
+				t1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
+
+				do, ok := dir.(fs.SetModTimer)
+				require.True(t, ok, "Expected to find SetMetadata method on Directory")
+				err := do.SetModTime(ctx, t1)
+				require.NoError(t, err)
+
+				fstest.CheckDirModTime(ctx, t, f, dir, t1)
+				fstest.CheckDirModTime(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), t1)
+			})
+
+			// Check to see if Fs that wrap other Directories implement all the optional methods
+			t.Run("DirectoryCheckWrap", func(t *testing.T) {
+				if opt.SkipDirectoryCheckWrap {
+					t.Skip("Skipping DirectoryCheckWrap on this Fs")
+				}
+				if !features.Overlay && features.UnWrap == nil {
+					t.Skip("Not a wrapping Fs")
+				}
+				_, unsupported := fs.DirectoryOptionalInterfaces(dir)
+				for _, name := range unsupported {
+					if !stringsContains(name, opt.UnimplementableDirectoryMethods) {
+						t.Errorf("Missing Directory wrapper for %s", name)
+					}
+				}
+			})
+
+			// Tidy up
+			err = f.Rmdir(ctx, name)
+			require.NoError(t, err)
+		})
+
 		// Purge the folder
 		err = operations.Purge(ctx, f, "")
 		if !errors.Is(err, fs.ErrorDirNotFound) {