From 61d76ae47dbea3f4db95368ab602008e33c1c5c2 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 22 Feb 2024 11:13:32 +0000 Subject: [PATCH] fstests: add integration tests for Directory Metadata and ModTime --- fs/operations/operations_test.go | 109 ++++++++++++++++ fstest/fstest.go | 68 ++++++++++ fstest/fstests/fstests.go | 216 ++++++++++++++++++++++++++++--- 3 files changed, 378 insertions(+), 15 deletions(-) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 57cf76d27..6f55b09c0 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1639,3 +1639,112 @@ func TestTouchDir(t *testing.T) { r.CheckRemoteItems(t, file1, file2, file3) } } + +var testMetadata = fs.Metadata{ + // System metadata supported by all backends + "mtime": t1.Format(time.RFC3339Nano), + // User metadata + "potato": "jersey", +} + +func TestMkdirMetadata(t *testing.T) { + const name = "dir with metadata" + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + r := fstest.NewRun(t) + features := r.Fremote.Features() + if features.MkdirMetadata == nil { + t.Skip("Skipping test as remote does not support MkdirMetadata") + } + + newDst, err := operations.MkdirMetadata(ctx, r.Fremote, name, testMetadata) + require.NoError(t, err) + require.NotNil(t, newDst) + + require.True(t, features.ReadDirMetadata, "Expecting ReadDirMetadata to be supported if MkdirMetadata is supported") + + // Check the returned directory and one read from the listing + fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata) + fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), testMetadata) +} + +func TestMkdirModTime(t *testing.T) { + const name = "directory with modtime" + ctx := context.Background() + r := fstest.NewRun(t) + if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil { + t.Skip("Skipping test as remote does not support DirSetModTime or MkdirMetadata") + } + newDst, err := operations.MkdirModTime(ctx, r.Fremote, name, t2) + require.NoError(t, err) + + // Check the returned directory and one read from the listing + fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2) + fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2) +} + +func TestCopyDirMetadata(t *testing.T) { + const nameNonExistent = "non existent directory" + const nameExistent = "existing directory" + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + r := fstest.NewRun(t) + if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil { + t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata") + } + + // Create a source local directory with metadata + newSrc, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata) + require.NoError(t, err) + require.NotNil(t, newSrc) + + // First try with the directory not existing + newDst, err := operations.CopyDirMetadata(ctx, r.Fremote, nil, nameNonExistent, newSrc) + require.NoError(t, err) + require.NotNil(t, newDst) + + // Check the returned directory and one read from the listing + fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata) + fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameNonExistent), testMetadata) + + // Then try with the directory existing + require.NoError(t, r.Fremote.Rmdir(ctx, nameNonExistent)) + require.NoError(t, r.Fremote.Mkdir(ctx, nameExistent)) + existingDir := fstest.NewDirectory(ctx, t, r.Fremote, nameExistent) + + newDst, err = operations.CopyDirMetadata(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", newSrc) + require.NoError(t, err) + require.NotNil(t, newDst) + + // Check the returned directory and one read from the listing + fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata) + fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameExistent), testMetadata) +} + +func TestSetDirModTime(t *testing.T) { + const name = "set modtime on existing directory" + ctx := context.Background() + r := fstest.NewRun(t) + if r.Fremote.Features().DirSetModTime == nil && !r.Fremote.Features().WriteDirSetModTime { + t.Skip("Skipping test as remote does not support DirSetModTime or WriteDirSetModTime") + } + + // First try with the directory not existing - should return an error + newDst, err := operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2) + require.Error(t, err) + require.Nil(t, newDst) + + // Then try with the directory existing + require.NoError(t, r.Fremote.Mkdir(ctx, name)) + existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name) + + newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2) + require.NoError(t, err) + require.NotNil(t, newDst) + + // Check the returned directory and one read from the listing + fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2) + fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2) +} diff --git a/fstest/fstest.go b/fstest/fstest.go index c74ae6e9a..bff0b027e 100644 --- a/fstest/fstest.go +++ b/fstest/fstest.go @@ -519,3 +519,71 @@ func Purge(f fs.Fs) { log.Printf("purge failed: %v", err) } } + +// NewDirectory finds the directory with remote in f +// +// One day this will be an rclone primitive +func NewDirectory(ctx context.Context, t *testing.T, f fs.Fs, remote string) fs.Directory { + var err error + var dir fs.Directory + sleepTime := 1 * time.Second + root := path.Dir(remote) + if root == "." { + root = "" + } + for i := 1; i <= *ListRetries; i++ { + var entries fs.DirEntries + entries, err = f.List(ctx, root) + if err != nil { + continue + } + for _, entry := range entries { + var ok bool + dir, ok = entry.(fs.Directory) + if ok && dir.Remote() == remote { + return dir + } + } + err = fmt.Errorf("directory %q not found in %q", remote, root) + t.Logf("Sleeping for %v for findDir eventual consistency: %d/%d (%v)", sleepTime, i, *ListRetries, err) + time.Sleep(sleepTime) + sleepTime = (sleepTime * 3) / 2 + } + require.NoError(t, err) + return dir +} + +// CheckEntryMetadata checks the metadata on the directory +// +// This checks a limited set of metadata on the directory +func CheckEntryMetadata(ctx context.Context, t *testing.T, f fs.Fs, entry fs.DirEntry, wantMeta fs.Metadata) { + features := f.Features() + do, ok := entry.(fs.Metadataer) + require.True(t, ok, "Didn't find expected Metadata() method on %T", entry) + gotMeta, err := do.Metadata(ctx) + require.NoError(t, err) + + for k, v := range wantMeta { + switch k { + case "mtime", "atime", "btime", "ctime": + // Check the system time Metadata + wantT, err := time.Parse(time.RFC3339, v) + require.NoError(t, err) + gotT, err := time.Parse(time.RFC3339, gotMeta[k]) + require.NoError(t, err) + AssertTimeEqualWithPrecision(t, entry.Remote(), wantT, gotT, f.Precision()) + default: + // Check the User metadata if we can + _, isDir := entry.(fs.Directory) + if (isDir && features.UserDirMetadata) || (!isDir && features.UserMetadata) { + assert.Equal(t, v, gotMeta[k]) + } + } + } +} + +// CheckDirModTime checks the modtime on the directory +func CheckDirModTime(ctx context.Context, t *testing.T, f fs.Fs, dir fs.Directory, wantT time.Time) { + gotT := dir.ModTime(ctx) + AssertTimeEqualWithPrecision(t, dir.Remote(), wantT, gotT, f.Precision()) +} diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 095a94d23..e978e159d 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -307,19 +307,21 @@ type ExtraConfigItem struct{ Name, Key, Value string } // Opt is options for Run type Opt struct { - RemoteName string - NilObject fs.Object - ExtraConfig []ExtraConfigItem - SkipBadWindowsCharacters bool // skips unusable characters for windows if set - SkipFsMatch bool // if set skip exact matching of Fs value - TiersToTest []string // List of tiers which can be tested in setTier test - ChunkedUpload ChunkedUploadConfig - UnimplementableFsMethods []string // List of methods which can't be implemented in this wrapping Fs - UnimplementableObjectMethods []string // List of methods which can't be implemented in this wrapping Fs - SkipFsCheckWrap bool // if set skip FsCheckWrap - SkipObjectCheckWrap bool // if set skip ObjectCheckWrap - SkipInvalidUTF8 bool // if set skip invalid UTF-8 checks - QuickTestOK bool // if set, run this test with make quicktest + RemoteName string + NilObject fs.Object + ExtraConfig []ExtraConfigItem + SkipBadWindowsCharacters bool // skips unusable characters for windows if set + SkipFsMatch bool // if set skip exact matching of Fs value + TiersToTest []string // List of tiers which can be tested in setTier test + ChunkedUpload ChunkedUploadConfig + UnimplementableFsMethods []string // List of Fs methods which can't be implemented in this wrapping Fs + UnimplementableObjectMethods []string // List of Object methods which can't be implemented in this wrapping Fs + UnimplementableDirectoryMethods []string // List of Directory methods which can't be implemented in this wrapping Fs + SkipFsCheckWrap bool // if set skip FsCheckWrap + SkipObjectCheckWrap bool // if set skip ObjectCheckWrap + SkipDirectoryCheckWrap bool // if set skip DirectoryCheckWrap + SkipInvalidUTF8 bool // if set skip invalid UTF-8 checks + QuickTestOK bool // if set, run this test with make quicktest } // returns true if x is found in ss @@ -1513,8 +1515,8 @@ func Run(t *testing.T, opt *Opt) { } } if !features.ReadMetadata { - if metadata != nil { - require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata") + if metadata != nil && !features.Overlay { + require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata: %#v", metadata) } } else if features.WriteMetadata { require.NotNil(t, metadata) @@ -2301,6 +2303,190 @@ func Run(t *testing.T, opt *Opt) { // somehow confused about root and absolute root. }) + // FsDirSetModTime tests setting the mod time on a directory if possible + t.Run("FsDirSetModTime", func(t *testing.T) { + const name = "dir-mod-time" + do := f.Features().DirSetModTime + if do == nil { + t.Skip("FS has no DirSetModTime interface") + } + + // Set ModTime on non existing directory should return error + t1 := fstest.Time("2001-02-03T04:05:06.499999999Z") + err := do(ctx, name, t1) + require.Error(t, err) + + // Make the directory and try again + err = f.Mkdir(ctx, name) + require.NoError(t, err) + err = do(ctx, name, t1) + require.NoError(t, err) + + // Check the modtime got set properly + dir := fstest.NewDirectory(ctx, t, f, name) + fstest.CheckDirModTime(ctx, t, f, dir, t1) + + // Tidy up + err = f.Rmdir(ctx, name) + require.NoError(t, err) + }) + + var testMetadata = fs.Metadata{ + // System metadata supported by all backends + "mtime": "2001-02-03T04:05:06.499999999Z", + // User metadata + "potato": "jersey", + } + var testMetadata2 = fs.Metadata{ + // System metadata supported by all backends + "mtime": "2002-02-03T04:05:06.499999999Z", + // User metadata + "potato": "king edwards", + } + + // FsMkdirMetadata tests creating a directory with metadata if possible + t.Run("FsMkdirMetadata", func(t *testing.T) { + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + const name = "dir-metadata" + do := f.Features().MkdirMetadata + if do == nil { + t.Skip("FS has no MkdirMetadata interface") + } + assert.True(t, f.Features().WriteDirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata") + + // Create the directory from fresh + dir, err := do(ctx, name, testMetadata) + require.NoError(t, err) + require.NotNil(t, dir) + + // Check the returned directory and one read from the listing + fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata) + fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata) + + // Now update the metadata on the existing directory + t.Run("Update", func(t *testing.T) { + dir, err := do(ctx, name, testMetadata2) + require.NoError(t, err) + require.NotNil(t, dir) + + // Check the returned directory and one read from the listing + fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata2) + // The TestUnionPolicy2 has randomness in it so it sets metadata on + // one directory but can read a different one from the listing. + if f.Name() != "TestUnionPolicy2" { + fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata2) + } + }) + + // Now test the Directory methods + t.Run("CheckDirectory", func(t *testing.T) { + _, ok := dir.(fs.Object) + assert.False(t, ok, "Directory must not type assert to Object") + _, ok = dir.(fs.ObjectInfo) + assert.False(t, ok, "Directory must not type assert to ObjectInfo") + }) + + // Tidy up + err = f.Rmdir(ctx, name) + require.NoError(t, err) + }) + + // FsDirectory checks methods on the directory object + t.Run("FsDirectory", func(t *testing.T) { + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + const name = "dir-methods" + features := f.Features() + + if !features.CanHaveEmptyDirectories { + t.Skip("Can't test if can't have empty directories") + } + if !features.ReadDirMetadata && + !features.WriteDirMetadata && + !features.WriteDirSetModTime && + !features.UserDirMetadata && + !features.Overlay && + features.UnWrap == nil { + t.Skip("FS has no Directory methods and doesn't Wrap") + } + + // Create a directory to start with + err := f.Mkdir(ctx, name) + require.NoError(t, err) + + // Get the directory object + dir := fstest.NewDirectory(ctx, t, f, name) + _, ok := dir.(fs.Object) + assert.False(t, ok, "Directory must not type assert to Object") + _, ok = dir.(fs.ObjectInfo) + assert.False(t, ok, "Directory must not type assert to ObjectInfo") + + // Now test the directory methods + t.Run("ReadDirMetadata", func(t *testing.T) { + if !features.ReadDirMetadata { + t.Skip("Directories don't support ReadDirMetadata") + } + if f.Name() == "TestUnionPolicy3" { + t.Skipf("Test unreliable on %q", f.Name()) + } + fstest.CheckEntryMetadata(ctx, t, f, dir, fs.Metadata{ + "mtime": dir.ModTime(ctx).Format(time.RFC3339Nano), + }) + }) + + t.Run("WriteDirMetadata", func(t *testing.T) { + if !features.WriteDirMetadata { + t.Skip("Directories don't support WriteDirMetadata") + } + assert.NotNil(t, features.MkdirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata") + do, ok := dir.(fs.SetMetadataer) + require.True(t, ok, "Expected to find SetMetadata method on Directory") + err := do.SetMetadata(ctx, testMetadata) + require.NoError(t, err) + + fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata) + fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata) + }) + + t.Run("WriteDirSetModTime", func(t *testing.T) { + if !features.WriteDirSetModTime { + t.Skip("Directories don't support WriteDirSetModTime") + } + assert.NotNil(t, features.DirSetModTime, "Backends must support Directory.SetModTime and Fs.DirSetModTime") + + t1 := fstest.Time("2001-02-03T04:05:10.123123123Z") + + do, ok := dir.(fs.SetModTimer) + require.True(t, ok, "Expected to find SetMetadata method on Directory") + err := do.SetModTime(ctx, t1) + require.NoError(t, err) + + fstest.CheckDirModTime(ctx, t, f, dir, t1) + fstest.CheckDirModTime(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), t1) + }) + + // Check to see if Fs that wrap other Directories implement all the optional methods + t.Run("DirectoryCheckWrap", func(t *testing.T) { + if opt.SkipDirectoryCheckWrap { + t.Skip("Skipping DirectoryCheckWrap on this Fs") + } + if !features.Overlay && features.UnWrap == nil { + t.Skip("Not a wrapping Fs") + } + _, unsupported := fs.DirectoryOptionalInterfaces(dir) + for _, name := range unsupported { + if !stringsContains(name, opt.UnimplementableDirectoryMethods) { + t.Errorf("Missing Directory wrapper for %s", name) + } + } + }) + + // Tidy up + err = f.Rmdir(ctx, name) + require.NoError(t, err) + }) + // Purge the folder err = operations.Purge(ctx, f, "") if !errors.Is(err, fs.ErrorDirNotFound) {