From 4d44937f6330b077ad9c1ec3ed0ae7f46e6da8e1 Mon Sep 17 00:00:00 2001 From: Filipe Azevedo Date: Wed, 14 Dec 2022 23:02:10 +0100 Subject: [PATCH] vfstest: make VFS test suite support symlinks --- vfs/vfstest/file.go | 228 ++++++++++++++++++++++++++++++++++++++++++++ vfs/vfstest/fs.go | 70 ++++++++++++-- vfs/vfstest/os.go | 12 +++ 3 files changed, 304 insertions(+), 6 deletions(-) diff --git a/vfs/vfstest/file.go b/vfs/vfstest/file.go index a4e490f0e..e923b521f 100644 --- a/vfs/vfstest/file.go +++ b/vfs/vfstest/file.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/rclone/rclone/fs" "github.com/rclone/rclone/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -72,3 +73,230 @@ func TestFileModTimeWithOpenWriters(t *testing.T) { run.rm(t, "cp-archive-test") } + +// TestSymlinks tests all the api of the VFS / Mount symlinks support +func TestSymlinks(t *testing.T) { + run.skipIfNoFUSE(t) + if !run.vfsOpt.Links { + t.Skip("No symlinks configured") + } + + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows") + } + + fs.Logf(nil, "Links: %v, useVFS: %v, suffix: %v", run.vfsOpt.Links, run.useVFS, fs.LinkSuffix) + + // Create initial setup of test files and directories we will create links to + run.mkdir(t, "dir1") + run.mkdir(t, "dir1/sub1dir1") + run.createFile(t, "dir1/file1", "potato") + run.mkdir(t, "dir2") + run.mkdir(t, "dir2/sub1dir2") + run.createFile(t, "dir2/file1", "chicken") + + // base state all the tests will be build off + baseState := "dir1/|dir1/sub1dir1/|dir1/file1 6|dir2/|dir2/sub1dir2/|dir2/file1 7" + // Check the tests return to the base state + checkBaseState := func() { + run.checkDir(t, baseState) + } + checkBaseState() + + t.Run("FileLink", func(t *testing.T) { + // Link to a file + run.symlink(t, "dir1/file1", "dir1file1_link") + run.checkDir(t, baseState+"|dir1file1_link 10") + run.checkMode(t, "dir1file1_link", os.FileMode(run.vfsOpt.LinkPerms), os.FileMode(run.vfsOpt.FilePerms)) + assert.Equal(t, "dir1/file1", run.readlink(t, "dir1file1_link")) + + // Read through a symlink + assert.Equal(t, "potato", run.readFile(t, "dir1file1_link")) + + // Write through a symlink + err := writeFile(run.path("dir1file1_link"), []byte("carrot"), 0600) + require.NoError(t, err) + assert.Equal(t, "carrot", run.readFile(t, "dir1file1_link")) + assert.Equal(t, "carrot", run.readFile(t, "dir1/file1")) + + // Rename a symlink + err = run.os.Rename(run.path("dir1file1_link"), run.path("dir1file1_link")+"_bla") + require.NoError(t, err) + run.checkDir(t, baseState+"|dir1file1_link_bla 10") + assert.Equal(t, "dir1/file1", run.readlink(t, "dir1file1_link_bla")) + + // Delete a symlink + run.rm(t, "dir1file1_link_bla") + checkBaseState() + }) + + t.Run("DirLink", func(t *testing.T) { + // Link to a dir + run.symlink(t, "dir1", "dir1_link") + run.checkDir(t, baseState+"|dir1_link 4") + run.checkMode(t, "dir1_link", os.FileMode(run.vfsOpt.LinkPerms), os.FileMode(run.vfsOpt.DirPerms)) + assert.Equal(t, "dir1", run.readlink(t, "dir1_link")) + + // Check you can't open a directory symlink + _, err := run.os.OpenFile(run.path("dir1_link"), os.O_WRONLY, 0600) + require.Error(t, err) + + // Our symlink resolution is very simple when using the VFS as when using the + // mount the OS will resolve the symlinks, so we don't recurse here + + // Read entries directly + dir1Entries := make(dirMap) + run.readLocalEx(t, dir1Entries, "dir1", false) + assert.Equal(t, newDirMap("dir1/sub1dir1/|dir1/file1 6"), dir1Entries) + + // Read entries through the directory symlink + dir1EntriesSymlink := make(dirMap) + run.readLocalEx(t, dir1EntriesSymlink, "dir1_link", false) + assert.Equal(t, newDirMap("dir1_link/sub1dir1/|dir1_link/file1 6"), dir1EntriesSymlink) + + // Rename directory symlink + err = run.os.Rename(run.path("dir1_link"), run.path("dir1_link")+"_bla") + require.NoError(t, err) + run.checkDir(t, baseState+"|dir1_link_bla 4") + assert.Equal(t, "dir1", run.readlink(t, "dir1_link_bla")) + + // Remove directory symlink + run.rm(t, "dir1_link_bla") + + checkBaseState() + }) + + // Corner case #1 - We do not allow creating regular and symlink files having the same name (ie, test.txt and test.txt.rclonelink) + + // Symlink first, then regular + t.Run("OverwriteSymlinkWithRegular", func(t *testing.T) { + link1Name := "link1.txt" + + run.symlink(t, "dir1/file1", link1Name) + run.checkDir(t, baseState+"|link1.txt 10") + + fh, err := run.os.OpenFile(run.path(link1Name), os.O_WRONLY|os.O_CREATE, os.FileMode(run.vfsOpt.FilePerms)) + + // On real mount with links enabled, that open the symlink target as expected, else that fails to create a new file + assert.NoError(t, err) + // Don't care about the result, in some cache mode the file can't be opened for writing, so closing would trigger an err + _ = fh.Close() + + run.rm(t, link1Name) + checkBaseState() + }) + + // Regular first, then symlink + t.Run("OverwriteRegularWithSymlink", func(t *testing.T) { + link1Name := "link1.txt" + + run.createFile(t, link1Name, "") + run.checkDir(t, baseState+"|link1.txt 0") + + err := run.os.Symlink(".", run.path(link1Name)) + assert.Error(t, err) + + run.rm(t, link1Name) + checkBaseState() + }) + + // Corner case #2 - We do not allow creating directory and symlink file having the same name (ie, test and test.rclonelink) + + // Symlink first, then directory + t.Run("OverwriteSymlinkWithDirectory", func(t *testing.T) { + link1Name := "link1" + + run.symlink(t, ".", link1Name) + run.checkDir(t, baseState+"|link1 1") + + err := run.os.Mkdir(run.path(link1Name), os.FileMode(run.vfsOpt.DirPerms)) + assert.Error(t, err) + + run.rm(t, link1Name) + checkBaseState() + }) + + // Directory first, then symlink + t.Run("OverwriteDirectoryWithSymlink", func(t *testing.T) { + link1Name := "link1" + + run.mkdir(t, link1Name) + run.checkDir(t, baseState+"|link1/") + + err := run.os.Symlink(".", run.path(link1Name)) + assert.Error(t, err) + + run.rm(t, link1Name) + checkBaseState() + }) + + // Corner case #3 - We do not allow moving directory or file having the same name in a target (ie, test and test.rclonelink) + + // Move symlink -> regular file + t.Run("MoveSymlinkToFile", func(t *testing.T) { + t.Skip("FIXME not implemented") + link1Name := "link1.txt" + + run.symlink(t, ".", link1Name) + run.createFile(t, "dir1/link1.txt", "") + run.checkDir(t, baseState+"|link1.txt 1|dir1/link1.txt 0") + + err := run.os.Rename(run.path(link1Name), run.path("dir1/"+link1Name)) + assert.Error(t, err) + + run.rm(t, link1Name) + run.rm(t, "dir1/link1.txt") + checkBaseState() + }) + + // Move regular file -> symlink + t.Run("MoveFileToSymlink", func(t *testing.T) { + t.Skip("FIXME not implemented") + link1Name := "link1.txt" + + run.createFile(t, link1Name, "") + run.symlink(t, ".", "dir1/"+link1Name) + run.checkDir(t, baseState+"|link1.txt 0|dir1/link1.txt 1") + + err := run.os.Rename(run.path(link1Name), run.path("dir1/link1.txt")) + assert.Error(t, err) + + run.rm(t, link1Name) + run.rm(t, "dir1/"+link1Name) + checkBaseState() + }) + + // Move symlink -> directory + t.Run("MoveSymlinkToDirectory", func(t *testing.T) { + t.Skip("FIXME not implemented") + link1Name := "link1" + + run.symlink(t, ".", link1Name) + run.mkdir(t, "dir1/link1") + run.checkDir(t, baseState+"|link1 1|dir1/link1/") + + err := run.os.Rename(run.path(link1Name), run.path("dir1/"+link1Name)) + assert.Error(t, err) + + run.rm(t, link1Name) + run.rm(t, "dir1/link1") + checkBaseState() + }) + + // Move directory -> symlink + t.Run("MoveDirectoryToSymlink", func(t *testing.T) { + t.Skip("FIXME not implemented") + link1Name := "dir1/link1" + + run.mkdir(t, "link1") + run.symlink(t, ".", link1Name) + run.checkDir(t, baseState+"|link1/|dir1/link1 1") + + err := run.os.Rename(run.path("link1"), run.path("dir1/link1")) + assert.Error(t, err) + + run.rm(t, "link1") + run.rm(t, link1Name) + checkBaseState() + }) +} diff --git a/vfs/vfstest/fs.go b/vfs/vfstest/fs.go index 1c74f54dc..944ff4a35 100644 --- a/vfs/vfstest/fs.go +++ b/vfs/vfstest/fs.go @@ -49,12 +49,15 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cach tests := []struct { cacheMode vfscommon.CacheMode writeBack fs.Duration + links bool }{ {cacheMode: vfscommon.CacheModeOff}, + {cacheMode: vfscommon.CacheModeOff, links: true}, {cacheMode: vfscommon.CacheModeMinimal}, {cacheMode: vfscommon.CacheModeWrites}, {cacheMode: vfscommon.CacheModeFull}, {cacheMode: vfscommon.CacheModeFull, writeBack: fs.Duration(100 * time.Millisecond)}, + {cacheMode: vfscommon.CacheModeFull, writeBack: fs.Duration(100 * time.Millisecond), links: true}, } for _, test := range tests { if test.cacheMode < minimumRequiredCacheMode { @@ -63,11 +66,15 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cach vfsOpt := vfscommon.Opt vfsOpt.CacheMode = test.cacheMode vfsOpt.WriteBack = test.writeBack + vfsOpt.Links = test.links run = newRun(useVFS, &vfsOpt, mountFn) what := fmt.Sprintf("CacheMode=%v", test.cacheMode) if test.writeBack > 0 { what += fmt.Sprintf(",WriteBack=%v", test.writeBack) } + if test.links { + what += fmt.Sprintf(",Links=%v", test.links) + } fs.Logf(nil, "Starting test run with %s", what) ok := t.Run(what, func(t *testing.T) { t.Run("TestTouchAndDelete", TestTouchAndDelete) @@ -98,6 +105,7 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cach t.Run("TestWriteFileFsync", TestWriteFileFsync) t.Run("TestWriteFileDup", TestWriteFileDup) t.Run("TestWriteFileAppend", TestWriteFileAppend) + t.Run("TestSymlinks", TestSymlinks) }) fs.Logf(nil, "Finished test run with %s (ok=%v)", what, ok) run.Finalise() @@ -213,10 +221,16 @@ func newDirMap(dirString string) (dm dirMap) { } // Returns a dirmap with only the files in -func (dm dirMap) filesOnly() dirMap { +func (dm dirMap) filesOnly(stripLinksSuffix bool) dirMap { newDm := make(dirMap) for name := range dm { if !strings.HasSuffix(name, "/") { + if stripLinksSuffix { + index := strings.LastIndex(name, " ") + if index != -1 { + name = strings.TrimSuffix(name[0:index], fs.LinkSuffix) + name[index:] + } + } newDm[name] = struct{}{} } } @@ -224,7 +238,9 @@ func (dm dirMap) filesOnly() dirMap { } // reads the local tree into dir -func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) { +// +// If recurse it set it will recurse into subdirectories +func (r *Run) readLocalEx(t *testing.T, dir dirMap, filePath string, recurse bool) { realPath := r.path(filePath) files, err := r.os.ReadDir(realPath) require.NoError(t, err) @@ -232,15 +248,26 @@ func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) { name := path.Join(filePath, fi.Name()) if fi.IsDir() { dir[name+"/"] = struct{}{} - r.readLocal(t, dir, name) + if recurse { + r.readLocalEx(t, dir, name, recurse) + } assert.Equal(t, os.FileMode(r.vfsOpt.DirPerms)&os.ModePerm, fi.Mode().Perm()) } else { dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{} - assert.Equal(t, os.FileMode(r.vfsOpt.FilePerms)&os.ModePerm, fi.Mode().Perm()) + if fi.Mode()&os.ModeSymlink != 0 { + assert.Equal(t, os.FileMode(r.vfsOpt.LinkPerms)&os.ModePerm, fi.Mode().Perm()) + } else { + assert.Equal(t, os.FileMode(r.vfsOpt.FilePerms)&os.ModePerm, fi.Mode().Perm()) + } } } } +// reads the local tree into dir +func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) { + r.readLocalEx(t, dir, filePath, true) +} + // reads the remote tree into dir func (r *Run) readRemote(t *testing.T, dir dirMap, filepath string) { objs, dirs, err := walk.GetAll(context.Background(), r.fremote, filepath, true, 1) @@ -271,7 +298,7 @@ func (r *Run) checkDir(t *testing.T, dirString string) { remoteDm = make(dirMap) r.readRemote(t, remoteDm, "") // Ignore directories for remote compare - remoteOK = reflect.DeepEqual(dm.filesOnly(), remoteDm.filesOnly()) + remoteOK = reflect.DeepEqual(dm.filesOnly(run.vfsOpt.Links), remoteDm.filesOnly(run.vfsOpt.Links)) fuseOK = reflect.DeepEqual(dm, localDm) if remoteOK && fuseOK { return @@ -280,7 +307,7 @@ func (r *Run) checkDir(t *testing.T, dirString string) { t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries) time.Sleep(sleep) } - assert.Equal(t, dm.filesOnly(), remoteDm.filesOnly(), "expected vs remote") + assert.Equal(t, dm.filesOnly(run.vfsOpt.Links), remoteDm.filesOnly(run.vfsOpt.Links), "expected vs remote") assert.Equal(t, dm, localDm, "expected vs fuse mount") } @@ -353,6 +380,37 @@ func (r *Run) rmdir(t *testing.T, filepath string) { require.NoError(t, err) } +func (r *Run) symlink(t *testing.T, oldname, newname string) { + newname = r.path(newname) + err := r.os.Symlink(oldname, newname) + require.NoError(t, err) +} + +func (r *Run) checkMode(t *testing.T, name string, lexpected os.FileMode, expected os.FileMode) { + if r.useVFS { + info, err := run.os.Stat(run.path(name)) + require.NoError(t, err) + assert.Equal(t, lexpected, info.Mode()) + assert.Equal(t, name, info.Name()) + } else { + info, err := os.Lstat(run.path(name)) + require.NoError(t, err) + assert.Equal(t, lexpected, info.Mode()) + assert.Equal(t, name, info.Name()) + + info, err = run.os.Stat(run.path(name)) + require.NoError(t, err) + assert.Equal(t, expected, info.Mode()) + assert.Equal(t, name, info.Name()) + } +} + +func (r *Run) readlink(t *testing.T, name string) string { + result, err := r.os.Readlink(r.path(name)) + require.NoError(t, err) + return result +} + // TestMount checks that the Fs is mounted by seeing if the mountpoint // is in the mount output func TestMount(t *testing.T) { diff --git a/vfs/vfstest/os.go b/vfs/vfstest/os.go index 6e7a1b640..ea6d00641 100644 --- a/vfs/vfstest/os.go +++ b/vfs/vfstest/os.go @@ -22,6 +22,8 @@ type Oser interface { Remove(name string) error Rename(oldName, newName string) error Stat(path string) (os.FileInfo, error) + Symlink(oldname, newname string) error + Readlink(name string) (s string, err error) } // realOs is an implementation of Oser backed by the "os" package @@ -130,6 +132,16 @@ func (r realOs) Stat(path string) (os.FileInfo, error) { return os.Stat(path) } +// Symlink +func (r realOs) Symlink(oldname, newname string) error { + return os.Symlink(oldname, newname) +} + +// Readlink +func (r realOs) Readlink(name string) (s string, err error) { + return os.Readlink(name) +} + // Check interfaces var _ Oser = &realOs{} var _ vfs.Handle = &realOsFile{}