package vfs import ( "context" "fmt" "os" "sort" "testing" "time" "unsafe" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fstest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func dirCreate(t *testing.T) (r *fstest.Run, vfs *VFS, dir *Dir, item fstest.Item) { r, vfs = newTestVFS(t) file1 := r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1) r.CheckRemoteItems(t, file1) node, err := vfs.Stat("dir") require.NoError(t, err) require.True(t, node.IsDir()) return r, vfs, node.(*Dir), file1 } func TestDirMethods(t *testing.T) { _, vfs, dir, _ := dirCreate(t) // String assert.Equal(t, "dir/", dir.String()) assert.Equal(t, "", (*Dir)(nil).String()) // IsDir assert.Equal(t, true, dir.IsDir()) // IsFile assert.Equal(t, false, dir.IsFile()) // Mode assert.Equal(t, vfs.Opt.DirPerms, dir.Mode()) // Name assert.Equal(t, "dir", dir.Name()) // Path assert.Equal(t, "dir", dir.Path()) // Sys assert.Equal(t, nil, dir.Sys()) // SetSys dir.SetSys(42) assert.Equal(t, 42, dir.Sys()) // Inode assert.NotEqual(t, uint64(0), dir.Inode()) // Node assert.Equal(t, dir, dir.Node()) // ModTime assert.WithinDuration(t, t1, dir.ModTime(), 100*365*24*60*60*time.Second) // Size assert.Equal(t, int64(0), dir.Size()) // Sync assert.NoError(t, dir.Sync()) // DirEntry assert.Equal(t, dir.entry, dir.DirEntry()) // VFS assert.Equal(t, vfs, dir.VFS()) } func TestDirForgetAll(t *testing.T) { _, vfs, dir, file1 := dirCreate(t) // Make sure / and dir are in cache _, err := vfs.Stat(file1.Path) require.NoError(t, err) root, err := vfs.Root() require.NoError(t, err) assert.Equal(t, 1, len(root.items)) assert.Equal(t, 1, len(dir.items)) assert.False(t, root.read.IsZero()) assert.False(t, dir.read.IsZero()) dir.ForgetAll() assert.Equal(t, 1, len(root.items)) assert.Equal(t, 0, len(dir.items)) assert.False(t, root.read.IsZero()) assert.True(t, dir.read.IsZero()) root.ForgetAll() assert.Equal(t, 0, len(root.items)) assert.Equal(t, 0, len(dir.items)) assert.True(t, root.read.IsZero()) } func TestDirForgetPath(t *testing.T) { _, vfs, dir, file1 := dirCreate(t) // Make sure / and dir are in cache _, err := vfs.Stat(file1.Path) require.NoError(t, err) root, err := vfs.Root() require.NoError(t, err) assert.Equal(t, 1, len(root.items)) assert.Equal(t, 1, len(dir.items)) assert.False(t, root.read.IsZero()) assert.False(t, dir.read.IsZero()) root.ForgetPath("dir/notfound", fs.EntryObject) assert.Equal(t, 1, len(root.items)) assert.Equal(t, 1, len(dir.items)) assert.False(t, root.read.IsZero()) assert.True(t, dir.read.IsZero()) root.ForgetPath("dir", fs.EntryDirectory) assert.Equal(t, 1, len(root.items)) assert.Equal(t, 0, len(dir.items)) assert.True(t, root.read.IsZero()) root.ForgetPath("not/in/cache", fs.EntryDirectory) assert.Equal(t, 1, len(root.items)) assert.Equal(t, 0, len(dir.items)) } func TestDirWalk(t *testing.T) { r, vfs, _, file1 := dirCreate(t) file2 := r.WriteObject(context.Background(), "fil/a/b/c", "super long file", t1) r.CheckRemoteItems(t, file1, file2) root, err := vfs.Root() require.NoError(t, err) // Forget the cache since we put another object in root.ForgetAll() // Read the directories in _, err = vfs.Stat("dir") require.NoError(t, err) _, err = vfs.Stat("fil/a/b") require.NoError(t, err) fil, err := vfs.Stat("fil") require.NoError(t, err) var result []string fn := func(d *Dir) { result = append(result, d.path) } result = nil root.walk(fn) sort.Strings(result) // sort as there is a map traversal involved assert.Equal(t, []string{"", "dir", "fil", "fil/a", "fil/a/b"}, result) assert.Nil(t, root.cachedDir("not found")) if dir := root.cachedDir("dir"); assert.NotNil(t, dir) { result = nil dir.walk(fn) assert.Equal(t, []string{"dir"}, result) } if dir := root.cachedDir("fil"); assert.NotNil(t, dir) { result = nil dir.walk(fn) assert.Equal(t, []string{"fil/a/b", "fil/a", "fil"}, result) } if dir := fil.(*Dir); assert.NotNil(t, dir) { result = nil dir.walk(fn) assert.Equal(t, []string{"fil/a/b", "fil/a", "fil"}, result) } if dir := root.cachedDir("fil/a"); assert.NotNil(t, dir) { result = nil dir.walk(fn) assert.Equal(t, []string{"fil/a/b", "fil/a"}, result) } if dir := fil.(*Dir).cachedDir("a"); assert.NotNil(t, dir) { result = nil dir.walk(fn) assert.Equal(t, []string{"fil/a/b", "fil/a"}, result) } if dir := root.cachedDir("fil/a"); assert.NotNil(t, dir) { result = nil dir.walk(fn) assert.Equal(t, []string{"fil/a/b", "fil/a"}, result) } if dir := root.cachedDir("fil/a/b"); assert.NotNil(t, dir) { result = nil dir.walk(fn) assert.Equal(t, []string{"fil/a/b"}, result) } } func TestDirSetModTime(t *testing.T) { _, vfs, dir, _ := dirCreate(t) err := dir.SetModTime(t1) require.NoError(t, err) assert.WithinDuration(t, t1, dir.ModTime(), time.Second) err = dir.SetModTime(t2) require.NoError(t, err) assert.WithinDuration(t, t2, dir.ModTime(), time.Second) vfs.Opt.ReadOnly = true err = dir.SetModTime(t2) assert.Equal(t, EROFS, err) } func TestDirStat(t *testing.T) { _, _, dir, _ := dirCreate(t) node, err := dir.Stat("file1") require.NoError(t, err) _, ok := node.(*File) assert.True(t, ok) assert.Equal(t, int64(14), node.Size()) assert.Equal(t, "file1", node.Name()) _, err = dir.Stat("not found") assert.Equal(t, ENOENT, err) } // This lists dir and checks the listing is as expected func checkListing(t *testing.T, dir *Dir, want []string) { var got []string nodes, err := dir.ReadDirAll() require.NoError(t, err) for _, node := range nodes { got = append(got, fmt.Sprintf("%s,%d,%v", node.Name(), node.Size(), node.IsDir())) } assert.Equal(t, want, got) } func TestDirReadDirAll(t *testing.T) { r, vfs := newTestVFS(t) file1 := r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1) file2 := r.WriteObject(context.Background(), "dir/file2", "file2- contents", t2) file3 := r.WriteObject(context.Background(), "dir/subdir/file3", "file3-- contents", t3) r.CheckRemoteItems(t, file1, file2, file3) node, err := vfs.Stat("dir") require.NoError(t, err) dir := node.(*Dir) checkListing(t, dir, []string{"file1,14,false", "file2,15,false", "subdir,0,true"}) node, err = vfs.Stat("") require.NoError(t, err) root := node.(*Dir) checkListing(t, root, []string{"dir,0,true"}) node, err = vfs.Stat("dir/subdir") require.NoError(t, err) subdir := node.(*Dir) checkListing(t, subdir, []string{"file3,16,false"}) t.Run("Virtual", func(t *testing.T) { // Add some virtual entries and check what happens dir.AddVirtual("virtualFile", 17, false) dir.AddVirtual("virtualDir", 0, true) // Remove some existing entries dir.DelVirtual("file2") dir.DelVirtual("subdir") checkListing(t, dir, []string{"file1,14,false", "virtualDir,0,true", "virtualFile,17,false"}) // Now action the deletes and uploads _ = r.WriteObject(context.Background(), "dir/virtualFile", "virtualFile contents", t1) _ = r.WriteObject(context.Background(), "dir/virtualDir/testFile", "testFile contents", t1) o, err := r.Fremote.NewObject(context.Background(), "dir/file2") require.NoError(t, err) require.NoError(t, o.Remove(context.Background())) require.NoError(t, operations.Purge(context.Background(), r.Fremote, "dir/subdir")) // Force a directory reload... dir.invalidateDir("dir") checkListing(t, dir, []string{"file1,14,false", "virtualDir,0,true", "virtualFile,20,false"}) // check no virtuals left dir.mu.Lock() assert.Nil(t, dir.virtual) dir.mu.Unlock() // Add some virtual entries and check what happens dir.AddVirtual("virtualFile2", 100, false) dir.AddVirtual("virtualDir2", 0, true) // Remove some existing entries dir.DelVirtual("file1") checkListing(t, dir, []string{"virtualDir,0,true", "virtualDir2,0,true", "virtualFile,20,false", "virtualFile2,100,false"}) // Force a directory reload... dir.invalidateDir("dir") want := []string{"file1,14,false", "virtualDir,0,true", "virtualDir2,0,true", "virtualFile,20,false", "virtualFile2,100,false"} features := r.Fremote.Features() if features.CanHaveEmptyDirectories { // snip out virtualDir2 which will only be present if can't have empty dirs want = append(want[:2], want[3:]...) } checkListing(t, dir, want) // Check that forgetting the root doesn't invalidate the virtual entries root.ForgetAll() checkListing(t, dir, want) }) } func TestDirOpen(t *testing.T) { _, _, dir, _ := dirCreate(t) fd, err := dir.Open(os.O_RDONLY) require.NoError(t, err) _, ok := fd.(*DirHandle) assert.True(t, ok) require.NoError(t, fd.Close()) _, err = dir.Open(os.O_WRONLY) assert.Equal(t, EPERM, err) } func TestDirCreate(t *testing.T) { _, vfs, dir, _ := dirCreate(t) origModTime := dir.ModTime() time.Sleep(100 * time.Millisecond) // for low rez Windows timers file, err := dir.Create("potato", os.O_WRONLY|os.O_CREATE) require.NoError(t, err) assert.Equal(t, int64(0), file.Size()) assert.True(t, dir.ModTime().After(origModTime)) fd, err := file.Open(os.O_WRONLY | os.O_CREATE) require.NoError(t, err) // FIXME Note that this fails with the current implementation // until the file has been opened. // file2, err := vfs.Stat("dir/potato") // require.NoError(t, err) // assert.Equal(t, file, file2) n, err := fd.Write([]byte("hello")) require.NoError(t, err) assert.Equal(t, 5, n) require.NoError(t, fd.Close()) file2, err := vfs.Stat("dir/potato") require.NoError(t, err) assert.Equal(t, int64(5), file2.Size()) // Try creating the file again - make sure we get the same file node file3, err := dir.Create("potato", os.O_RDWR|os.O_CREATE) require.NoError(t, err) assert.Equal(t, int64(5), file3.Size()) assert.Equal(t, fmt.Sprintf("%p", file), fmt.Sprintf("%p", file3), "didn't return same node") // Test read only fs creating new vfs.Opt.ReadOnly = true _, err = dir.Create("sausage", os.O_WRONLY|os.O_CREATE) assert.Equal(t, EROFS, err) } func TestDirMkdir(t *testing.T) { r, vfs, dir, file1 := dirCreate(t) _, err := dir.Mkdir("file1") assert.Error(t, err) origModTime := dir.ModTime() time.Sleep(100 * time.Millisecond) // for low rez Windows timers sub, err := dir.Mkdir("sub") assert.NoError(t, err) assert.True(t, dir.ModTime().After(origModTime)) // check the vfs checkListing(t, dir, []string{"file1,14,false", "sub,0,true"}) checkListing(t, sub, []string(nil)) // check the underlying r.Fremote fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"dir", "dir/sub"}, r.Fremote.Precision()) vfs.Opt.ReadOnly = true _, err = dir.Mkdir("sausage") assert.Equal(t, EROFS, err) } func TestDirMkdirSub(t *testing.T) { r, vfs, dir, file1 := dirCreate(t) _, err := dir.Mkdir("file1") assert.Error(t, err) sub, err := dir.Mkdir("sub") assert.NoError(t, err) subsub, err := sub.Mkdir("subsub") assert.NoError(t, err) // check the vfs checkListing(t, dir, []string{"file1,14,false", "sub,0,true"}) checkListing(t, sub, []string{"subsub,0,true"}) checkListing(t, subsub, []string(nil)) // check the underlying r.Fremote fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"dir", "dir/sub", "dir/sub/subsub"}, r.Fremote.Precision()) vfs.Opt.ReadOnly = true _, err = dir.Mkdir("sausage") assert.Equal(t, EROFS, err) } func TestDirRemove(t *testing.T) { r, vfs, dir, _ := dirCreate(t) // check directory is there node, err := vfs.Stat("dir") require.NoError(t, err) assert.True(t, node.IsDir()) err = dir.Remove() assert.Equal(t, ENOTEMPTY, err) // Delete the sub file node, err = vfs.Stat("dir/file1") require.NoError(t, err) err = node.Remove() require.NoError(t, err) // Remove the now empty directory err = dir.Remove() require.NoError(t, err) // check directory is not there _, err = vfs.Stat("dir") assert.Equal(t, ENOENT, err) // check the vfs root, err := vfs.Root() require.NoError(t, err) checkListing(t, root, []string(nil)) // check the underlying r.Fremote fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, r.Fremote.Precision()) // read only check vfs.Opt.ReadOnly = true err = dir.Remove() assert.Equal(t, EROFS, err) } func TestDirRemoveAll(t *testing.T) { r, vfs, dir, _ := dirCreate(t) // Remove the directory and contents err := dir.RemoveAll() require.NoError(t, err) // check the vfs root, err := vfs.Root() require.NoError(t, err) checkListing(t, root, []string(nil)) // check the underlying r.Fremote fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, r.Fremote.Precision()) // read only check vfs.Opt.ReadOnly = true err = dir.RemoveAll() assert.Equal(t, EROFS, err) } func TestDirRemoveName(t *testing.T) { r, vfs, dir, _ := dirCreate(t) origModTime := dir.ModTime() time.Sleep(100 * time.Millisecond) // for low rez Windows timers err := dir.RemoveName("file1") require.NoError(t, err) assert.True(t, dir.ModTime().After(origModTime)) checkListing(t, dir, []string(nil)) root, err := vfs.Root() require.NoError(t, err) checkListing(t, root, []string{"dir,0,true"}) // check the underlying r.Fremote fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"dir"}, r.Fremote.Precision()) // read only check vfs.Opt.ReadOnly = true err = dir.RemoveName("potato") assert.Equal(t, EROFS, err) } func TestDirRename(t *testing.T) { r, vfs, dir, file1 := dirCreate(t) features := r.Fremote.Features() if features.DirMove == nil && features.Move == nil && features.Copy == nil { t.Skip("can't rename directories") } file3 := r.WriteObject(context.Background(), "dir/file3", "file3 contents!", t1) fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file3}, []string{"dir"}, r.Fremote.Precision()) root, err := vfs.Root() require.NoError(t, err) err = dir.Rename("not found", "tuba", dir) assert.Equal(t, ENOENT, err) // Rename a directory err = root.Rename("dir", "dir2", root) assert.NoError(t, err) checkListing(t, root, []string{"dir2,0,true"}) checkListing(t, dir, []string{"file1,14,false", "file3,15,false"}) // check the underlying r.Fremote file1.Path = "dir2/file1" file3.Path = "dir2/file3" fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file3}, []string{"dir2"}, r.Fremote.Precision()) // refetch dir node, err := vfs.Stat("dir2") assert.NoError(t, err) dir = node.(*Dir) // Rename a file origModTime := dir.ModTime() time.Sleep(100 * time.Millisecond) // for low rez Windows timers err = dir.Rename("file1", "file2", root) assert.NoError(t, err) assert.True(t, dir.ModTime().After(origModTime)) checkListing(t, root, []string{"dir2,0,true", "file2,14,false"}) checkListing(t, dir, []string{"file3,15,false"}) // check the underlying r.Fremote file1.Path = "file2" fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file3}, []string{"dir2"}, r.Fremote.Precision()) // Rename a file on top of another file err = root.Rename("file2", "file3", dir) assert.NoError(t, err) checkListing(t, root, []string{"dir2,0,true"}) checkListing(t, dir, []string{"file3,14,false"}) // check the underlying r.Fremote file1.Path = "dir2/file3" fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"dir2"}, r.Fremote.Precision()) // rename an empty directory _, err = root.Mkdir("empty directory") assert.NoError(t, err) checkListing(t, root, []string{ "dir2,0,true", "empty directory,0,true", }) err = root.Rename("empty directory", "renamed empty directory", root) assert.NoError(t, err) checkListing(t, root, []string{ "dir2,0,true", "renamed empty directory,0,true", }) // ...we don't check the underlying f.Fremote because on // bucket-based remotes the directory won't be there // read only check vfs.Opt.ReadOnly = true err = dir.Rename("potato", "tuba", dir) assert.Equal(t, EROFS, err) } func TestDirStructSize(t *testing.T) { t.Logf("Dir struct has size %d bytes", unsafe.Sizeof(Dir{})) }