rclone/vfs/dir_test.go
David Pedersen 071c3f28e5 vfs: Update parent directory modtimes on vfs actions
This isn't written back to the storage so might change on remount but
makes the VFS just a little more POSIX compatible.
2023-09-08 17:19:52 +01:00

597 lines
16 KiB
Go

package vfs
import (
"context"
"fmt"
"os"
"sort"
"testing"
"time"
"unsafe"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func dirCreate(t *testing.T) (r *fstest.Run, vfs *VFS, dir *Dir, item fstest.Item) {
r, vfs = newTestVFS(t)
file1 := r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1)
r.CheckRemoteItems(t, file1)
node, err := vfs.Stat("dir")
require.NoError(t, err)
require.True(t, node.IsDir())
return r, vfs, node.(*Dir), file1
}
func TestDirMethods(t *testing.T) {
_, vfs, dir, _ := dirCreate(t)
// String
assert.Equal(t, "dir/", dir.String())
assert.Equal(t, "<nil *Dir>", (*Dir)(nil).String())
// IsDir
assert.Equal(t, true, dir.IsDir())
// IsFile
assert.Equal(t, false, dir.IsFile())
// Mode
assert.Equal(t, vfs.Opt.DirPerms, dir.Mode())
// Name
assert.Equal(t, "dir", dir.Name())
// Path
assert.Equal(t, "dir", dir.Path())
// Sys
assert.Equal(t, nil, dir.Sys())
// SetSys
dir.SetSys(42)
assert.Equal(t, 42, dir.Sys())
// Inode
assert.NotEqual(t, uint64(0), dir.Inode())
// Node
assert.Equal(t, dir, dir.Node())
// ModTime
assert.WithinDuration(t, t1, dir.ModTime(), 100*365*24*60*60*time.Second)
// Size
assert.Equal(t, int64(0), dir.Size())
// Sync
assert.NoError(t, dir.Sync())
// DirEntry
assert.Equal(t, dir.entry, dir.DirEntry())
// VFS
assert.Equal(t, vfs, dir.VFS())
}
func TestDirForgetAll(t *testing.T) {
_, vfs, dir, file1 := dirCreate(t)
// Make sure / and dir are in cache
_, err := vfs.Stat(file1.Path)
require.NoError(t, err)
root, err := vfs.Root()
require.NoError(t, err)
assert.Equal(t, 1, len(root.items))
assert.Equal(t, 1, len(dir.items))
assert.False(t, root.read.IsZero())
assert.False(t, dir.read.IsZero())
dir.ForgetAll()
assert.Equal(t, 1, len(root.items))
assert.Equal(t, 0, len(dir.items))
assert.False(t, root.read.IsZero())
assert.True(t, dir.read.IsZero())
root.ForgetAll()
assert.Equal(t, 0, len(root.items))
assert.Equal(t, 0, len(dir.items))
assert.True(t, root.read.IsZero())
}
func TestDirForgetPath(t *testing.T) {
_, vfs, dir, file1 := dirCreate(t)
// Make sure / and dir are in cache
_, err := vfs.Stat(file1.Path)
require.NoError(t, err)
root, err := vfs.Root()
require.NoError(t, err)
assert.Equal(t, 1, len(root.items))
assert.Equal(t, 1, len(dir.items))
assert.False(t, root.read.IsZero())
assert.False(t, dir.read.IsZero())
root.ForgetPath("dir/notfound", fs.EntryObject)
assert.Equal(t, 1, len(root.items))
assert.Equal(t, 1, len(dir.items))
assert.False(t, root.read.IsZero())
assert.True(t, dir.read.IsZero())
root.ForgetPath("dir", fs.EntryDirectory)
assert.Equal(t, 1, len(root.items))
assert.Equal(t, 0, len(dir.items))
assert.True(t, root.read.IsZero())
root.ForgetPath("not/in/cache", fs.EntryDirectory)
assert.Equal(t, 1, len(root.items))
assert.Equal(t, 0, len(dir.items))
}
func TestDirWalk(t *testing.T) {
r, vfs, _, file1 := dirCreate(t)
file2 := r.WriteObject(context.Background(), "fil/a/b/c", "super long file", t1)
r.CheckRemoteItems(t, file1, file2)
root, err := vfs.Root()
require.NoError(t, err)
// Forget the cache since we put another object in
root.ForgetAll()
// Read the directories in
_, err = vfs.Stat("dir")
require.NoError(t, err)
_, err = vfs.Stat("fil/a/b")
require.NoError(t, err)
fil, err := vfs.Stat("fil")
require.NoError(t, err)
var result []string
fn := func(d *Dir) {
result = append(result, d.path)
}
result = nil
root.walk(fn)
sort.Strings(result) // sort as there is a map traversal involved
assert.Equal(t, []string{"", "dir", "fil", "fil/a", "fil/a/b"}, result)
assert.Nil(t, root.cachedDir("not found"))
if dir := root.cachedDir("dir"); assert.NotNil(t, dir) {
result = nil
dir.walk(fn)
assert.Equal(t, []string{"dir"}, result)
}
if dir := root.cachedDir("fil"); assert.NotNil(t, dir) {
result = nil
dir.walk(fn)
assert.Equal(t, []string{"fil/a/b", "fil/a", "fil"}, result)
}
if dir := fil.(*Dir); assert.NotNil(t, dir) {
result = nil
dir.walk(fn)
assert.Equal(t, []string{"fil/a/b", "fil/a", "fil"}, result)
}
if dir := root.cachedDir("fil/a"); assert.NotNil(t, dir) {
result = nil
dir.walk(fn)
assert.Equal(t, []string{"fil/a/b", "fil/a"}, result)
}
if dir := fil.(*Dir).cachedDir("a"); assert.NotNil(t, dir) {
result = nil
dir.walk(fn)
assert.Equal(t, []string{"fil/a/b", "fil/a"}, result)
}
if dir := root.cachedDir("fil/a"); assert.NotNil(t, dir) {
result = nil
dir.walk(fn)
assert.Equal(t, []string{"fil/a/b", "fil/a"}, result)
}
if dir := root.cachedDir("fil/a/b"); assert.NotNil(t, dir) {
result = nil
dir.walk(fn)
assert.Equal(t, []string{"fil/a/b"}, result)
}
}
func TestDirSetModTime(t *testing.T) {
_, vfs, dir, _ := dirCreate(t)
err := dir.SetModTime(t1)
require.NoError(t, err)
assert.WithinDuration(t, t1, dir.ModTime(), time.Second)
err = dir.SetModTime(t2)
require.NoError(t, err)
assert.WithinDuration(t, t2, dir.ModTime(), time.Second)
vfs.Opt.ReadOnly = true
err = dir.SetModTime(t2)
assert.Equal(t, EROFS, err)
}
func TestDirStat(t *testing.T) {
_, _, dir, _ := dirCreate(t)
node, err := dir.Stat("file1")
require.NoError(t, err)
_, ok := node.(*File)
assert.True(t, ok)
assert.Equal(t, int64(14), node.Size())
assert.Equal(t, "file1", node.Name())
_, err = dir.Stat("not found")
assert.Equal(t, ENOENT, err)
}
// This lists dir and checks the listing is as expected
func checkListing(t *testing.T, dir *Dir, want []string) {
var got []string
nodes, err := dir.ReadDirAll()
require.NoError(t, err)
for _, node := range nodes {
got = append(got, fmt.Sprintf("%s,%d,%v", node.Name(), node.Size(), node.IsDir()))
}
assert.Equal(t, want, got)
}
func TestDirReadDirAll(t *testing.T) {
r, vfs := newTestVFS(t)
file1 := r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1)
file2 := r.WriteObject(context.Background(), "dir/file2", "file2- contents", t2)
file3 := r.WriteObject(context.Background(), "dir/subdir/file3", "file3-- contents", t3)
r.CheckRemoteItems(t, file1, file2, file3)
node, err := vfs.Stat("dir")
require.NoError(t, err)
dir := node.(*Dir)
checkListing(t, dir, []string{"file1,14,false", "file2,15,false", "subdir,0,true"})
node, err = vfs.Stat("")
require.NoError(t, err)
root := node.(*Dir)
checkListing(t, root, []string{"dir,0,true"})
node, err = vfs.Stat("dir/subdir")
require.NoError(t, err)
subdir := node.(*Dir)
checkListing(t, subdir, []string{"file3,16,false"})
t.Run("Virtual", func(t *testing.T) {
// Add some virtual entries and check what happens
dir.AddVirtual("virtualFile", 17, false)
dir.AddVirtual("virtualDir", 0, true)
// Remove some existing entries
dir.DelVirtual("file2")
dir.DelVirtual("subdir")
checkListing(t, dir, []string{"file1,14,false", "virtualDir,0,true", "virtualFile,17,false"})
// Now action the deletes and uploads
_ = r.WriteObject(context.Background(), "dir/virtualFile", "virtualFile contents", t1)
_ = r.WriteObject(context.Background(), "dir/virtualDir/testFile", "testFile contents", t1)
o, err := r.Fremote.NewObject(context.Background(), "dir/file2")
require.NoError(t, err)
require.NoError(t, o.Remove(context.Background()))
require.NoError(t, operations.Purge(context.Background(), r.Fremote, "dir/subdir"))
// Force a directory reload...
dir.invalidateDir("dir")
checkListing(t, dir, []string{"file1,14,false", "virtualDir,0,true", "virtualFile,20,false"})
// check no virtuals left
dir.mu.Lock()
assert.Nil(t, dir.virtual)
dir.mu.Unlock()
// Add some virtual entries and check what happens
dir.AddVirtual("virtualFile2", 100, false)
dir.AddVirtual("virtualDir2", 0, true)
// Remove some existing entries
dir.DelVirtual("file1")
checkListing(t, dir, []string{"virtualDir,0,true", "virtualDir2,0,true", "virtualFile,20,false", "virtualFile2,100,false"})
// Force a directory reload...
dir.invalidateDir("dir")
want := []string{"file1,14,false", "virtualDir,0,true", "virtualDir2,0,true", "virtualFile,20,false", "virtualFile2,100,false"}
features := r.Fremote.Features()
if features.CanHaveEmptyDirectories {
// snip out virtualDir2 which will only be present if can't have empty dirs
want = append(want[:2], want[3:]...)
}
checkListing(t, dir, want)
// Check that forgetting the root doesn't invalidate the virtual entries
root.ForgetAll()
checkListing(t, dir, want)
})
}
func TestDirOpen(t *testing.T) {
_, _, dir, _ := dirCreate(t)
fd, err := dir.Open(os.O_RDONLY)
require.NoError(t, err)
_, ok := fd.(*DirHandle)
assert.True(t, ok)
require.NoError(t, fd.Close())
_, err = dir.Open(os.O_WRONLY)
assert.Equal(t, EPERM, err)
}
func TestDirCreate(t *testing.T) {
_, vfs, dir, _ := dirCreate(t)
origModTime := dir.ModTime()
time.Sleep(100 * time.Millisecond) // for low rez Windows timers
file, err := dir.Create("potato", os.O_WRONLY|os.O_CREATE)
require.NoError(t, err)
assert.Equal(t, int64(0), file.Size())
assert.True(t, dir.ModTime().After(origModTime))
fd, err := file.Open(os.O_WRONLY | os.O_CREATE)
require.NoError(t, err)
// FIXME Note that this fails with the current implementation
// until the file has been opened.
// file2, err := vfs.Stat("dir/potato")
// require.NoError(t, err)
// assert.Equal(t, file, file2)
n, err := fd.Write([]byte("hello"))
require.NoError(t, err)
assert.Equal(t, 5, n)
require.NoError(t, fd.Close())
file2, err := vfs.Stat("dir/potato")
require.NoError(t, err)
assert.Equal(t, int64(5), file2.Size())
// Try creating the file again - make sure we get the same file node
file3, err := dir.Create("potato", os.O_RDWR|os.O_CREATE)
require.NoError(t, err)
assert.Equal(t, int64(5), file3.Size())
assert.Equal(t, fmt.Sprintf("%p", file), fmt.Sprintf("%p", file3), "didn't return same node")
// Test read only fs creating new
vfs.Opt.ReadOnly = true
_, err = dir.Create("sausage", os.O_WRONLY|os.O_CREATE)
assert.Equal(t, EROFS, err)
}
func TestDirMkdir(t *testing.T) {
r, vfs, dir, file1 := dirCreate(t)
_, err := dir.Mkdir("file1")
assert.Error(t, err)
origModTime := dir.ModTime()
time.Sleep(100 * time.Millisecond) // for low rez Windows timers
sub, err := dir.Mkdir("sub")
assert.NoError(t, err)
assert.True(t, dir.ModTime().After(origModTime))
// check the vfs
checkListing(t, dir, []string{"file1,14,false", "sub,0,true"})
checkListing(t, sub, []string(nil))
// check the underlying r.Fremote
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"dir", "dir/sub"}, r.Fremote.Precision())
vfs.Opt.ReadOnly = true
_, err = dir.Mkdir("sausage")
assert.Equal(t, EROFS, err)
}
func TestDirMkdirSub(t *testing.T) {
r, vfs, dir, file1 := dirCreate(t)
_, err := dir.Mkdir("file1")
assert.Error(t, err)
sub, err := dir.Mkdir("sub")
assert.NoError(t, err)
subsub, err := sub.Mkdir("subsub")
assert.NoError(t, err)
// check the vfs
checkListing(t, dir, []string{"file1,14,false", "sub,0,true"})
checkListing(t, sub, []string{"subsub,0,true"})
checkListing(t, subsub, []string(nil))
// check the underlying r.Fremote
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"dir", "dir/sub", "dir/sub/subsub"}, r.Fremote.Precision())
vfs.Opt.ReadOnly = true
_, err = dir.Mkdir("sausage")
assert.Equal(t, EROFS, err)
}
func TestDirRemove(t *testing.T) {
r, vfs, dir, _ := dirCreate(t)
// check directory is there
node, err := vfs.Stat("dir")
require.NoError(t, err)
assert.True(t, node.IsDir())
err = dir.Remove()
assert.Equal(t, ENOTEMPTY, err)
// Delete the sub file
node, err = vfs.Stat("dir/file1")
require.NoError(t, err)
err = node.Remove()
require.NoError(t, err)
// Remove the now empty directory
err = dir.Remove()
require.NoError(t, err)
// check directory is not there
_, err = vfs.Stat("dir")
assert.Equal(t, ENOENT, err)
// check the vfs
root, err := vfs.Root()
require.NoError(t, err)
checkListing(t, root, []string(nil))
// check the underlying r.Fremote
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, r.Fremote.Precision())
// read only check
vfs.Opt.ReadOnly = true
err = dir.Remove()
assert.Equal(t, EROFS, err)
}
func TestDirRemoveAll(t *testing.T) {
r, vfs, dir, _ := dirCreate(t)
// Remove the directory and contents
err := dir.RemoveAll()
require.NoError(t, err)
// check the vfs
root, err := vfs.Root()
require.NoError(t, err)
checkListing(t, root, []string(nil))
// check the underlying r.Fremote
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, r.Fremote.Precision())
// read only check
vfs.Opt.ReadOnly = true
err = dir.RemoveAll()
assert.Equal(t, EROFS, err)
}
func TestDirRemoveName(t *testing.T) {
r, vfs, dir, _ := dirCreate(t)
origModTime := dir.ModTime()
time.Sleep(100 * time.Millisecond) // for low rez Windows timers
err := dir.RemoveName("file1")
require.NoError(t, err)
assert.True(t, dir.ModTime().After(origModTime))
checkListing(t, dir, []string(nil))
root, err := vfs.Root()
require.NoError(t, err)
checkListing(t, root, []string{"dir,0,true"})
// check the underlying r.Fremote
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"dir"}, r.Fremote.Precision())
// read only check
vfs.Opt.ReadOnly = true
err = dir.RemoveName("potato")
assert.Equal(t, EROFS, err)
}
func TestDirRename(t *testing.T) {
r, vfs, dir, file1 := dirCreate(t)
features := r.Fremote.Features()
if features.DirMove == nil && features.Move == nil && features.Copy == nil {
t.Skip("can't rename directories")
}
file3 := r.WriteObject(context.Background(), "dir/file3", "file3 contents!", t1)
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file3}, []string{"dir"}, r.Fremote.Precision())
root, err := vfs.Root()
require.NoError(t, err)
err = dir.Rename("not found", "tuba", dir)
assert.Equal(t, ENOENT, err)
// Rename a directory
err = root.Rename("dir", "dir2", root)
assert.NoError(t, err)
checkListing(t, root, []string{"dir2,0,true"})
checkListing(t, dir, []string{"file1,14,false", "file3,15,false"})
// check the underlying r.Fremote
file1.Path = "dir2/file1"
file3.Path = "dir2/file3"
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file3}, []string{"dir2"}, r.Fremote.Precision())
// refetch dir
node, err := vfs.Stat("dir2")
assert.NoError(t, err)
dir = node.(*Dir)
// Rename a file
origModTime := dir.ModTime()
time.Sleep(100 * time.Millisecond) // for low rez Windows timers
err = dir.Rename("file1", "file2", root)
assert.NoError(t, err)
assert.True(t, dir.ModTime().After(origModTime))
checkListing(t, root, []string{"dir2,0,true", "file2,14,false"})
checkListing(t, dir, []string{"file3,15,false"})
// check the underlying r.Fremote
file1.Path = "file2"
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file3}, []string{"dir2"}, r.Fremote.Precision())
// Rename a file on top of another file
err = root.Rename("file2", "file3", dir)
assert.NoError(t, err)
checkListing(t, root, []string{"dir2,0,true"})
checkListing(t, dir, []string{"file3,14,false"})
// check the underlying r.Fremote
file1.Path = "dir2/file3"
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"dir2"}, r.Fremote.Precision())
// rename an empty directory
_, err = root.Mkdir("empty directory")
assert.NoError(t, err)
checkListing(t, root, []string{
"dir2,0,true",
"empty directory,0,true",
})
err = root.Rename("empty directory", "renamed empty directory", root)
assert.NoError(t, err)
checkListing(t, root, []string{
"dir2,0,true",
"renamed empty directory,0,true",
})
// ...we don't check the underlying f.Fremote because on
// bucket-based remotes the directory won't be there
// read only check
vfs.Opt.ReadOnly = true
err = dir.Rename("potato", "tuba", dir)
assert.Equal(t, EROFS, err)
}
func TestDirStructSize(t *testing.T) {
t.Logf("Dir struct has size %d bytes", unsafe.Sizeof(Dir{}))
}