vfs: convert vfs options to new style

This also
- move in use options (Opt) from vfsflags to vfscommon
- change os.FileMode to vfscommon.FileMode in parameters
- rework vfscommon.FileMode and add tests
This commit is contained in:
Nick Craig-Wood 2024-07-03 11:34:29 +01:00
parent fc1d8dafd5
commit a28287e96d
39 changed files with 408 additions and 236 deletions

View File

@ -33,7 +33,7 @@ import (
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/testy"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/require"
)
@ -123,10 +123,10 @@ func TestInternalListRootAndInnerRemotes(t *testing.T) {
/* TODO: is this testing something?
func TestInternalVfsCache(t *testing.T) {
vfsflags.Opt.DirCacheTime = time.Second * 30
vfscommon.Opt.DirCacheTime = time.Second * 30
testSize := int64(524288000)
vfsflags.Opt.CacheMode = vfs.CacheModeWrites
vfscommon.Opt.CacheMode = vfs.CacheModeWrites
id := "tiuufo"
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"})
defer runInstance.cleanupFs(t, rootFs, boltDb)
@ -338,7 +338,7 @@ func TestInternalCachedUpdatedContentMatches(t *testing.T) {
func TestInternalWrappedWrittenContentMatches(t *testing.T) {
id := fmt.Sprintf("tiwwcm%v", time.Now().Unix())
vfsflags.Opt.DirCacheTime = fs.Duration(time.Second)
vfscommon.Opt.DirCacheTime = fs.Duration(time.Second)
rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
if runInstance.rootIsCrypt {
t.Skip("test skipped with crypt remote")
@ -368,7 +368,7 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
func TestInternalLargeWrittenContentMatches(t *testing.T) {
id := fmt.Sprintf("tilwcm%v", time.Now().Unix())
vfsflags.Opt.DirCacheTime = fs.Duration(time.Second)
vfscommon.Opt.DirCacheTime = fs.Duration(time.Second)
rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
if runInstance.rootIsCrypt {
t.Skip("test skipped with crypt remote")
@ -708,7 +708,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
func TestInternalExpiredEntriesRemoved(t *testing.T) {
id := fmt.Sprintf("tieer%v", time.Now().Unix())
vfsflags.Opt.DirCacheTime = fs.Duration(time.Second * 4) // needs to be lower than the defined
vfscommon.Opt.DirCacheTime = fs.Duration(time.Second * 4) // needs to be lower than the defined
rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
cfs, err := runInstance.getCacheFs(rootFs)
require.NoError(t, err)
@ -743,7 +743,7 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) {
}
func TestInternalBug2117(t *testing.T) {
vfsflags.Opt.DirCacheTime = fs.Duration(time.Second * 10)
vfscommon.Opt.DirCacheTime = fs.Duration(time.Second * 10)
id := fmt.Sprintf("tib2117%v", time.Now().Unix())
rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})

View File

@ -33,7 +33,7 @@ func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) (err error) {
a.Valid = time.Duration(d.fsys.opt.AttrTimeout)
a.Gid = d.VFS().Opt.GID
a.Uid = d.VFS().Opt.UID
a.Mode = os.ModeDir | d.VFS().Opt.DirPerms
a.Mode = os.ModeDir | os.FileMode(d.VFS().Opt.DirPerms)
modTime := d.ModTime()
a.Atime = modTime
a.Mtime = modTime

View File

@ -4,6 +4,7 @@ package mount
import (
"context"
"os"
"syscall"
"time"
@ -31,7 +32,7 @@ func (f *File) Attr(ctx context.Context, a *fuse.Attr) (err error) {
Blocks := (Size + 511) / 512
a.Gid = f.VFS().Opt.GID
a.Uid = f.VFS().Opt.UID
a.Mode = f.VFS().Opt.FilePerms
a.Mode = os.FileMode(f.VFS().Opt.FilePerms)
a.Size = Size
a.Atime = modTime
a.Mtime = modTime

View File

@ -228,7 +228,7 @@ func NewMountCommand(commandName string, hidden bool, mount MountFn) *cobra.Comm
defer cmd.StartStats()()
}
mnt := NewMountPoint(mount, args[1], cmd.NewFsDir(args), &Opt, &vfsflags.Opt)
mnt := NewMountPoint(mount, args[1], cmd.NewFsDir(args), &Opt, &vfscommon.Opt)
mountDaemon, err := mnt.Mount()
// Wait for foreground mount, if any...

View File

@ -10,7 +10,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/rclone/rclone/vfs/vfscommon"
)
var (
@ -85,7 +85,7 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) {
return nil, err
}
vfsOpt := vfsflags.Opt
vfsOpt := vfscommon.Opt
err = in.GetStructMissingOK("vfsOpt", &vfsOpt)
if err != nil {
return nil, err

View File

@ -26,6 +26,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/systemd"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
)
@ -134,7 +135,7 @@ func newServer(f fs.Fs, opt *dlnaflags.Options) (*server, error) {
waitChan: make(chan struct{}),
httpListenAddr: opt.ListenAddr,
f: f,
vfs: vfs.New(f, &vfsflags.Opt),
vfs: vfs.New(f, &vfscommon.Opt),
}
s.services = map[string]UPnPService{

View File

@ -19,7 +19,6 @@ import (
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/file"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/rclone/rclone/vfs/vfsflags"
)
// Driver implements docker driver api
@ -55,7 +54,7 @@ func NewDriver(ctx context.Context, root string, mntOpt *mountlib.Options, vfsOp
mntOpt = &mountlib.Opt
}
if vfsOpt == nil {
vfsOpt = &vfsflags.Opt
vfsOpt = &vfscommon.Opt
}
drv := &Driver{
root: root,

View File

@ -2,7 +2,6 @@ package docker
import (
"fmt"
"strconv"
"strings"
"github.com/rclone/rclone/cmd/mountlib"
@ -11,7 +10,6 @@ import (
"github.com/rclone/rclone/fs/fspath"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/spf13/pflag"
)
@ -265,22 +263,13 @@ func getVFSOption(vfsOpt *vfscommon.Options, opt rc.Params, key string) (ok bool
case "read-only":
vfsOpt.ReadOnly, err = opt.GetBool(key)
case "dir-perms":
perms := &vfsflags.FileMode{Mode: &vfsOpt.DirPerms}
err = getFVarP(perms, opt, key)
err = getFVarP(&vfsOpt.DirPerms, opt, key)
case "file-perms":
perms := &vfsflags.FileMode{Mode: &vfsOpt.FilePerms}
err = getFVarP(perms, opt, key)
err = getFVarP(&vfsOpt.FilePerms, opt, key)
// unprefixed unix-only vfs options
case "umask":
// GetInt64 doesn't support the `0octal` umask syntax - parse locally
var strVal string
if strVal, err = opt.GetString(key); err == nil {
var longVal int64
if longVal, err = strconv.ParseInt(strVal, 0, 0); err == nil {
vfsOpt.Umask = int(longVal)
}
}
err = getFVarP(&vfsOpt.Umask, opt, key)
case "uid":
intVal, err = opt.GetInt64(key)
vfsOpt.UID = uint32(intVal)

View File

@ -27,6 +27,7 @@ import (
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -157,7 +158,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (*driver, error) {
d.proxy = proxy.New(ctx, &proxyflags.Opt)
d.userPass = make(map[string]string, 16)
} else {
d.globalVFS = vfs.New(f, &vfsflags.Opt)
d.globalVFS = vfs.New(f, &vfscommon.Opt)
}
d.useTLS = d.opt.TLSKey != ""

View File

@ -24,6 +24,7 @@ import (
"github.com/rclone/rclone/lib/http/serve"
"github.com/rclone/rclone/lib/systemd"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
)
@ -148,7 +149,7 @@ func run(ctx context.Context, f fs.Fs, opt Options) (s *HTTP, err error) {
// override auth
s.opt.Auth.CustomAuthFn = s.auth
} else {
s._vfs = vfs.New(f, &vfsflags.Opt)
s._vfs = vfs.New(f, &vfscommon.Opt)
}
s.server, err = libhttp.NewServer(ctx,
@ -215,7 +216,7 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string
// Make the entries for display
directory := serve.NewDirectory(dirRemote, s.server.HTMLTemplate())
for _, node := range dirEntries {
if vfsflags.Opt.NoModTime {
if vfscommon.Opt.NoModTime {
directory.AddHTMLEntry(node.Path(), node.IsDir(), node.Size(), time.Time{})
} else {
directory.AddHTMLEntry(node.Path(), node.IsDir(), node.Size(), node.ModTime().UTC())

View File

@ -17,6 +17,7 @@ import (
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -48,7 +49,7 @@ func Run(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
f = cmd.NewFsSrc(args)
cmd.Run(false, true, command, func() error {
s, err := NewServer(context.Background(), vfs.New(f, &vfsflags.Opt), &opt)
s, err := NewServer(context.Background(), vfs.New(f, &vfscommon.Opt), &opt)
if err != nil {
return err
}

View File

@ -19,7 +19,7 @@ import (
"github.com/rclone/rclone/fs/config/obscure"
libcache "github.com/rclone/rclone/lib/cache"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/rclone/rclone/vfs/vfscommon"
)
// Help contains text describing how to use the proxy
@ -242,7 +242,7 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value interface{}, er
// need to in memory. An attacker would find it easier to go
// after the unencrypted password in memory most likely.
entry := cacheEntry{
vfs: vfs.New(f, &vfsflags.Opt),
vfs: vfs.New(f, &vfscommon.Opt),
pwHash: sha256.Sum256([]byte(auth)),
}
return entry, true, nil

View File

@ -13,7 +13,7 @@ import (
"github.com/rclone/rclone/fs/hash"
httplib "github.com/rclone/rclone/lib/http"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/rclone/rclone/vfs/vfscommon"
)
// Options contains options for the http Server
@ -42,7 +42,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *Server, err error
w := &Server{
f: f,
ctx: ctx,
vfs: vfs.New(f, &vfsflags.Opt),
vfs: vfs.New(f, &vfscommon.Opt),
}
if len(opt.authPair) == 0 {

View File

@ -17,7 +17,7 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/terminal"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/rclone/rclone/vfs/vfscommon"
"golang.org/x/crypto/ssh"
)
@ -307,7 +307,7 @@ func serveStdio(f fs.Fs) error {
stdin: os.Stdin,
stdout: os.Stdout,
}
handlers := newVFSHandler(vfs.New(f, &vfsflags.Opt))
handlers := newVFSHandler(vfs.New(f, &vfscommon.Opt))
return serveChannel(sshChannel, handlers, "stdio")
}

View File

@ -28,7 +28,7 @@ import (
"github.com/rclone/rclone/lib/env"
"github.com/rclone/rclone/lib/file"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/rclone/rclone/vfs/vfscommon"
"golang.org/x/crypto/ssh"
)
@ -54,7 +54,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) *server {
if proxyflags.Opt.AuthProxy != "" {
s.proxy = proxy.New(ctx, &proxyflags.Opt)
} else {
s.vfs = vfs.New(f, &vfsflags.Opt)
s.vfs = vfs.New(f, &vfscommon.Opt)
}
return s
}

View File

@ -26,6 +26,7 @@ import (
"github.com/rclone/rclone/lib/http/serve"
"github.com/rclone/rclone/lib/systemd"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
"golang.org/x/net/webdav"
@ -193,7 +194,7 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options) (w *WebDAV, err error
// override auth
w.opt.Auth.CustomAuthFn = w.auth
} else {
w._vfs = vfs.New(f, &vfsflags.Opt)
w._vfs = vfs.New(f, &vfscommon.Opt)
}
w.Server, err = libhttp.NewServer(ctx,
@ -365,7 +366,7 @@ func (w *WebDAV) serveDir(rw http.ResponseWriter, r *http.Request, dirRemote str
// Make the entries for display
directory := serve.NewDirectory(dirRemote, w.Server.HTMLTemplate())
for _, node := range dirEntries {
if vfsflags.Opt.NoModTime {
if vfscommon.Opt.NoModTime {
directory.AddHTMLEntry(node.Path(), node.IsDir(), node.Size(), time.Time{})
} else {
directory.AddHTMLEntry(node.Path(), node.IsDir(), node.Size(), node.ModTime().UTC())

View File

@ -157,7 +157,7 @@ func (d *Dir) IsDir() bool {
// Mode bits of the directory - satisfies Node interface
func (d *Dir) Mode() (mode os.FileMode) {
return d.vfs.Opt.DirPerms
return os.FileMode(d.vfs.Opt.DirPerms)
}
// Name (base) of the directory - satisfies Node interface

View File

@ -43,7 +43,7 @@ func TestDirMethods(t *testing.T) {
assert.Equal(t, false, dir.IsFile())
// Mode
assert.Equal(t, vfs.Opt.DirPerms, dir.Mode())
assert.Equal(t, os.FileMode(vfs.Opt.DirPerms), dir.Mode())
// Name
assert.Equal(t, "dir", dir.Name())

View File

@ -94,7 +94,7 @@ func (f *File) IsDir() bool {
func (f *File) Mode() (mode os.FileMode) {
f.mu.RLock()
defer f.mu.RUnlock()
mode = f.d.vfs.Opt.FilePerms
mode = os.FileMode(f.d.vfs.Opt.FilePerms)
if f.appendMode {
mode |= os.ModeAppend
}

View File

@ -19,7 +19,7 @@ import (
)
func fileCreate(t *testing.T, mode vfscommon.CacheMode) (r *fstest.Run, vfs *VFS, fh *File, item fstest.Item) {
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
opt.CacheMode = mode
opt.WriteBack = writeBackDelay
r, vfs = newTestVFSOpt(t, &opt)
@ -48,7 +48,7 @@ func TestFileMethods(t *testing.T) {
assert.Equal(t, true, file.IsFile())
// Mode
assert.Equal(t, vfs.Opt.FilePerms, file.Mode())
assert.Equal(t, os.FileMode(vfs.Opt.FilePerms), file.Mode())
// Name
assert.Equal(t, "file1", file.Name())

View File

@ -46,7 +46,7 @@ func TestRcGetVFS(t *testing.T) {
assert.Contains(t, err.Error(), "no VFS found with name")
assert.Nil(t, vfs)
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
opt.NoModTime = true
vfs3 := New(r.Fremote, &opt)
defer vfs3.Shutdown()

View File

@ -30,7 +30,7 @@ var (
// Create a file and open it with the flags passed in
func rwHandleCreateFlags(t *testing.T, create bool, filename string, flags int) (r *fstest.Run, vfs *VFS, fh *RWFileHandle) {
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
opt.CacheMode = vfscommon.CacheModeFull
opt.WriteBack = writeBackDelay
r, vfs = newTestVFSOpt(t, &opt)
@ -634,7 +634,7 @@ func testRWFileHandleOpenTest(t *testing.T, vfs *VFS, test *openTest) {
func TestRWFileHandleOpenTests(t *testing.T) {
for _, cacheMode := range []vfscommon.CacheMode{vfscommon.CacheModeWrites, vfscommon.CacheModeFull} {
t.Run(cacheMode.String(), func(t *testing.T) {
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
opt.CacheMode = cacheMode
opt.WriteBack = writeBackDelay
_, vfs := newTestVFSOpt(t, &opt)
@ -685,7 +685,7 @@ func TestRWFileModTimeWithOpenWriters(t *testing.T) {
}
func TestRWCacheRename(t *testing.T) {
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
opt.CacheMode = vfscommon.CacheModeFull
opt.WriteBack = writeBackDelay
r, vfs := newTestVFSOpt(t, &opt)
@ -719,7 +719,7 @@ func TestRWCacheRename(t *testing.T) {
//
// See: https://github.com/rclone/rclone/issues/6053
func TestRWCacheUpdate(t *testing.T) {
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
opt.CacheMode = vfscommon.CacheModeFull
opt.WriteBack = writeBackDelay
opt.DirCacheTime = fs.Duration(100 * time.Millisecond)

View File

@ -204,7 +204,7 @@ func New(f fs.Fs, opt *vfscommon.Options) *VFS {
if opt != nil {
vfs.Opt = *opt
} else {
vfs.Opt = vfscommon.DefaultOpt
vfs.Opt = vfscommon.Opt
}
// Fill out anything else
@ -776,7 +776,7 @@ func (vfs *VFS) AddVirtual(remote string, size int64, isDir bool) (err error) {
} else {
// Create parent of virtual directory since backend can't have empty directories
parent, leaf = path.Split(remote)
dir, err = vfs.mkdirAll(parent, vfs.Opt.DirPerms)
dir, err = vfs.mkdirAll(parent, os.FileMode(vfs.Opt.DirPerms))
}
if err != nil {
return err

View File

@ -34,12 +34,12 @@ func TestCaseSensitivity(t *testing.T) {
file3 := r.WriteObject(ctx, "FilEb", "data3", t3)
// Create a case-Sensitive and case-INsensitive VFS
optCS := vfscommon.DefaultOpt
optCS := vfscommon.Opt
optCS.CaseInsensitive = false
vfsCS := New(r.Fremote, &optCS)
defer cleanupVFS(t, vfsCS)
optCI := vfscommon.DefaultOpt
optCI := vfscommon.Opt
optCI.CaseInsensitive = true
vfsCI := New(r.Fremote, &optCI)
defer cleanupVFS(t, vfsCI)
@ -179,7 +179,7 @@ func TestUnicodeNormalization(t *testing.T) {
r.CheckRemoteItems(t, file1, file2)
// Create VFS
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
vfs := New(r.Fremote, &opt)
defer cleanupVFS(t, vfs)

View File

@ -138,10 +138,8 @@ func TestVFSNew(t *testing.T) {
r, vfs := newTestVFS(t)
// Check making a VFS with nil options
var defaultOpt = vfscommon.DefaultOpt
defaultOpt.DirPerms |= os.ModeDir
assert.Equal(t, vfs.Opt, defaultOpt)
assert.Equal(t, vfs.f, r.Fremote)
var defaultOpt = vfscommon.Opt
defaultOpt.Init()
checkActiveCacheEntries(1)
@ -164,14 +162,14 @@ func TestVFSNew(t *testing.T) {
// TestVFSNewWithOpts sees if the New command works properly
func TestVFSNewWithOpts(t *testing.T) {
var opt = vfscommon.DefaultOpt
var opt = vfscommon.Opt
opt.DirPerms = 0777
opt.FilePerms = 0666
opt.Umask = 0002
_, vfs := newTestVFSOpt(t, &opt)
assert.Equal(t, os.FileMode(0775)|os.ModeDir, vfs.Opt.DirPerms)
assert.Equal(t, os.FileMode(0664), vfs.Opt.FilePerms)
assert.Equal(t, vfscommon.FileMode(0775)|vfscommon.FileMode(os.ModeDir), vfs.Opt.DirPerms)
assert.Equal(t, vfscommon.FileMode(0664), vfs.Opt.FilePerms)
}
// TestVFSRoot checks root directory is present and correct
@ -182,7 +180,7 @@ func TestVFSRoot(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, vfs.root, root)
assert.True(t, root.IsDir())
assert.Equal(t, vfs.Opt.DirPerms.Perm(), root.Mode().Perm())
assert.Equal(t, os.FileMode(vfs.Opt.DirPerms).Perm(), root.Mode().Perm())
}
func TestVFSStat(t *testing.T) {

View File

@ -104,7 +104,7 @@ func newTestCacheOpt(t *testing.T, opt vfscommon.Options) (r *fstest.Run, c *Cac
}
func newTestCache(t *testing.T) (r *fstest.Run, c *Cache) {
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
// Disable the cache cleaner as it interferes with these tests
opt.CachePollInterval = 0
@ -627,7 +627,7 @@ func TestCacheRename(t *testing.T) {
}
func TestCacheCleaner(t *testing.T) {
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
opt.CachePollInterval = fs.Duration(10 * time.Millisecond)
opt.CacheMaxAge = fs.Duration(20 * time.Millisecond)
_, c := newTestCacheOpt(t, opt)

View File

@ -93,7 +93,7 @@ func TestDownloaders(t *testing.T) {
t: t,
size: size,
}
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
dls := New(item, &opt, remote, src)
return item, dls
}

View File

@ -24,7 +24,7 @@ import (
var zeroes = string(make([]byte, 100))
func newItemTestCache(t *testing.T) (r *fstest.Run, c *Cache) {
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
// Disable the cache cleaner as it interferes with these tests
opt.CachePollInterval = 0

View File

@ -17,7 +17,7 @@ import (
func newTestWriteBack(t *testing.T) (wb *WriteBack, cancel func()) {
ctx, cancel := context.WithCancel(context.Background())
opt := vfscommon.DefaultOpt
opt := vfscommon.Opt
opt.WriteBack = fs.Duration(100 * time.Millisecond)
wb = New(ctx, &opt)
return wb, cancel

40
vfs/vfscommon/filemode.go Normal file
View File

@ -0,0 +1,40 @@
package vfscommon
import (
"fmt"
"os"
"strconv"
"github.com/rclone/rclone/fs"
)
// FileMode is a command line friendly os.FileMode
type FileMode os.FileMode
// String turns FileMode into a string
func (x FileMode) String() string {
return fmt.Sprintf("%03o", x)
}
// Set a FileMode
func (x *FileMode) Set(s string) error {
i, err := strconv.ParseInt(s, 8, 32)
if err != nil {
return fmt.Errorf("bad FileMode - must be octal digits: %w", err)
}
*x = (FileMode)(i)
return nil
}
// Type of the value
func (x FileMode) Type() string {
return "FileMode"
}
// UnmarshalJSON makes sure the value can be parsed as a string or integer in JSON
func (x *FileMode) UnmarshalJSON(in []byte) error {
return fs.UnmarshalJSONFlag(in, x, func(i int64) error {
*x = FileMode(i)
return nil
})
}

View File

@ -0,0 +1,77 @@
package vfscommon
import (
"encoding/json"
"testing"
"github.com/rclone/rclone/fs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Check it satisfies the interfaces
var (
_ fs.Flagger = (*FileMode)(nil)
_ fs.FlaggerNP = FileMode(0)
)
func TestFileModeString(t *testing.T) {
for _, test := range []struct {
in FileMode
want string
}{
{0, "000"},
{0666, "666"},
{02666, "2666"},
} {
got := test.in.String()
assert.Equal(t, test.want, got)
}
}
func TestFileModeSet(t *testing.T) {
for _, test := range []struct {
in string
want FileMode
err bool
}{
{"0", 0, false},
{"0666", 0666, false},
{"666", 0666, false},
{"2666", 02666, false},
{"999", 0, true},
} {
got := FileMode(0)
err := got.Set(test.in)
if test.err {
require.Error(t, err, test.in)
} else {
require.NoError(t, err, test.in)
}
assert.Equal(t, test.want, got)
}
}
func TestFileModeUnmarshalJSON(t *testing.T) {
for _, test := range []struct {
in string
want FileMode
err bool
}{
{`"0"`, 0, false},
{`"666"`, 0666, false},
{`"02666"`, 02666, false},
{`"999"`, 0, true},
{`438`, 0666, false},
{`"999"`, 0, true},
} {
var ss FileMode
err := json.Unmarshal([]byte(test.in), &ss)
if test.err {
require.Error(t, err, test.in)
} else {
require.NoError(t, err, test.in)
}
assert.Equal(t, test.want, ss, test.in)
}
}

View File

@ -8,75 +8,194 @@ import (
"github.com/rclone/rclone/fs"
)
// Options is options for creating the vfs
type Options struct {
NoSeek bool // don't allow seeking if set
NoChecksum bool // don't check checksums if set
ReadOnly bool // if set VFS is read only
NoModTime bool // don't read mod times for files
DirCacheTime fs.Duration // how long to consider directory listing cache valid
Refresh bool // refreshes the directory listing recursively on start
PollInterval fs.Duration
Umask int
UID uint32
GID uint32
DirPerms os.FileMode
FilePerms os.FileMode
ChunkSize fs.SizeSuffix // if > 0 read files in chunks
ChunkSizeLimit fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached
CacheMode CacheMode
CacheMaxAge fs.Duration
CacheMaxSize fs.SizeSuffix
CacheMinFreeSpace fs.SizeSuffix
CachePollInterval fs.Duration
CaseInsensitive bool
BlockNormDupes bool
WriteWait fs.Duration // time to wait for in-sequence write
ReadWait fs.Duration // time to wait for in-sequence read
WriteBack fs.Duration // time to wait before writing back dirty files
ReadAhead fs.SizeSuffix // bytes to read ahead in cache mode "full"
UsedIsSize bool // if true, use the `rclone size` algorithm for Used size
FastFingerprint bool // if set use fast fingerprints
DiskSpaceTotalSize fs.SizeSuffix
// OptionsInfo describes the Options in use
var OptionsInfo = fs.Options{{
Name: "no_modtime",
Default: false,
Help: "Don't read/write the modification time (can speed things up)",
Groups: "VFS",
}, {
Name: "no_checksum",
Default: false,
Help: "Don't compare checksums on up/download",
Groups: "VFS",
}, {
Name: "no_seek",
Default: false,
Help: "Don't allow seeking in files",
Groups: "VFS",
}, {
Name: "dir_cache_time",
Default: fs.Duration(5 * 60 * time.Second),
Help: "Time to cache directory entries for",
Groups: "VFS",
}, {
Name: "vfs_refresh",
Default: false,
Help: "Refreshes the directory cache recursively in the background on start",
Groups: "VFS",
}, {
Name: "poll_interval",
Default: fs.Duration(time.Minute),
Help: "Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable)",
Groups: "VFS",
}, {
Name: "read_only",
Default: false,
Help: "Only allow read-only access",
Groups: "VFS",
}, {
Name: "vfs_cache_mode",
Default: CacheModeOff,
Help: "Cache mode off|minimal|writes|full",
Groups: "VFS",
}, {
Name: "vfs_cache_poll_interval",
Default: fs.Duration(60 * time.Second),
Help: "Interval to poll the cache for stale objects",
Groups: "VFS",
}, {
Name: "vfs_cache_max_age",
Default: fs.Duration(3600 * time.Second),
Help: "Max time since last access of objects in the cache",
Groups: "VFS",
}, {
Name: "vfs_cache_max_size",
Default: fs.SizeSuffix(-1),
Help: "Max total size of objects in the cache",
Groups: "VFS",
}, {
Name: "vfs_cache_min_free_space",
Default: fs.SizeSuffix(-1),
Help: "Target minimum free space on the disk containing the cache",
Groups: "VFS",
}, {
Name: "vfs_read_chunk_size",
Default: 128 * fs.Mebi,
Help: "Read the source objects in chunks",
Groups: "VFS",
}, {
Name: "vfs_read_chunk_size_limit",
Default: fs.SizeSuffix(-1),
Help: "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited)",
Groups: "VFS",
}, {
Name: "dir_perms",
Default: FileMode(0777),
Help: "Directory permissions",
Groups: "VFS",
}, {
Name: "file_perms",
Default: FileMode(0666),
Help: "File permissions",
Groups: "VFS",
}, {
Name: "vfs_case_insensitive",
Default: runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise,
Help: "If a file name not found, find a case insensitive match",
Groups: "VFS",
}, {
Name: "vfs_block_norm_dupes",
Default: false,
Help: "If duplicate filenames exist in the same directory (after normalization), log an error and hide the duplicates (may have a performance cost)",
Groups: "VFS",
}, {
Name: "vfs_write_wait",
Default: fs.Duration(1000 * time.Millisecond),
Help: "Time to wait for in-sequence write before giving error",
Groups: "VFS",
}, {
Name: "vfs_read_wait",
Default: fs.Duration(20 * time.Millisecond),
Help: "Time to wait for in-sequence read before seeking",
Groups: "VFS",
}, {
Name: "vfs_write_back",
Default: fs.Duration(5 * time.Second),
Help: "Time to writeback files after last use when using cache",
Groups: "VFS",
}, {
Name: "vfs_read_ahead",
Default: 0 * fs.Mebi,
Help: "Extra read ahead over --buffer-size when using cache-mode full",
Groups: "VFS",
}, {
Name: "vfs_used_is_size",
Default: false,
Help: "Use the `rclone size` algorithm for Used size",
Groups: "VFS",
}, {
Name: "vfs_fast_fingerprint",
Default: false,
Help: "Use fast (less accurate) fingerprints for change detection",
Groups: "VFS",
}, {
Name: "vfs_disk_space_total_size",
Default: fs.SizeSuffix(-1),
Help: "Specify the total space of disk",
Groups: "VFS",
}, {
Name: "umask",
Default: FileMode(getUmask()),
Help: "Override the permission bits set by the filesystem (not supported on Windows)",
Groups: "VFS",
}, {
Name: "uid",
Default: getUID(),
Help: "Override the uid field set by the filesystem (not supported on Windows)",
Groups: "VFS",
}, {
Name: "gid",
Default: getGID(),
Help: "Override the gid field set by the filesystem (not supported on Windows)",
Groups: "VFS",
}}
func init() {
fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "vfs", Opt: &Opt, Options: OptionsInfo})
}
// DefaultOpt is the default values uses for Opt
var DefaultOpt = Options{
NoModTime: false,
NoChecksum: false,
NoSeek: false,
DirCacheTime: fs.Duration(5 * 60 * time.Second),
Refresh: false,
PollInterval: fs.Duration(time.Minute),
ReadOnly: false,
Umask: 0,
UID: ^uint32(0), // these values instruct WinFSP-FUSE to use the current user
GID: ^uint32(0), // overridden for non windows in mount_unix.go
DirPerms: os.FileMode(0777),
FilePerms: os.FileMode(0666),
CacheMode: CacheModeOff,
CacheMaxAge: fs.Duration(3600 * time.Second),
CachePollInterval: fs.Duration(60 * time.Second),
ChunkSize: 128 * fs.Mebi,
ChunkSizeLimit: -1,
CacheMaxSize: -1,
CacheMinFreeSpace: -1,
CaseInsensitive: runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
WriteWait: fs.Duration(1000 * time.Millisecond),
ReadWait: fs.Duration(20 * time.Millisecond),
WriteBack: fs.Duration(5 * time.Second),
ReadAhead: 0 * fs.Mebi,
UsedIsSize: false,
DiskSpaceTotalSize: -1,
// Options is options for creating the vfs
type Options struct {
NoSeek bool `config:"no_seek"` // don't allow seeking if set
NoChecksum bool `config:"no_checksum"` // don't check checksums if set
ReadOnly bool `config:"read_only"` // if set VFS is read only
NoModTime bool `config:"no_modtime"` // don't read mod times for files
DirCacheTime fs.Duration `config:"dir_cache_time"` // how long to consider directory listing cache valid
Refresh bool `config:"vfs_refresh"` // refreshes the directory listing recursively on start
PollInterval fs.Duration `config:"poll_interval"`
Umask FileMode `config:"umask"`
UID uint32 `config:"uid"`
GID uint32 `config:"gid"`
DirPerms FileMode `config:"dir_perms"`
FilePerms FileMode `config:"file_perms"`
ChunkSize fs.SizeSuffix `config:"vfs_read_chunk_size"` // if > 0 read files in chunks
ChunkSizeLimit fs.SizeSuffix `config:"vfs_read_chunk_size_limit"` // if > ChunkSize double the chunk size after each chunk until reached
CacheMode CacheMode `config:"vfs_cache_mode"`
CacheMaxAge fs.Duration `config:"vfs_cache_max_age"`
CacheMaxSize fs.SizeSuffix `config:"vfs_cache_max_size"`
CacheMinFreeSpace fs.SizeSuffix `config:"vfs_cache_min_free_space"`
CachePollInterval fs.Duration `config:"vfs_cache_poll_interval"`
CaseInsensitive bool `config:"vfs_case_insensitive"`
BlockNormDupes bool `config:"vfs_block_norm_dupes"`
WriteWait fs.Duration `config:"vfs_write_wait"` // time to wait for in-sequence write
ReadWait fs.Duration `config:"vfs_read_wait"` // time to wait for in-sequence read
WriteBack fs.Duration `config:"vfs_write_back"` // time to wait before writing back dirty files
ReadAhead fs.SizeSuffix `config:"vfs_read_ahead"` // bytes to read ahead in cache mode "full"
UsedIsSize bool `config:"vfs_used_is_size"` // if true, use the `rclone size` algorithm for Used size
FastFingerprint bool `config:"vfs_fast_fingerprint"` // if set use fast fingerprints
DiskSpaceTotalSize fs.SizeSuffix `config:"vfs_disk_space_total_size"`
}
// Opt is the default options modified by the environment variables and command line flags
var Opt Options
// Init the options, making sure everything is within range
func (opt *Options) Init() {
// Mask the permissions with the umask
opt.DirPerms &= ^os.FileMode(opt.Umask)
opt.FilePerms &= ^os.FileMode(opt.Umask)
opt.DirPerms &= ^opt.Umask
opt.FilePerms &= ^opt.Umask
// Make sure directories are returned as directories
opt.DirPerms |= os.ModeDir
opt.DirPerms |= FileMode(os.ModeDir)
}

View File

@ -0,0 +1,18 @@
//go:build !linux && !darwin && !freebsd
package vfscommon
// get the current umask
func getUmask() int {
return 0000
}
// get the current uid
func getUID() uint32 {
return ^uint32(0) // these values instruct WinFSP-FUSE to use the current user
}
// get the current gid
func getGID() uint32 {
return ^uint32(0) // these values instruct WinFSP-FUSE to use the current user
}

View File

@ -0,0 +1,24 @@
//go:build linux || darwin || freebsd
package vfscommon
import (
"golang.org/x/sys/unix"
)
// get the current umask
func getUmask() int {
umask := unix.Umask(0) // read the umask
unix.Umask(umask) // set it back to what it was
return umask
}
// get the current uid
func getUID() uint32 {
return uint32(unix.Geteuid())
}
// get the current gid
func getGID() uint32 {
return uint32(unix.Getegid())
}

View File

@ -1,32 +0,0 @@
package vfsflags
import (
"fmt"
"os"
"strconv"
)
// FileMode is a command line friendly os.FileMode
type FileMode struct {
Mode *os.FileMode
}
// String turns FileMode into a string
func (x *FileMode) String() string {
return fmt.Sprintf("0%3o", *x.Mode)
}
// Set a FileMode
func (x *FileMode) Set(s string) error {
i, err := strconv.ParseInt(s, 8, 32)
if err != nil {
return fmt.Errorf("bad FileMode - must be octal digits: %w", err)
}
*x.Mode = (os.FileMode)(i)
return nil
}
// Type of the value
func (x *FileMode) Type() string {
return "FileMode"
}

View File

@ -3,45 +3,11 @@ package vfsflags
import (
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/spf13/pflag"
)
// Options set by command line flags
var (
Opt = vfscommon.DefaultOpt
DirPerms = &FileMode{Mode: &Opt.DirPerms}
FilePerms = &FileMode{Mode: &Opt.FilePerms}
)
// AddFlags adds the non filing system specific flags to the command
func AddFlags(flagSet *pflag.FlagSet) {
rc.AddOption("vfs", &Opt)
flags.BoolVarP(flagSet, &Opt.NoModTime, "no-modtime", "", Opt.NoModTime, "Don't read/write the modification time (can speed things up)", "VFS")
flags.BoolVarP(flagSet, &Opt.NoChecksum, "no-checksum", "", Opt.NoChecksum, "Don't compare checksums on up/download", "VFS")
flags.BoolVarP(flagSet, &Opt.NoSeek, "no-seek", "", Opt.NoSeek, "Don't allow seeking in files", "VFS")
flags.FVarP(flagSet, &Opt.DirCacheTime, "dir-cache-time", "", "Time to cache directory entries for", "VFS")
flags.BoolVarP(flagSet, &Opt.Refresh, "vfs-refresh", "", Opt.Refresh, "Refreshes the directory cache recursively in the background on start", "VFS")
flags.FVarP(flagSet, &Opt.PollInterval, "poll-interval", "", "Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable)", "VFS")
flags.BoolVarP(flagSet, &Opt.ReadOnly, "read-only", "", Opt.ReadOnly, "Only allow read-only access", "VFS")
flags.FVarP(flagSet, &Opt.CacheMode, "vfs-cache-mode", "", "Cache mode off|minimal|writes|full", "VFS")
flags.FVarP(flagSet, &Opt.CachePollInterval, "vfs-cache-poll-interval", "", "Interval to poll the cache for stale objects", "VFS")
flags.FVarP(flagSet, &Opt.CacheMaxAge, "vfs-cache-max-age", "", "Max time since last access of objects in the cache", "VFS")
flags.FVarP(flagSet, &Opt.CacheMaxSize, "vfs-cache-max-size", "", "Max total size of objects in the cache", "VFS")
flags.FVarP(flagSet, &Opt.CacheMinFreeSpace, "vfs-cache-min-free-space", "", "Target minimum free space on the disk containing the cache", "VFS")
flags.FVarP(flagSet, &Opt.ChunkSize, "vfs-read-chunk-size", "", "Read the source objects in chunks", "VFS")
flags.FVarP(flagSet, &Opt.ChunkSizeLimit, "vfs-read-chunk-size-limit", "", "If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited)", "VFS")
flags.FVarP(flagSet, DirPerms, "dir-perms", "", "Directory permissions", "VFS")
flags.FVarP(flagSet, FilePerms, "file-perms", "", "File permissions", "VFS")
flags.BoolVarP(flagSet, &Opt.CaseInsensitive, "vfs-case-insensitive", "", Opt.CaseInsensitive, "If a file name not found, find a case insensitive match", "VFS")
flags.BoolVarP(flagSet, &Opt.BlockNormDupes, "vfs-block-norm-dupes", "", Opt.BlockNormDupes, "If duplicate filenames exist in the same directory (after normalization), log an error and hide the duplicates (may have a performance cost)", "VFS")
flags.FVarP(flagSet, &Opt.WriteWait, "vfs-write-wait", "", "Time to wait for in-sequence write before giving error", "VFS")
flags.FVarP(flagSet, &Opt.ReadWait, "vfs-read-wait", "", "Time to wait for in-sequence read before seeking", "VFS")
flags.FVarP(flagSet, &Opt.WriteBack, "vfs-write-back", "", "Time to writeback files after last use when using cache", "VFS")
flags.FVarP(flagSet, &Opt.ReadAhead, "vfs-read-ahead", "", "Extra read ahead over --buffer-size when using cache-mode full", "VFS")
flags.BoolVarP(flagSet, &Opt.UsedIsSize, "vfs-used-is-size", "", Opt.UsedIsSize, "Use the `rclone size` algorithm for Used size", "VFS")
flags.BoolVarP(flagSet, &Opt.FastFingerprint, "vfs-fast-fingerprint", "", Opt.FastFingerprint, "Use fast (less accurate) fingerprints for change detection", "VFS")
flags.FVarP(flagSet, &Opt.DiskSpaceTotalSize, "vfs-disk-space-total-size", "", "Specify the total space of disk", "VFS")
platformFlags(flagSet)
flags.AddFlagsFromOptions(flagSet, "", vfscommon.OptionsInfo)
}

View File

@ -1,11 +0,0 @@
//go:build !linux && !darwin && !freebsd
package vfsflags
import (
"github.com/spf13/pflag"
)
// add any extra platform specific flags
func platformFlags(flags *pflag.FlagSet) {
}

View File

@ -1,20 +0,0 @@
//go:build linux || darwin || freebsd
package vfsflags
import (
"github.com/rclone/rclone/fs/config/flags"
"github.com/spf13/pflag"
"golang.org/x/sys/unix"
)
// add any extra platform specific flags
func platformFlags(flagSet *pflag.FlagSet) {
Opt.Umask = unix.Umask(0) // read the umask
unix.Umask(Opt.Umask) // set it back to what it was
flags.IntVarP(flagSet, &Opt.Umask, "umask", "", Opt.Umask, "Override the permission bits set by the filesystem (not supported on Windows)", "VFS")
Opt.UID = uint32(unix.Geteuid())
Opt.GID = uint32(unix.Getegid())
flags.Uint32VarP(flagSet, &Opt.UID, "uid", "", Opt.UID, "Override the uid field set by the filesystem (not supported on Windows)", "VFS")
flags.Uint32VarP(flagSet, &Opt.GID, "gid", "", Opt.GID, "Override the gid field set by the filesystem (not supported on Windows)", "VFS")
}

View File

@ -26,7 +26,6 @@ import (
"github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/rclone/rclone/vfs/vfsflags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -62,7 +61,7 @@ func RunTests(t *testing.T, useVFS bool, minimumRequiredCacheMode vfscommon.Cach
if test.cacheMode < minimumRequiredCacheMode {
continue
}
vfsOpt := vfsflags.Opt
vfsOpt := vfscommon.Opt
vfsOpt.CacheMode = test.cacheMode
vfsOpt.WriteBack = test.writeBack
run = newRun(useVFS, &vfsOpt, mountFn)
@ -235,10 +234,10 @@ func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {
if fi.IsDir() {
dir[name+"/"] = struct{}{}
r.readLocal(t, dir, name)
assert.Equal(t, r.vfsOpt.DirPerms&os.ModePerm, fi.Mode().Perm())
assert.Equal(t, os.FileMode(r.vfsOpt.DirPerms)&os.ModePerm, fi.Mode().Perm())
} else {
dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{}
assert.Equal(t, r.vfsOpt.FilePerms&os.ModePerm, fi.Mode().Perm())
assert.Equal(t, os.FileMode(r.vfsOpt.FilePerms)&os.ModePerm, fi.Mode().Perm())
}
}
}
@ -377,5 +376,5 @@ func TestRoot(t *testing.T) {
fi, err := os.Lstat(run.mountPath)
require.NoError(t, err)
assert.True(t, fi.IsDir())
assert.Equal(t, run.vfsOpt.DirPerms&os.ModePerm, fi.Mode().Perm())
assert.Equal(t, os.FileMode(run.vfsOpt.DirPerms)&os.ModePerm, fi.Mode().Perm())
}