vfs: add an optional fs parameter to vfs rc methods

Before this change we initialized the rc for a single VFS. However
rclone can have multiple VFSes in use now so this is no longer
adequate.

This change adds an optional fs parameter to all the VFS methods to
disambiguate VFSes when there is more than one in use.

It also adds a method vfs/list to show all the active VFSes.

This adds outline tests for the rc commands which didn't have tests
before.
This commit is contained in:
Nick Craig-Wood 2020-06-15 14:02:41 +01:00
parent ce065614e2
commit 7781ea8d59
3 changed files with 443 additions and 235 deletions

557
vfs/rc.go
View File

@ -2,155 +2,57 @@ package vfs
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/rc"
)
// Add remote control for the VFS
func (vfs *VFS) addRC() {
const getVFSHelp = `
This command takes an "fs" parameter. If this parameter is not
supplied and if there is only one VFS in use then that VFS will be
used. If there is more than one VFS in use then the "fs" parameter
must be supplied.`
// GetVFS gets a VFS with config name "fs" from the cache or returns an error.
//
// If "fs" is not set and there is one and only one VFS in the active
// cache then it returns it. This is for backwards compatibility.
func getVFS(in rc.Params) (vfs *VFS, err error) {
fsString, err := in.GetString("fs")
if rc.IsErrParamNotFound(err) {
var count int
vfs, count = activeCacheEntries()
if count == 1 {
return vfs, nil
} else if count == 0 {
return nil, errors.New(`no VFS active and "fs" parameter not supplied`)
}
return nil, errors.New(`more than one VFS active - need "fs" parameter`)
} else if err != nil {
return nil, err
}
activeMu.Lock()
defer activeMu.Unlock()
fsString = cache.Canonicalize(fsString)
activeVFS := active[fsString]
if len(activeVFS) == 0 {
return nil, errors.Errorf("no VFS found with name %q", fsString)
} else if len(activeVFS) > 1 {
return nil, errors.Errorf("more than one VFS active with name %q", fsString)
}
return activeVFS[0], nil
}
func init() {
rc.Add(rc.Call{
Path: "vfs/forget",
Fn: func(ctx context.Context, in rc.Params) (out rc.Params, err error) {
root, err := vfs.Root()
if err != nil {
return nil, err
}
forgotten := []string{}
if len(in) == 0 {
root.ForgetAll()
} else {
for k, v := range in {
path, ok := v.(string)
if !ok {
return out, errors.Errorf("value must be string %q=%v", k, v)
}
path = strings.Trim(path, "/")
if strings.HasPrefix(k, "file") {
root.ForgetPath(path, fs.EntryObject)
} else if strings.HasPrefix(k, "dir") {
root.ForgetPath(path, fs.EntryDirectory)
} else {
return out, errors.Errorf("unknown key %q", k)
}
forgotten = append(forgotten, path)
}
}
out = rc.Params{
"forgotten": forgotten,
}
return out, nil
},
Title: "Forget files or directories in the directory cache.",
Help: `
This forgets the paths in the directory cache causing them to be
re-read from the remote when needed.
If no paths are passed in then it will forget all the paths in the
directory cache.
rclone rc vfs/forget
Otherwise pass files or dirs in as file=path or dir=path. Any
parameter key starting with file will forget that file and any
starting with dir will forget that dir, eg
rclone rc vfs/forget file=hello file2=goodbye dir=home/junk
`,
})
rc.Add(rc.Call{
Path: "vfs/refresh",
Fn: func(ctx context.Context, in rc.Params) (out rc.Params, err error) {
root, err := vfs.Root()
if err != nil {
return nil, err
}
getDir := func(path string) (*Dir, error) {
path = strings.Trim(path, "/")
segments := strings.Split(path, "/")
var node Node = root
for _, s := range segments {
if dir, ok := node.(*Dir); ok {
node, err = dir.stat(s)
if err != nil {
return nil, err
}
}
}
if dir, ok := node.(*Dir); ok {
return dir, nil
}
return nil, EINVAL
}
recursive := false
{
const k = "recursive"
if v, ok := in[k]; ok {
s, ok := v.(string)
if !ok {
return out, errors.Errorf("value must be string %q=%v", k, v)
}
recursive, err = strconv.ParseBool(s)
if err != nil {
return out, errors.Errorf("invalid value %q=%v", k, v)
}
delete(in, k)
}
}
result := map[string]string{}
if len(in) == 0 {
if recursive {
err = root.readDirTree()
} else {
err = root.readDir()
}
if err != nil {
result[""] = err.Error()
} else {
result[""] = "OK"
}
} else {
for k, v := range in {
path, ok := v.(string)
if !ok {
return out, errors.Errorf("value must be string %q=%v", k, v)
}
if strings.HasPrefix(k, "dir") {
dir, err := getDir(path)
if err != nil {
result[path] = err.Error()
} else {
if recursive {
err = dir.readDirTree()
} else {
err = dir.readDir()
}
if err != nil {
result[path] = err.Error()
} else {
result[path] = "OK"
}
}
} else {
return out, errors.Errorf("unknown key %q", k)
}
}
}
out = rc.Params{
"result": result,
}
return out, nil
},
Path: "vfs/refresh",
Fn: rcRefresh,
Title: "Refresh the directory cache.",
Help: `
This reads the directories for the specified paths and freshens the
@ -167,12 +69,223 @@ starting with dir will refresh that directory, eg
If the parameter recursive=true is given the whole directory tree
will get refreshed. This refresh will use --fast-list if enabled.
`,
` + getVFSHelp,
})
}
func rcRefresh(ctx context.Context, in rc.Params) (out rc.Params, err error) {
vfs, err := getVFS(in)
if err != nil {
return nil, err
}
root, err := vfs.Root()
if err != nil {
return nil, err
}
getDir := func(path string) (*Dir, error) {
path = strings.Trim(path, "/")
segments := strings.Split(path, "/")
var node Node = root
for _, s := range segments {
if dir, ok := node.(*Dir); ok {
node, err = dir.stat(s)
if err != nil {
return nil, err
}
}
}
if dir, ok := node.(*Dir); ok {
return dir, nil
}
return nil, EINVAL
}
recursive := false
{
const k = "recursive"
if v, ok := in[k]; ok {
s, ok := v.(string)
if !ok {
return out, errors.Errorf("value must be string %q=%v", k, v)
}
recursive, err = strconv.ParseBool(s)
if err != nil {
return out, errors.Errorf("invalid value %q=%v", k, v)
}
delete(in, k)
}
}
result := map[string]string{}
if len(in) == 0 {
if recursive {
err = root.readDirTree()
} else {
err = root.readDir()
}
if err != nil {
result[""] = err.Error()
} else {
result[""] = "OK"
}
} else {
for k, v := range in {
path, ok := v.(string)
if !ok {
return out, errors.Errorf("value must be string %q=%v", k, v)
}
if strings.HasPrefix(k, "dir") {
dir, err := getDir(path)
if err != nil {
result[path] = err.Error()
} else {
if recursive {
err = dir.readDirTree()
} else {
err = dir.readDir()
}
if err != nil {
result[path] = err.Error()
} else {
result[path] = "OK"
}
}
} else {
return out, errors.Errorf("unknown key %q", k)
}
}
}
out = rc.Params{
"result": result,
}
return out, nil
}
// Add remote control for the VFS
func init() {
rc.Add(rc.Call{
Path: "vfs/forget",
Fn: rcForget,
Title: "Forget files or directories in the directory cache.",
Help: `
This forgets the paths in the directory cache causing them to be
re-read from the remote when needed.
If no paths are passed in then it will forget all the paths in the
directory cache.
rclone rc vfs/forget
Otherwise pass files or dirs in as file=path or dir=path. Any
parameter key starting with file will forget that file and any
starting with dir will forget that dir, eg
rclone rc vfs/forget file=hello file2=goodbye dir=home/junk
` + getVFSHelp,
})
}
func rcForget(ctx context.Context, in rc.Params) (out rc.Params, err error) {
vfs, err := getVFS(in)
if err != nil {
return nil, err
}
root, err := vfs.Root()
if err != nil {
return nil, err
}
forgotten := []string{}
if len(in) == 0 {
root.ForgetAll()
} else {
for k, v := range in {
path, ok := v.(string)
if !ok {
return out, errors.Errorf("value must be string %q=%v", k, v)
}
path = strings.Trim(path, "/")
if strings.HasPrefix(k, "file") {
root.ForgetPath(path, fs.EntryObject)
} else if strings.HasPrefix(k, "dir") {
root.ForgetPath(path, fs.EntryDirectory)
} else {
return out, errors.Errorf("unknown key %q", k)
}
forgotten = append(forgotten, path)
}
}
out = rc.Params{
"forgotten": forgotten,
}
return out, nil
}
func getDuration(k string, v interface{}) (time.Duration, error) {
s, ok := v.(string)
if !ok {
return 0, errors.Errorf("value must be string %q=%v", k, v)
}
interval, err := fs.ParseDuration(s)
if err != nil {
return 0, errors.Wrap(err, "parse duration")
}
return interval, nil
}
func getInterval(in rc.Params) (time.Duration, bool, error) {
k := "interval"
v, ok := in[k]
if !ok {
return 0, false, nil
}
interval, err := getDuration(k, v)
if err != nil {
return 0, true, err
}
if interval < 0 {
return 0, true, errors.New("interval must be >= 0")
}
delete(in, k)
return interval, true, nil
}
func getTimeout(in rc.Params) (time.Duration, error) {
k := "timeout"
v, ok := in[k]
if !ok {
return 10 * time.Second, nil
}
timeout, err := getDuration(k, v)
if err != nil {
return 0, err
}
delete(in, k)
return timeout, nil
}
func getStatus(vfs *VFS, in rc.Params) (out rc.Params, err error) {
for k, v := range in {
return nil, errors.Errorf("invalid parameter: %s=%s", k, v)
}
return rc.Params{
"enabled": vfs.Opt.PollInterval != 0,
"supported": vfs.pollChan != nil,
"interval": map[string]interface{}{
"raw": vfs.Opt.PollInterval,
"seconds": vfs.Opt.PollInterval / time.Second,
"string": vfs.Opt.PollInterval.String(),
},
}, nil
}
func init() {
rc.Add(rc.Call{
Path: "vfs/poll-interval",
Fn: rcPollFunc(vfs),
Fn: rcPollInterval,
Title: "Get the status or update the value of the poll-interval option.",
Help: `
Without any parameter given this returns the current status of the
@ -194,102 +307,82 @@ not reached.
If poll-interval is updated or disabled temporarily, some changes
might not get picked up by the polling function, depending on the
used remote.
`,
` + getVFSHelp,
})
}
func rcPollFunc(vfs *VFS) (rcPollFunc rc.Func) {
getDuration := func(k string, v interface{}) (time.Duration, error) {
s, ok := v.(string)
if !ok {
return 0, errors.Errorf("value must be string %q=%v", k, v)
}
interval, err := fs.ParseDuration(s)
if err != nil {
return 0, errors.Wrap(err, "parse duration")
}
return interval, nil
}
getInterval := func(in rc.Params) (time.Duration, bool, error) {
k := "interval"
v, ok := in[k]
if !ok {
return 0, false, nil
}
interval, err := getDuration(k, v)
if err != nil {
return 0, true, err
}
if interval < 0 {
return 0, true, errors.New("interval must be >= 0")
}
delete(in, k)
return interval, true, nil
}
getTimeout := func(in rc.Params) (time.Duration, error) {
k := "timeout"
v, ok := in[k]
if !ok {
return 10 * time.Second, nil
}
timeout, err := getDuration(k, v)
if err != nil {
return 0, err
}
delete(in, k)
return timeout, nil
func rcPollInterval(ctx context.Context, in rc.Params) (out rc.Params, err error) {
vfs, err := getVFS(in)
if err != nil {
return nil, err
}
_status := func(in rc.Params) (out rc.Params, err error) {
for k, v := range in {
return nil, errors.Errorf("invalid parameter: %s=%s", k, v)
}
return rc.Params{
"enabled": vfs.Opt.PollInterval != 0,
"supported": vfs.pollChan != nil,
"interval": map[string]interface{}{
"raw": vfs.Opt.PollInterval,
"seconds": vfs.Opt.PollInterval / time.Second,
"string": vfs.Opt.PollInterval.String(),
},
}, nil
interval, intervalPresent, err := getInterval(in)
if err != nil {
return nil, err
}
timeout, err := getTimeout(in)
if err != nil {
return nil, err
}
for k, v := range in {
return nil, errors.Errorf("invalid parameter: %s=%s", k, v)
}
if vfs.pollChan == nil {
return nil, errors.New("poll-interval is not supported by this remote")
}
return func(ctx context.Context, in rc.Params) (out rc.Params, err error) {
interval, intervalPresent, err := getInterval(in)
if err != nil {
return nil, err
}
timeout, err := getTimeout(in)
if err != nil {
return nil, err
}
for k, v := range in {
return nil, errors.Errorf("invalid parameter: %s=%s", k, v)
}
if vfs.pollChan == nil {
return nil, errors.New("poll-interval is not supported by this remote")
}
if !intervalPresent {
return _status(in)
}
var timeoutHit bool
var timeoutChan <-chan time.Time
if timeout > 0 {
timer := time.NewTimer(timeout)
defer timer.Stop()
timeoutChan = timer.C
}
select {
case vfs.pollChan <- interval:
vfs.Opt.PollInterval = interval
case <-timeoutChan:
timeoutHit = true
}
out, err = _status(in)
if out != nil {
out["timeout"] = timeoutHit
}
return
if !intervalPresent {
return getStatus(vfs, in)
}
var timeoutHit bool
var timeoutChan <-chan time.Time
if timeout > 0 {
timer := time.NewTimer(timeout)
defer timer.Stop()
timeoutChan = timer.C
}
select {
case vfs.pollChan <- interval:
vfs.Opt.PollInterval = interval
case <-timeoutChan:
timeoutHit = true
}
out, err = getStatus(vfs, in)
if out != nil {
out["timeout"] = timeoutHit
}
return
}
func init() {
rc.Add(rc.Call{
Path: "vfs/list",
Title: "List active VFSes.",
Help: `
This lists the active VFSes.
It returns a list under the key "vfses" where the values are the VFS
names that could be passed to the other VFS commands in the "fs"
parameter.`,
Fn: rcList,
})
}
func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) {
activeMu.Lock()
defer activeMu.Unlock()
var names = []string{}
for name, vfses := range active {
if len(vfses) == 1 {
names = append(names, name)
} else {
for i := range vfses {
names = append(names, fmt.Sprintf("%s[%d]", name, i))
}
}
}
out = rc.Params{}
out["vfses"] = names
return out, nil
}

118
vfs/rc_test.go Normal file
View File

@ -0,0 +1,118 @@
package vfs
import (
"context"
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func rcNewRun(t *testing.T, method string) (r *fstest.Run, vfs *VFS, cleanup func(), call *rc.Call) {
if *fstest.RemoteName != "" {
t.Skip("Skipping test on non local remote")
}
r, vfs, cleanup = newTestVFS(t)
call = rc.Calls.Get(method)
assert.NotNil(t, call)
return r, vfs, cleanup, call
}
func TestRcGetVFS(t *testing.T) {
in := rc.Params{}
vfs, err := getVFS(in)
require.Error(t, err)
assert.Contains(t, err.Error(), "no VFS active")
assert.Nil(t, vfs)
r, vfs2, cleanup := newTestVFS(t)
defer cleanup()
vfs, err = getVFS(in)
require.NoError(t, err)
assert.True(t, vfs == vfs2)
inPresent := rc.Params{"fs": fs.ConfigString(r.Fremote)}
vfs, err = getVFS(inPresent)
require.NoError(t, err)
assert.True(t, vfs == vfs2)
inWrong := rc.Params{"fs": fs.ConfigString(r.Fremote) + "notfound"}
vfs, err = getVFS(inWrong)
require.Error(t, err)
assert.Contains(t, err.Error(), "no VFS found with name")
assert.Nil(t, vfs)
opt := vfscommon.DefaultOpt
opt.NoModTime = true
vfs3 := New(r.Fremote, &opt)
defer vfs3.Shutdown()
vfs, err = getVFS(in)
require.Error(t, err)
assert.Contains(t, err.Error(), "more than one VFS active - need")
assert.Nil(t, vfs)
vfs, err = getVFS(inPresent)
require.Error(t, err)
assert.Contains(t, err.Error(), "more than one VFS active with name")
assert.Nil(t, vfs)
}
func TestRcForget(t *testing.T) {
r, vfs, cleanup, call := rcNewRun(t, "vfs/forget")
defer cleanup()
_, _ = r, vfs
out, err := call.Fn(context.Background(), nil)
require.NoError(t, err)
assert.Equal(t, rc.Params{
"forgotten": []string{},
}, out)
// FIXME needs more tests
}
func TestRcRefresh(t *testing.T) {
r, vfs, cleanup, call := rcNewRun(t, "vfs/refresh")
defer cleanup()
_, _ = r, vfs
out, err := call.Fn(context.Background(), nil)
require.NoError(t, err)
assert.Equal(t, rc.Params{
"result": map[string]string{
"": "OK",
},
}, out)
// FIXME needs more tests
}
func TestRcPollInterval(t *testing.T) {
r, vfs, cleanup, call := rcNewRun(t, "vfs/poll-interval")
defer cleanup()
_ = vfs
if r.Fremote.Features().ChangeNotify == nil {
t.Skip("ChangeNotify not supported")
}
out, err := call.Fn(context.Background(), nil)
require.NoError(t, err)
assert.Equal(t, rc.Params{}, out)
// FIXME needs more tests
}
func TestRcList(t *testing.T) {
r, vfs, cleanup, call := rcNewRun(t, "vfs/list")
defer cleanup()
_ = vfs
out, err := call.Fn(context.Background(), nil)
require.NoError(t, err)
assert.Equal(t, rc.Params{
"vfses": []string{
fs.ConfigString(r.Fremote),
},
}, out)
}

View File

@ -204,9 +204,6 @@ func New(f fs.Fs, opt *vfscommon.Options) *VFS {
vfs.SetCacheMode(vfs.Opt.CacheMode)
// add the remote control
vfs.addRC()
// Pin the Fs into the cache so that when we use cache.NewFs
// with the same remote string we get this one. The Pin is
// removed by Shutdown