fs/cache: add --fs-cache-expire-duration to control the fs cache

This commit makes the previously statically configured fs cache configurable.

It introduces two parameters `--fs-cache-expire-duration` and
`--fs-cache-expire-interval` to control the caching of the items.

It also adds new interfaces to lib/cache to set these.

docs/content/docs.md

@@ -787,6 +787,27 @@ triggering follow-on actions if data was copied, or skipping if not.
 NB: Enabling this option turns a usually non-fatal error into a potentially
 fatal one - please check and adjust your scripts accordingly!
 
+### --fs-cache-expire-duration=TIME
+
+When using rclone via the API rclone caches created remotes for 5
+minutes by default in the "fs cache". This means that if you do
+repeated actions on the same remote then rclone won't have to build it
+again from scratch, which makes it more efficient.
+
+This flag sets the time that the remotes are cached for. If you set it
+to `0` (or negative) then rclone won't cache the remotes at all.
+
+Note that if you use some flags, eg `--backup-dir` and if this is set
+to `0` rclone may build two remotes (one for the source or destination
+and one for the `--backup-dir` where it may have only built one
+before.
+
+### --fs-cache-expire-interval=TIME
+
+This controls how often rclone checks for cached remotes to expire.
+See the `--fs-cache-expire-duration` documentation above for more
+info. The default is 60s, set to 0 to disable expiry.
+
 ### --header ###
 
 Add an HTTP header for all transactions. The flag can be repeated to

fs/cache/cache.go

@@ -12,14 +12,26 @@ import (
 )
 
 var (
-	c     = cache.New()
+	once  sync.Once // creation
+	c     *cache.Cache
 	mu    sync.Mutex            // mutex to protect remap
 	remap = map[string]string{} // map user supplied names to canonical names
 )
 
+// Create the cache just once
+func createOnFirstUse() {
+	once.Do(func() {
+		ci := fs.GetConfig(context.Background())
+		c = cache.New()
+		c.SetExpireDuration(ci.FsCacheExpireDuration)
+		c.SetExpireInterval(ci.FsCacheExpireInterval)
+	})
+}
+
 // Canonicalize looks up fsString in the mapping from user supplied
 // names to canonical names and return the canonical form
 func Canonicalize(fsString string) string {
+	createOnFirstUse()
 	mu.Lock()
 	canonicalName, ok := remap[fsString]
 	mu.Unlock()
@@ -43,6 +55,7 @@ func addMapping(fsString, canonicalName string) {
 // GetFn gets an fs.Fs named fsString either from the cache or creates
 // it afresh with the create function
 func GetFn(ctx context.Context, fsString string, create func(ctx context.Context, fsString string) (fs.Fs, error)) (f fs.Fs, err error) {
+	createOnFirstUse()
 	fsString = Canonicalize(fsString)
 	created := false
 	value, err := c.Get(fsString, func(fsString string) (f interface{}, ok bool, err error) {
@@ -80,6 +93,7 @@ func GetFn(ctx context.Context, fsString string, create func(ctx context.Context
 
 // Pin f into the cache until Unpin is called
 func Pin(f fs.Fs) {
+	createOnFirstUse()
 	c.Pin(fs.ConfigString(f))
 }
 
@@ -97,6 +111,7 @@ func PinUntilFinalized(f fs.Fs, x interface{}) {
 
 // Unpin f from the cache
 func Unpin(f fs.Fs) {
+	createOnFirstUse()
 	c.Pin(fs.ConfigString(f))
 }
 
@@ -127,6 +142,7 @@ func GetArr(ctx context.Context, fsStrings []string) (f []fs.Fs, err error) {
 
 // Put puts an fs.Fs named fsString into the cache
 func Put(fsString string, f fs.Fs) {
+	createOnFirstUse()
 	canonicalName := fs.ConfigString(f)
 	c.Put(canonicalName, f)
 	addMapping(fsString, canonicalName)
@@ -136,15 +152,18 @@ func Put(fsString string, f fs.Fs) {
 //
 // Returns number of entries deleted
 func ClearConfig(name string) (deleted int) {
+	createOnFirstUse()
 	return c.DeletePrefix(name + ":")
 }
 
 // Clear removes everything from the cache
 func Clear() {
+	createOnFirstUse()
 	c.Clear()
 }
 
 // Entries returns the number of entries in the cache
 func Entries() int {
+	createOnFirstUse()
 	return c.Entries()
 }

fs/cache/cache_test.go

@@ -33,7 +33,7 @@ func mockNewFs(t *testing.T) (func(), func(ctx context.Context, path string) (fs
 		panic("unreachable")
 	}
 	cleanup := func() {
-		c.Clear()
+		Clear()
 	}
 	return cleanup, create
 }
@@ -42,12 +42,12 @@ func TestGet(t *testing.T) {
 	cleanup, create := mockNewFs(t)
 	defer cleanup()
 
-	assert.Equal(t, 0, c.Entries())
+	assert.Equal(t, 0, Entries())
 
 	f, err := GetFn(context.Background(), "mock:/", create)
 	require.NoError(t, err)
 
-	assert.Equal(t, 1, c.Entries())
+	assert.Equal(t, 1, Entries())
 
 	f2, err := GetFn(context.Background(), "mock:/", create)
 	require.NoError(t, err)
@@ -59,13 +59,13 @@ func TestGetFile(t *testing.T) {
 	cleanup, create := mockNewFs(t)
 	defer cleanup()
 
-	assert.Equal(t, 0, c.Entries())
+	assert.Equal(t, 0, Entries())
 
 	f, err := GetFn(context.Background(), "mock:/file.txt", create)
 	require.Equal(t, fs.ErrorIsFile, err)
 	require.NotNil(t, f)
 
-	assert.Equal(t, 2, c.Entries())
+	assert.Equal(t, 2, Entries())
 
 	f2, err := GetFn(context.Background(), "mock:/file.txt", create)
 	require.Equal(t, fs.ErrorIsFile, err)
@@ -85,13 +85,13 @@ func TestGetFile2(t *testing.T) {
 	cleanup, create := mockNewFs(t)
 	defer cleanup()
 
-	assert.Equal(t, 0, c.Entries())
+	assert.Equal(t, 0, Entries())
 
 	f, err := GetFn(context.Background(), "mock:file.txt", create)
 	require.Equal(t, fs.ErrorIsFile, err)
 	require.NotNil(t, f)
 
-	assert.Equal(t, 2, c.Entries())
+	assert.Equal(t, 2, Entries())
 
 	f2, err := GetFn(context.Background(), "mock:file.txt", create)
 	require.Equal(t, fs.ErrorIsFile, err)
@@ -111,13 +111,13 @@ func TestGetError(t *testing.T) {
 	cleanup, create := mockNewFs(t)
 	defer cleanup()
 
-	assert.Equal(t, 0, c.Entries())
+	assert.Equal(t, 0, Entries())
 
 	f, err := GetFn(context.Background(), "mock:/error", create)
 	require.Equal(t, errSentinel, err)
 	require.Equal(t, nil, f)
 
-	assert.Equal(t, 0, c.Entries())
+	assert.Equal(t, 0, Entries())
 }
 
 func TestPut(t *testing.T) {
@@ -126,17 +126,17 @@ func TestPut(t *testing.T) {
 
 	f := mockfs.NewFs(context.Background(), "mock", "/alien")
 
-	assert.Equal(t, 0, c.Entries())
+	assert.Equal(t, 0, Entries())
 
 	Put("mock:/alien", f)
 
-	assert.Equal(t, 1, c.Entries())
+	assert.Equal(t, 1, Entries())
 
 	fNew, err := GetFn(context.Background(), "mock:/alien", create)
 	require.NoError(t, err)
 	require.Equal(t, f, fNew)
 
-	assert.Equal(t, 1, c.Entries())
+	assert.Equal(t, 1, Entries())
 
 	// Check canonicalisation
 
@@ -146,7 +146,7 @@ func TestPut(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, f, fNew)
 
-	assert.Equal(t, 1, c.Entries())
+	assert.Equal(t, 1, Entries())
 
 }
 
@@ -170,7 +170,7 @@ func TestClearConfig(t *testing.T) {
 	cleanup, create := mockNewFs(t)
 	defer cleanup()
 
-	assert.Equal(t, 0, c.Entries())
+	assert.Equal(t, 0, Entries())
 
 	_, err := GetFn(context.Background(), "mock:/file.txt", create)
 	require.Equal(t, fs.ErrorIsFile, err)
@@ -190,11 +190,11 @@ func TestClear(t *testing.T) {
 	_, err := GetFn(context.Background(), "mock:/", create)
 	require.NoError(t, err)
 
-	assert.Equal(t, 1, c.Entries())
+	assert.Equal(t, 1, Entries())
 
 	Clear()
 
-	assert.Equal(t, 0, c.Entries())
+	assert.Equal(t, 0, Entries())
 }
 
 func TestEntries(t *testing.T) {

fs/config.go

@@ -123,6 +123,8 @@ type ConfigInfo struct {
 	RefreshTimes           bool
 	NoConsole              bool
 	TrafficClass           uint8
+	FsCacheExpireDuration  time.Duration
+	FsCacheExpireInterval  time.Duration
 }
 
 // NewConfig creates a new config with everything set to the default
@@ -160,6 +162,8 @@ func NewConfig() *ConfigInfo {
 	c.MultiThreadStreams = 4
 
 	c.TrackRenamesStrategy = "hash"
+	c.FsCacheExpireDuration = 300 * time.Second
+	c.FsCacheExpireInterval = 60 * time.Second
 
 	return c
 }

fs/config/configflags/configflags.go

@@ -128,6 +128,8 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files.")
 	flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window. Supported on Windows only.")
 	flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections. Can be value or names, eg. CS1, LE, DF, AF21.")
+	flags.DurationVarP(flagSet, &ci.FsCacheExpireDuration, "fs-cache-expire-duration", "", ci.FsCacheExpireDuration, "cache remotes for this long (0 to disable caching)")
+	flags.DurationVarP(flagSet, &ci.FsCacheExpireInterval, "fs-cache-expire-interval", "", ci.FsCacheExpireInterval, "interval to check for expired remotes")
 }
 
 // ParseHeaders converts the strings passed in via the header flags into HTTPOptions

lib/cache/cache.go

@@ -28,6 +28,30 @@ func New() *Cache {
 	}
 }
 
+// SetExpireDuration sets the interval at which things expire
+//
+// If it is less than or equal to 0 then things are never cached
+func (c *Cache) SetExpireDuration(d time.Duration) *Cache {
+	c.expireDuration = d
+	return c
+}
+
+// returns true if we aren't to cache anything
+func (c *Cache) noCache() bool {
+	return c.expireDuration <= 0
+}
+
+// SetExpireInterval sets the interval at which the cache expiry runs
+//
+// Set to 0 or a -ve number to disable
+func (c *Cache) SetExpireInterval(d time.Duration) *Cache {
+	if d <= 0 {
+		d = 100 * 365 * 24 * time.Hour
+	}
+	c.expireInterval = d
+	return c
+}
+
 // cacheEntry is stored in the cache
 type cacheEntry struct {
 	value    interface{} // cached item
@@ -69,7 +93,9 @@ func (c *Cache) Get(key string, create CreateFunc) (value interface{}, err error
 			err:   err,
 		}
 		c.mu.Lock()
-		c.cache[key] = entry
+		if !c.noCache() {
+			c.cache[key] = entry
+		}
 	}
 	defer c.mu.Unlock()
 	c.used(entry)
@@ -100,6 +126,9 @@ func (c *Cache) Unpin(key string) {
 func (c *Cache) Put(key string, value interface{}) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	if c.noCache() {
+		return
+	}
 	entry := &cacheEntry{
 		value: value,
 		key:   key,

lib/cache/cache_test.go

@@ -100,7 +100,7 @@ func TestPut(t *testing.T) {
 func TestCacheExpire(t *testing.T) {
 	c, create := setup(t)
 
-	c.expireInterval = time.Millisecond
+	c.SetExpireInterval(time.Millisecond)
 	assert.Equal(t, false, c.expireRunning)
 
 	_, err := c.Get("/", create)
@@ -127,6 +127,31 @@ func TestCacheExpire(t *testing.T) {
 	c.mu.Unlock()
 }
 
+func TestCacheNoExpire(t *testing.T) {
+	c, create := setup(t)
+
+	assert.False(t, c.noCache())
+
+	c.SetExpireDuration(0)
+	assert.Equal(t, false, c.expireRunning)
+
+	assert.True(t, c.noCache())
+
+	f, err := c.Get("/", create)
+	require.NoError(t, err)
+	require.NotNil(t, f)
+
+	c.mu.Lock()
+	assert.Equal(t, 0, len(c.cache))
+	c.mu.Unlock()
+
+	c.Put("/alien", "slime")
+
+	c.mu.Lock()
+	assert.Equal(t, 0, len(c.cache))
+	c.mu.Unlock()
+}
+
 func TestCachePin(t *testing.T) {
 	c, create := setup(t)