rclone/fs/cache/cache.go

// Package cache implements the Fs cache
package cache

import (
	"context"
	"runtime"
	"strings"
	"sync"

	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/filter"
	"github.com/rclone/rclone/lib/cache"
)

var (
	once           sync.Once // creation
	c              *cache.Cache
	mu             sync.Mutex            // mutex to protect remap
	remap          = map[string]string{} // map user supplied names to canonical names - [fsString]canonicalName
	childParentMap = map[string]string{} // tracks a one-to-many relationship between parent dirs and their direct children files - [child]parent
)

// Create the cache just once
func createOnFirstUse() {
	once.Do(func() {
		ci := fs.GetConfig(context.Background())
		c = cache.New()
		c.SetExpireDuration(ci.FsCacheExpireDuration)
		c.SetExpireInterval(ci.FsCacheExpireInterval)
		c.SetFinalizer(func(value interface{}) {
			if s, ok := value.(fs.Shutdowner); ok {
				_ = fs.CountError(s.Shutdown(context.Background()))
			}
		})
	})
}

// Canonicalize looks up fsString in the mapping from user supplied
// names to canonical names and return the canonical form
func Canonicalize(fsString string) string {
	createOnFirstUse()
	mu.Lock()
	canonicalName, ok := remap[fsString]
	mu.Unlock()
	if !ok {
		return fsString
	}
	fs.Debugf(nil, "fs cache: switching user supplied name %q for canonical name %q", fsString, canonicalName)
	return canonicalName
}

// Put in a mapping from fsString => canonicalName if they are different
func addMapping(fsString, canonicalName string) {
	if canonicalName == fsString {
		return
	}
	mu.Lock()
	remap[fsString] = canonicalName
	mu.Unlock()
}

// addChild tracks known file (child) to directory (parent) relationships.
// Note that the canonicalName of a child will always equal that of its parent,
// but not everything with an equal canonicalName is a child.
// It could be an alias or overridden version of a directory.
func addChild(child, parent string) {
	if child == parent {
		return
	}
	mu.Lock()
	childParentMap[child] = parent
	mu.Unlock()
}

// returns true if name is definitely known to be a child (i.e. a file, not a dir).
// returns false if name is a dir or if we don't know.
func isChild(child string) bool {
	mu.Lock()
	_, found := childParentMap[child]
	mu.Unlock()
	return found
}

// ensures that we return fs.ErrorIsFile when necessary
func getError(fsString string, err error) error {
	if err != nil && err != fs.ErrorIsFile {
		return err
	}
	if isChild(fsString) {
		return fs.ErrorIsFile
	}
	return nil
}

// GetFn gets an fs.Fs named fsString either from the cache or creates
// it afresh with the create function
func GetFn(ctx context.Context, fsString string, create func(ctx context.Context, fsString string) (fs.Fs, error)) (f fs.Fs, err error) {
	createOnFirstUse()
	canonicalFsString := Canonicalize(fsString)
	created := false
	value, err := c.Get(canonicalFsString, func(canonicalFsString string) (f interface{}, ok bool, err error) {
		f, err = create(ctx, fsString) // always create the backend with the original non-canonicalised string
		ok = err == nil || err == fs.ErrorIsFile
		created = ok
		return f, ok, err
	})
	f, ok := value.(fs.Fs)
	if err != nil && err != fs.ErrorIsFile {
		if ok {
			return f, err // for possible future uses of PutErr
		}
		return nil, err
	}
	// Check we stored the Fs at the canonical name
	if created {
		canonicalName := fs.ConfigString(f)
		if canonicalName != canonicalFsString {
			if err == nil { // it's a dir
				fs.Debugf(nil, "fs cache: renaming cache item %q to be canonical %q", canonicalFsString, canonicalName)
				value, found := c.Rename(canonicalFsString, canonicalName)
				if found {
					f = value.(fs.Fs)
				}
				addMapping(canonicalFsString, canonicalName)
			} else { // it's a file
				// the fs we cache is always the file's parent, never the file,
				// but we use the childParentMap to return the correct error status based on the fsString passed in.
				fs.Debugf(nil, "fs cache: renaming child cache item %q to be canonical for parent %q", canonicalFsString, canonicalName)
				value, found := c.Rename(canonicalFsString, canonicalName) // rename the file entry to parent
				if found {
					f = value.(fs.Fs) // if parent already exists, use it
				}
				Put(canonicalName, f)                        // force err == nil for the cache
				addMapping(canonicalFsString, canonicalName) // note the fsString-canonicalName connection for future lookups
				addChild(fsString, canonicalName)            // note the file-directory connection for future lookups
			}
		}
	}
	return f, getError(fsString, err) // ensure fs.ErrorIsFile is returned when necessary
}

// Pin f into the cache until Unpin is called
func Pin(f fs.Fs) {
	createOnFirstUse()
	c.Pin(fs.ConfigString(f))
}

// PinUntilFinalized pins f into the cache until x is garbage collected
//
// This calls runtime.SetFinalizer on x so it shouldn't have a
// finalizer already.
func PinUntilFinalized(f fs.Fs, x interface{}) {
	Pin(f)
	runtime.SetFinalizer(x, func(_ interface{}) {
		Unpin(f)
	})
}

// Unpin f from the cache
func Unpin(f fs.Fs) {
	createOnFirstUse()
	c.Unpin(fs.ConfigString(f))
}

// To avoid circular dependencies these are filled in by fs/rc/jobs/job.go
var (
	// JobGetJobID for internal use only
	JobGetJobID func(context.Context) (int64, bool)
	// JobOnFinish for internal use only
	JobOnFinish func(int64, func()) (func(), error)
)

// Get gets an fs.Fs named fsString either from the cache or creates it afresh
func Get(ctx context.Context, fsString string) (f fs.Fs, err error) {
	// If we are making a long lived backend which lives longer
	// than this request, we want to disconnect it from the
	// current context and in particular any WithCancel contexts,
	// but we want to preserve the config embedded in the context.
	newCtx := context.Background()
	newCtx = fs.CopyConfig(newCtx, ctx)
	newCtx = filter.CopyConfig(newCtx, ctx)
	f, err = GetFn(newCtx, fsString, fs.NewFs)
	if f == nil || (err != nil && err != fs.ErrorIsFile) {
		return f, err
	}
	// If this is part of an rc job then pin the backend until it finishes
	if JobOnFinish != nil && JobGetJobID != nil {
		if jobID, ok := JobGetJobID(ctx); ok {
			// fs.Debugf(f, "Pin for job %d", jobID)
			Pin(f)
			_, _ = JobOnFinish(jobID, func() {
				// fs.Debugf(f, "Unpin for job %d", jobID)
				Unpin(f)
			})
		}
	}
	return f, err
}

// GetArr gets []fs.Fs from []fsStrings either from the cache or creates it afresh
func GetArr(ctx context.Context, fsStrings []string) (f []fs.Fs, err error) {
	var fArr []fs.Fs
	for _, fsString := range fsStrings {
		f1, err1 := GetFn(ctx, fsString, fs.NewFs)
		if err1 != nil {
			return fArr, err1
		}
		fArr = append(fArr, f1)
	}
	return fArr, nil
}

// PutErr puts an fs.Fs named fsString into the cache with err
func PutErr(fsString string, f fs.Fs, err error) {
	createOnFirstUse()
	canonicalName := fs.ConfigString(f)
	c.PutErr(canonicalName, f, err)
	addMapping(fsString, canonicalName)
	if err == fs.ErrorIsFile {
		addChild(fsString, canonicalName)
	}
}

// Put puts an fs.Fs named fsString into the cache
func Put(fsString string, f fs.Fs) {
	PutErr(fsString, f, nil)
}

// ClearConfig deletes all entries which were based on the config name passed in
//
// Returns number of entries deleted
func ClearConfig(name string) (deleted int) {
	createOnFirstUse()
	ClearMappingsPrefix(name)
	return c.DeletePrefix(name + ":")
}

// Clear removes everything from the cache
func Clear() {
	createOnFirstUse()
	c.Clear()
	ClearMappings()
}

// Entries returns the number of entries in the cache
func Entries() int {
	createOnFirstUse()
	return c.Entries()
}

// ClearMappings removes everything from remap and childParentMap
func ClearMappings() {
	mu.Lock()
	defer mu.Unlock()
	remap = map[string]string{}
	childParentMap = map[string]string{}
}

// ClearMappingsPrefix deletes all mappings to parents with given prefix
//
// Returns number of entries deleted
func ClearMappingsPrefix(prefix string) (deleted int) {
	mu.Lock()
	do := func(mapping map[string]string) {
		for key, val := range mapping {
			if !strings.HasPrefix(val, prefix) {
				continue
			}
			delete(mapping, key)
			deleted++
		}
	}
	do(remap)
	do(childParentMap)
	mu.Unlock()
	return deleted
}

// EntriesWithPinCount returns the number of pinned and unpinned entries in the cache
//
// Each entry is counted only once, regardless of entry.pinCount
func EntriesWithPinCount() (pinned, unpinned int) {
	createOnFirstUse()
	return c.EntriesWithPinCount()
}