rclone/vfs/cache.go

// This deals with caching of files locally

package vfs

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/djherbis/times"
	"github.com/ncw/rclone/fs"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
)

// CacheMode controls the functionality of the cache
type CacheMode byte

// CacheMode options
const (
	CacheModeOff     CacheMode = iota // cache nothing - return errors for writes which can't be satisfied
	CacheModeMinimal                  // cache only the minimum, eg read/write opens
	CacheModeWrites                   // cache all files opened with write intent
	CacheModeFull                     // cache all files opened in any mode
)

var cacheModeToString = []string{
	CacheModeOff:     "off",
	CacheModeMinimal: "minimal",
	CacheModeWrites:  "writes",
	CacheModeFull:    "full",
}

// String turns a CacheMode into a string
func (l CacheMode) String() string {
	if l >= CacheMode(len(cacheModeToString)) {
		return fmt.Sprintf("CacheMode(%d)", l)
	}
	return cacheModeToString[l]
}

// Set a CacheMode
func (l *CacheMode) Set(s string) error {
	for n, name := range cacheModeToString {
		if s != "" && name == s {
			*l = CacheMode(n)
			return nil
		}
	}
	return errors.Errorf("Unknown cache mode level %q", s)
}

// Type of the value
func (l *CacheMode) Type() string {
	return "string"
}

// cache opened files
type cache struct {
	f      fs.Fs                 // fs for the cache directory
	opt    *Options              // vfs Options
	root   string                // root of the cache directory
	itemMu sync.Mutex            // protects the next two maps
	item   map[string]*cacheItem // files in the cache
}

// cacheItem is stored in the item map
type cacheItem struct {
	opens int       // number of times file is open
	atime time.Time // last time file was accessed
}

// newCacheItem returns an item for the cache
func newCacheItem() *cacheItem {
	return &cacheItem{atime: time.Now()}
}

// newCache creates a new cache heirachy for f
//
// This starts background goroutines which can be cancelled with the
// context passed in.
func newCache(ctx context.Context, f fs.Fs, opt *Options) (*cache, error) {
	fRoot := filepath.FromSlash(f.Root())
	if runtime.GOOS == "windows" {
		if strings.HasPrefix(fRoot, `\\?`) {
			fRoot = fRoot[3:]
		}
		fRoot = strings.Replace(fRoot, ":", "", -1)
	}
	root := filepath.Join(fs.CacheDir, "vfs", f.Name(), fRoot)
	fs.Debugf(nil, "vfs cache root is %q", root)

	f, err := fs.NewFs(root)
	if err != nil {
		return nil, errors.Wrap(err, "failed to create cache remote")
	}

	c := &cache{
		f:    f,
		opt:  opt,
		root: root,
		item: make(map[string]*cacheItem),
	}

	go c.cleaner(ctx)

	return c, nil
}

// mkdir makes the directory for name in the cache and returns an os
// path for the file
func (c *cache) mkdir(name string) (string, error) {
	parent := path.Dir(name)
	if parent == "." {
		parent = ""
	}
	leaf := path.Base(name)
	parentPath := filepath.Join(c.root, filepath.FromSlash(parent))
	err := os.MkdirAll(parentPath, 0700)
	if err != nil {
		return "", errors.Wrap(err, "make cache directory failed")
	}
	return filepath.Join(parentPath, leaf), nil
}

// _get gets name from the cache or creates a new one
//
// must be called with itemMu held
func (c *cache) _get(name string) *cacheItem {
	item := c.item[name]
	if item == nil {
		item = newCacheItem()
		c.item[name] = item
	}
	return item
}

// get gets name from the cache or creates a new one
func (c *cache) get(name string) *cacheItem {
	c.itemMu.Lock()
	item := c._get(name)
	c.itemMu.Unlock()
	return item
}

// updateTime sets the atime of the name to that passed in if it is
// newer than the existing or there isn't an existing time.
func (c *cache) updateTime(name string, when time.Time) {
	c.itemMu.Lock()
	item := c._get(name)
	if when.Sub(item.atime) > 0 {
		fs.Debugf(name, "updateTime: setting atime to %v", when)
		item.atime = when
	}
	c.itemMu.Unlock()
}

// open marks name as open
func (c *cache) open(name string) {
	c.itemMu.Lock()
	item := c._get(name)
	item.opens++
	item.atime = time.Now()
	c.itemMu.Unlock()
}

// close marks name as closed
func (c *cache) close(name string) {
	c.itemMu.Lock()
	item := c._get(name)
	item.opens--
	item.atime = time.Now()
	if item.opens < 0 {
		fs.Errorf(name, "cache: double close")
	}
	c.itemMu.Unlock()
}

// cleanUp empties the cache of everything
func (c *cache) cleanUp() error {
	return os.RemoveAll(c.root)
}

// updateAtimes walks the cache updating any atimes it finds
func (c *cache) updateAtimes() error {
	return filepath.Walk(c.root, func(osPath string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !fi.IsDir() {
			// Find path relative to the cache root
			name, err := filepath.Rel(c.root, osPath)
			if err != nil {
				return errors.Wrap(err, "filepath.Rel failed in updatAtimes")
			}
			// And convert into slashes
			name = filepath.ToSlash(name)

			// Update the atime with that of the file
			atime := times.Get(fi).AccessTime()
			c.updateTime(name, atime)
		}
		return nil
	})
}

// purgeOld gets rid of any files that are over age
func (c *cache) purgeOld(maxAge time.Duration) {
	c.itemMu.Lock()
	defer c.itemMu.Unlock()
	cutoff := time.Now().Add(-maxAge)
	for name, item := range c.item {
		// If not locked and access time too long ago - delete the file
		dt := item.atime.Sub(cutoff)
		// fs.Debugf(name, "atime=%v cutoff=%v, dt=%v", item.atime, cutoff, dt)
		if item.opens == 0 && dt < 0 {
			osPath := filepath.Join(c.root, filepath.FromSlash(name))
			err := os.Remove(osPath)
			if err != nil {
				fs.Errorf(name, "Failed to remove from cache: %v", err)
			} else {
				fs.Debugf(name, "Removed from cache")
			}
			// Remove the entry
			delete(c.item, name)
		}
	}
}

// clean empties the cache of stuff if it can
func (c *cache) clean() {
	// Cache may be empty so end
	_, err := os.Stat(c.root)
	if os.IsNotExist(err) {
		return
	}

	fs.Debugf(nil, "Cleaning the cache")

	// first walk the FS to update the atimes
	err = c.updateAtimes()
	if err != nil {
		fs.Errorf(nil, "Error traversing cache %q: %v", c.root, err)
	}

	// Now remove any files that are over age
	c.purgeOld(c.opt.CacheMaxAge)

	// Now tidy up any empty directories
	err = fs.Rmdirs(c.f, "")
	if err != nil {
		fs.Errorf(c.f, "Failed to remove empty directories from cache: %v", err)
	}
}

// cleaner calls clean at regular intervals
//
// doesn't return until context is cancelled
func (c *cache) cleaner(ctx context.Context) {
	timer := time.NewTicker(c.opt.CachePollInterval)
	defer timer.Stop()
	for {
		select {
		case <-timer.C:
			c.clean()
		case <-ctx.Done():
			fs.Debugf(nil, "cache cleaner exiting")
			return
		}
	}
}