From 40af98b0b3f22b0c16ecef64ba30e0b008c74a03 Mon Sep 17 00:00:00 2001 From: remusb Date: Tue, 30 Jan 2018 00:05:04 +0200 Subject: [PATCH] cache: offline uploading --- backend/cache/cache.go | 646 ++++++-- backend/cache/cache_internal_test.go | 1741 +++++++++++++++------ backend/cache/cache_mount_unix_test.go | 78 + backend/cache/cache_mount_windows_test.go | 124 ++ backend/cache/directory.go | 10 +- backend/cache/handle.go | 188 ++- backend/cache/object.go | 203 +-- backend/cache/plex.go | 39 +- backend/cache/storage_memory.go | 2 - backend/cache/storage_persistent.go | 383 ++++- docs/content/cache.md | 58 +- 11 files changed, 2690 insertions(+), 782 deletions(-) create mode 100644 backend/cache/cache_mount_unix_test.go create mode 100644 backend/cache/cache_mount_windows_test.go diff --git a/backend/cache/cache.go b/backend/cache/cache.go index 9863f92a4..6a08dba5f 100644 --- a/backend/cache/cache.go +++ b/backend/cache/cache.go @@ -47,6 +47,8 @@ const ( DefCacheRps = -1 // DefCacheWrites will cache file data on writes through the cache DefCacheWrites = false + // DefCacheTmpWaitTime says how long should files be stored in local cache before being uploaded + DefCacheTmpWaitTime = "15m" ) // Globals @@ -64,6 +66,8 @@ var ( cacheChunkNoMemory = flags.BoolP("cache-chunk-no-memory", "", DefCacheChunkNoMemory, "Disable the in-memory cache for storing chunks during streaming") cacheRps = flags.IntP("cache-rps", "", int(DefCacheRps), "Limits the number of requests per second to the source FS. -1 disables the rate limiter") cacheStoreWrites = flags.BoolP("cache-writes", "", DefCacheWrites, "Will cache file data on writes through the FS") + cacheTempWritePath = flags.StringP("cache-tmp-upload-path", "", "", "Directory to keep temporary files until they are uploaded to the cloud storage") + cacheTempWaitTime = flags.StringP("cache-tmp-wait-time", "", DefCacheTmpWaitTime, "How long should files be stored in local cache before being uploaded") ) // Register with Fs @@ -140,66 +144,6 @@ func init() { }) } -// ChunkStorage is a storage type that supports only chunk operations (i.e in RAM) -type ChunkStorage interface { - // will check if the chunk is in storage. should be fast and not read the chunk itself if possible - HasChunk(cachedObject *Object, offset int64) bool - - // returns the chunk in storage. return an error if it's not - GetChunk(cachedObject *Object, offset int64) ([]byte, error) - - // add a new chunk - AddChunk(fp string, data []byte, offset int64) error - - // if the storage can cleanup on a cron basis - // otherwise it can do a noop operation - CleanChunksByAge(chunkAge time.Duration) - - // if the storage can cleanup chunks after we no longer need them - // otherwise it can do a noop operation - CleanChunksByNeed(offset int64) - - // if the storage can cleanup chunks after the total size passes a certain point - // otherwise it can do a noop operation - CleanChunksBySize(maxSize int64) -} - -// Storage is a storage type (Bolt) which needs to support both chunk and file based operations -type Storage interface { - ChunkStorage - - // will update/create a directory or an error if it's not found - AddDir(cachedDir *Directory) error - - // will return a directory with all the entries in it or an error if it's not found - GetDirEntries(cachedDir *Directory) (fs.DirEntries, error) - - // remove a directory and all the objects and chunks in it - RemoveDir(fp string) error - - // remove a directory and all the objects and chunks in it - ExpireDir(cd *Directory) error - - // will return an object (file) or error if it doesn't find it - GetObject(cachedObject *Object) (err error) - - // add a new object to its parent directory - // the directory structure (all the parents of this object) is created if its not found - AddObject(cachedObject *Object) error - - // remove an object and all its chunks - RemoveObject(fp string) error - - // Stats returns stats about the cache storage - Stats() (map[string]map[string]interface{}, error) - - // Purge will flush the entire cache - Purge() - - // Close should be called when the program ends gracefully - Close() -} - // Fs represents a wrapped fs.Fs type Fs struct { fs.Fs @@ -208,7 +152,7 @@ type Fs struct { name string root string features *fs.Features // optional features - cache Storage + cache *Persistent fileAge time.Duration chunkSize int64 @@ -219,27 +163,36 @@ type Fs struct { totalMaxWorkers int chunkMemory bool cacheWrites bool + tempWritePath string + tempWriteWait time.Duration + tempFs fs.Fs lastChunkCleanup time.Time cleanupMu sync.Mutex rateLimiter *rate.Limiter plexConnector *plexConnector + backgroundRunner *backgroundWriter + cleanupChan chan bool } -// NewFs contstructs an Fs from the path, container:path +// NewFs constructs a Fs from the path, container:path func NewFs(name, rpath string) (fs.Fs, error) { remote := config.FileGet(name, "remote") if strings.HasPrefix(remote, name+":") { return nil, errors.New("can't point cache remote at itself - check the value of the remote setting") } - // Look for a file first + rpath = strings.Trim(rpath, "/") remotePath := path.Join(remote, rpath) wrappedFs, wrapErr := fs.NewFs(remotePath) - if wrapErr != fs.ErrorIsFile && wrapErr != nil { + if wrapErr != nil && wrapErr != fs.ErrorIsFile { return nil, errors.Wrapf(wrapErr, "failed to make remote %q to wrap", remotePath) } + var fsErr error fs.Debugf(name, "wrapped %v:%v at root %v", wrappedFs.Name(), wrappedFs.Root(), rpath) - + if wrapErr == fs.ErrorIsFile { + fsErr = fs.ErrorIsFile + rpath = cleanPath(path.Dir(rpath)) + } plexURL := config.FileGet(name, "plex_url") plexToken := config.FileGet(name, "plex_token") var chunkSize fs.SizeSuffix @@ -249,7 +202,7 @@ func NewFs(name, rpath string) (fs.Fs, error) { } err := chunkSize.Set(chunkSizeString) if err != nil { - return nil, errors.Wrapf(err, "failed to understand chunk size", chunkSizeString) + return nil, errors.Wrapf(err, "failed to understand chunk size %v", chunkSizeString) } var chunkTotalSize fs.SizeSuffix chunkTotalSizeString := config.FileGet(name, "chunk_total_size", DefCacheTotalChunkSize) @@ -258,7 +211,7 @@ func NewFs(name, rpath string) (fs.Fs, error) { } err = chunkTotalSize.Set(chunkTotalSizeString) if err != nil { - return nil, errors.Wrapf(err, "failed to understand chunk total size", chunkTotalSizeString) + return nil, errors.Wrapf(err, "failed to understand chunk total size %v", chunkTotalSizeString) } chunkCleanIntervalStr := *cacheChunkCleanInterval chunkCleanInterval, err := time.ParseDuration(chunkCleanIntervalStr) @@ -271,7 +224,11 @@ func NewFs(name, rpath string) (fs.Fs, error) { } infoDuration, err := time.ParseDuration(infoAge) if err != nil { - return nil, errors.Wrapf(err, "failed to understand duration", infoAge) + return nil, errors.Wrapf(err, "failed to understand duration %v", infoAge) + } + waitTime, err := time.ParseDuration(*cacheTempWaitTime) + if err != nil { + return nil, errors.Wrapf(err, "failed to understand duration %v", *cacheTempWaitTime) } // configure cache backend if *cacheDbPurge { @@ -291,6 +248,9 @@ func NewFs(name, rpath string) (fs.Fs, error) { chunkMemory: !*cacheChunkNoMemory, cacheWrites: *cacheStoreWrites, lastChunkCleanup: time.Now().Truncate(time.Hour * 24 * 30), + tempWritePath: *cacheTempWritePath, + tempWriteWait: waitTime, + cleanupChan: make(chan bool, 1), } if f.chunkTotalSize < (f.chunkSize * int64(f.totalWorkers)) { return nil, errors.Errorf("don't set cache-total-chunk-size(%v) less than cache-chunk-size(%v) * cache-workers(%v)", @@ -361,7 +321,7 @@ func NewFs(name, rpath string) (fs.Fs, error) { s := <-c if s == syscall.SIGINT || s == syscall.SIGTERM { fs.Debugf(f, "Got signal: %v", s) - f.cache.Close() + f.StopBackgroundRunners() } else if s == syscall.SIGHUP { fs.Infof(f, "Clearing cache from signal") f.DirCacheFlush() @@ -375,9 +335,39 @@ func NewFs(name, rpath string) (fs.Fs, error) { fs.Infof(name, "Chunk Clean Interval: %v", f.chunkCleanInterval.String()) fs.Infof(name, "Workers: %v", f.totalWorkers) fs.Infof(name, "File Age: %v", f.fileAge.String()) - fs.Infof(name, "Cache Writes: %v", f.cacheWrites) + if f.cacheWrites { + fs.Infof(name, "Cache Writes: enabled") + } - go f.CleanUpCache(false) + if f.tempWritePath != "" { + err = os.MkdirAll(f.tempWritePath, os.ModePerm) + if err != nil { + return nil, errors.Wrapf(err, "failed to create cache directory %v", f.tempWritePath) + } + f.tempWritePath = filepath.ToSlash(f.tempWritePath) + f.tempFs, err = fs.NewFs(f.tempWritePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to create temp fs: %v", err) + } + fs.Infof(name, "Upload Temp Rest Time: %v", f.tempWriteWait.String()) + fs.Infof(name, "Upload Temp FS: %v", f.tempWritePath) + f.backgroundRunner, _ = initBackgroundUploader(f) + go f.backgroundRunner.run() + } + + go func() { + for { + time.Sleep(f.chunkCleanInterval) + select { + case <-f.cleanupChan: + fs.Infof(f, "stopping cleanup") + return + default: + fs.Debugf(f, "starting cleanup") + f.CleanUpCache(false) + } + } + }() // TODO: Explore something here but now it's not something we want // when writing from cache, source FS will send a notification and clear it out immediately @@ -394,21 +384,30 @@ func NewFs(name, rpath string) (fs.Fs, error) { f.features = (&fs.Features{ CanHaveEmptyDirectories: true, DuplicateFiles: false, // storage doesn't permit this - Purge: f.Purge, - Copy: f.Copy, - Move: f.Move, - DirMove: f.DirMove, DirChangeNotify: nil, - PutUnchecked: f.PutUnchecked, - PutStream: f.PutStream, - CleanUp: f.CleanUp, - UnWrap: f.UnWrap, - WrapFs: f.WrapFs, - SetWrapper: f.SetWrapper, }).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs) + // override only those features that use a temp fs and it doesn't support them + if f.tempWritePath != "" { + if f.tempFs.Features().Copy == nil { + f.features.Copy = nil + } + if f.tempFs.Features().Move == nil { + f.features.Move = nil + } + if f.tempFs.Features().Move == nil { + f.features.Move = nil + } + if f.tempFs.Features().DirMove == nil { + f.features.DirMove = nil + } + if f.tempFs.Features().MergeDirs == nil { + f.features.MergeDirs = nil + } + } + // even if the wrapped fs doesn't support it, we still want it f.features.DirCacheFlush = f.DirCacheFlush - return f, wrapErr + return f, fsErr } // Name of the remote (as passed into NewFs) @@ -428,7 +427,7 @@ func (f *Fs) Features() *fs.Features { // String returns a description of the FS func (f *Fs) String() string { - return fmt.Sprintf("%s:%s", f.name, f.root) + return fmt.Sprintf("Cache remote %s:%s", f.name, f.root) } // ChunkSize returns the configured chunk size @@ -436,33 +435,67 @@ func (f *Fs) ChunkSize() int64 { return f.chunkSize } +// InfoAge returns the configured file age +func (f *Fs) InfoAge() time.Duration { + return f.fileAge +} + +// TempUploadWaitTime returns the configured temp file upload wait time +func (f *Fs) TempUploadWaitTime() time.Duration { + return f.tempWriteWait +} + // NewObject finds the Object at remote. func (f *Fs) NewObject(remote string) (fs.Object, error) { + var err error + + fs.Debugf(f, "new object '%s'", remote) co := NewObject(f, remote) - err := f.cache.GetObject(co) + // search for entry in cache and validate it + err = f.cache.GetObject(co) if err != nil { fs.Debugf(remote, "find: error: %v", err) } else if time.Now().After(co.CacheTs.Add(f.fileAge)) { - fs.Debugf(remote, "find: cold object ts: %v", co.CacheTs) + fs.Debugf(co, "find: cold object: %+v", co) } else { - fs.Debugf(remote, "find: warm object ts: %v", co.CacheTs) + fs.Debugf(co, "find: warm object: %v, expiring on: %v", co, co.CacheTs.Add(f.fileAge)) return co, nil } - obj, err := f.Fs.NewObject(remote) + + // search for entry in source or temp fs + var obj fs.Object + err = nil + if f.tempWritePath != "" { + obj, err = f.tempFs.NewObject(remote) + // not found in temp fs + if err != nil { + fs.Debugf(remote, "find: not found in local cache fs") + obj, err = f.Fs.NewObject(remote) + } else { + fs.Debugf(obj, "find: found in local cache fs") + } + } else { + obj, err = f.Fs.NewObject(remote) + } + + // not found in either fs if err != nil { + fs.Debugf(obj, "find failed: not found in either local or remote fs") return nil, err } - co = ObjectFromOriginal(f, obj) - co.persist() + + // cache the new entry + co = ObjectFromOriginal(f, obj).persist() + fs.Debugf(co, "find: cached object") return co, nil } // List the objects and directories in dir into entries func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { - // clean cache - go f.CleanUpCache(false) - + fs.Debugf(f, "list '%s'", dir) cd := ShallowDirectory(f, dir) + + // search for cached dir entries and validate them entries, err = f.cache.GetDirEntries(cd) if err != nil { fs.Debugf(dir, "list: error: %v", err) @@ -472,40 +505,83 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) { // TODO: read empty dirs from source? fs.Debugf(dir, "list: empty listing") } else { - fs.Debugf(dir, "list: warm %v from cache for: %v, ts: %v", len(entries), cd.abs(), cd.CacheTs) + fs.Debugf(dir, "list: warm %v from cache for: %v, expiring on: %v", len(entries), cd.abs(), cd.CacheTs.Add(f.fileAge)) + fs.Debugf(dir, "list: cached entries: %v", entries) return entries, nil } + // FIXME need to clean existing cached listing + // we first search any temporary files stored locally + var cachedEntries fs.DirEntries + if f.tempWritePath != "" { + queuedEntries, err := f.cache.searchPendingUploadFromDir(cd.abs()) + if err != nil { + fs.Errorf(dir, "list: error getting pending uploads: %v", err) + } else { + fs.Debugf(dir, "list: read %v from temp fs", len(queuedEntries)) + fs.Debugf(dir, "list: temp fs entries: %v", queuedEntries) + + for _, queuedRemote := range queuedEntries { + queuedEntry, err := f.tempFs.NewObject(f.cleanRootFromPath(queuedRemote)) + if err != nil { + fs.Debugf(dir, "list: temp file not found in local fs: %v", err) + continue + } + co := ObjectFromOriginal(f, queuedEntry).persist() + fs.Debugf(co, "list: cached temp object") + cachedEntries = append(cachedEntries, co) + } + } + } + + // search from the source entries, err = f.Fs.List(dir) if err != nil { return nil, err } fs.Debugf(dir, "list: read %v from source", len(entries)) + fs.Debugf(dir, "list: source entries: %v", entries) - var cachedEntries fs.DirEntries + // and then iterate over the ones from source (temp Objects will override source ones) for _, entry := range entries { switch o := entry.(type) { case fs.Object: - co := ObjectFromOriginal(f, o) - co.persist() + // skip over temporary objects (might be uploading) + found := false + for _, t := range cachedEntries { + if t.Remote() == o.Remote() { + found = true + break + } + } + if found { + continue + } + co := ObjectFromOriginal(f, o).persist() cachedEntries = append(cachedEntries, co) + fs.Debugf(dir, "list: cached object: %v", co) case fs.Directory: - cd := DirectoryFromOriginal(f, o) - err = f.cache.AddDir(cd) - cachedEntries = append(cachedEntries, cd) + cdd := DirectoryFromOriginal(f, o) + err := f.cache.AddDir(cdd) + if err != nil { + fs.Errorf(dir, "list: error caching dir from listing %v", o) + } else { + fs.Debugf(dir, "list: cached dir: %v", cdd) + } + cachedEntries = append(cachedEntries, cdd) default: - err = errors.Errorf("Unknown object type %T", entry) + fs.Debugf(entry, "list: Unknown object type %T", entry) } } + + // cache dir meta + t := time.Now() + cd.CacheTs = &t + err = f.cache.AddDir(cd) if err != nil { - fs.Errorf(dir, "err caching listing: %v", err) + fs.Errorf(cd, "list: save error: '%v'", err) } else { - t := time.Now() - cd.CacheTs = &t - err := f.cache.AddDir(cd) - if err != nil { - fs.Errorf(cd, "list: save error: %v", err) - } + fs.Debugf(dir, "list: cached dir: '%v', cache ts: %v", cd.abs(), cd.CacheTs) } return cachedEntries, nil @@ -574,62 +650,105 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) { // Mkdir makes the directory (container, bucket) func (f *Fs) Mkdir(dir string) error { + fs.Debugf(f, "mkdir '%s'", dir) err := f.Fs.Mkdir(dir) if err != nil { return err } - if dir == "" && f.Root() == "" { // creating the root is possible but we don't need that cached as we have it already - fs.Debugf(dir, "skipping empty dir in cache") - return nil - } - fs.Infof(f, "create dir '%s'", dir) + fs.Debugf(dir, "mkdir: created dir in source fs") - // expire parent of new dir cd := NewDirectory(f, cleanPath(dir)) err = f.cache.AddDir(cd) if err != nil { fs.Errorf(dir, "mkdir: add error: %v", err) + } else { + fs.Debugf(cd, "mkdir: added to cache") } + // expire parent of new dir parentCd := NewDirectory(f, cleanPath(path.Dir(dir))) err = f.cache.ExpireDir(parentCd) if err != nil { - fs.Errorf(dir, "mkdir: expire error: %v", err) + fs.Errorf(parentCd, "mkdir: cache expire error: %v", err) + } else { + fs.Infof(parentCd, "mkdir: cache expired") } - // clean cache - go f.CleanUpCache(false) return nil } // Rmdir removes the directory (container, bucket) if empty func (f *Fs) Rmdir(dir string) error { - err := f.Fs.Rmdir(dir) - if err != nil { - return err + fs.Debugf(f, "rmdir '%s'", dir) + + if f.tempWritePath != "" { + // pause background uploads + f.backgroundRunner.pause() + defer f.backgroundRunner.play() + + // we check if the source exists on the remote and make the same move on it too if it does + // otherwise, we skip this step + _, err := f.UnWrap().List(dir) + if err == nil { + err := f.Fs.Rmdir(dir) + if err != nil { + return err + } + fs.Debugf(dir, "rmdir: removed dir in source fs") + } + + var queuedEntries []*Object + err = walk.Walk(f.tempFs, dir, true, -1, func(path string, entries fs.DirEntries, err error) error { + for _, o := range entries { + if oo, ok := o.(fs.Object); ok { + co := ObjectFromOriginal(f, oo) + queuedEntries = append(queuedEntries, co) + } + } + return nil + }) + if err != nil { + fs.Errorf(dir, "rmdir: error getting pending uploads: %v", err) + } else { + fs.Debugf(dir, "rmdir: read %v from temp fs", len(queuedEntries)) + fs.Debugf(dir, "rmdir: temp fs entries: %v", queuedEntries) + if len(queuedEntries) > 0 { + fs.Errorf(dir, "rmdir: temporary dir not empty") + return fs.ErrorDirectoryNotEmpty + } + } + } else { + err := f.Fs.Rmdir(dir) + if err != nil { + return err + } + fs.Debugf(dir, "rmdir: removed dir in source fs") } - fs.Infof(f, "rm dir '%s'", dir) // remove dir data d := NewDirectory(f, dir) - err = f.cache.RemoveDir(d.abs()) + err := f.cache.RemoveDir(d.abs()) if err != nil { fs.Errorf(dir, "rmdir: remove error: %v", err) + } else { + fs.Debugf(d, "rmdir: removed from cache") } // expire parent parentCd := NewDirectory(f, cleanPath(path.Dir(dir))) err = f.cache.ExpireDir(parentCd) if err != nil { - fs.Errorf(dir, "rmdir: expire error: %v", err) + fs.Errorf(dir, "rmdir: cache expire error: %v", err) + } else { + fs.Infof(parentCd, "rmdir: cache expired") } - // clean cache - go f.CleanUpCache(false) return nil } // DirMove moves src, srcRemote to this remote at dstRemote // using server side move operations. func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { + fs.Debugf(f, "move dir '%s'/'%s' -> '%s'/'%s'", src.Root(), srcRemote, f.Root(), dstRemote) + do := f.Fs.Features().DirMove if do == nil { return fs.ErrorCantDirMove @@ -643,36 +762,91 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error { fs.Errorf(srcFs, "can't move directory - not wrapping same remotes") return fs.ErrorCantDirMove } - fs.Infof(f, "move dir '%s'/'%s' -> '%s'", srcRemote, srcFs.Root(), dstRemote) - err := do(src.Features().UnWrap(), srcRemote, dstRemote) - if err != nil { - return err + if f.tempWritePath != "" { + // pause background uploads + f.backgroundRunner.pause() + defer f.backgroundRunner.play() + + // we check if the source exists on the remote and make the same move on it too if it does + // otherwise, we skip this step + _, err := srcFs.UnWrap().List(srcRemote) + if err == nil { + err := do(srcFs.UnWrap(), srcRemote, dstRemote) + if err != nil { + return err + } + fs.Debugf(srcRemote, "movedir: dir moved in the source fs") + } + + var queuedEntries []*Object + err = walk.Walk(f.tempFs, srcRemote, true, -1, func(path string, entries fs.DirEntries, err error) error { + for _, o := range entries { + if oo, ok := o.(fs.Object); ok { + co := ObjectFromOriginal(f, oo) + queuedEntries = append(queuedEntries, co) + if co.tempFileStartedUpload() { + fs.Errorf(co, "can't move - upload has already started. need to finish that") + return fs.ErrorCantDirMove + } + } + } + return nil + }) + if err != nil { + return err + } + fs.Debugf(srcRemote, "dirmove: read %v from temp fs", len(queuedEntries)) + fs.Debugf(srcRemote, "dirmove: temp fs entries: %v", queuedEntries) + + do := f.tempFs.Features().DirMove + if do == nil { + fs.Errorf(srcRemote, "dirmove: can't move dir in temp fs") + return fs.ErrorCantDirMove + } + err = do(f.tempFs, srcRemote, dstRemote) + if err != nil { + return err + } + err = f.cache.ReconcileTempUploads(f) + if err != nil { + return err + } + } else { + err := do(srcFs.UnWrap(), srcRemote, dstRemote) + if err != nil { + return err + } + fs.Debugf(srcRemote, "movedir: dir moved in the source fs") } // delete src dir from cache along with all chunks srcDir := NewDirectory(srcFs, srcRemote) - err = f.cache.RemoveDir(srcDir.abs()) + err := f.cache.RemoveDir(srcDir.abs()) if err != nil { - fs.Errorf(srcRemote, "dirmove: remove error: %v", err) + fs.Errorf(srcDir, "dirmove: remove error: %v", err) + } else { + fs.Debugf(srcDir, "dirmove: removed cached dir") } // expire src parent srcParent := NewDirectory(f, cleanPath(path.Dir(srcRemote))) err = f.cache.ExpireDir(srcParent) if err != nil { - fs.Errorf(srcRemote, "dirmove: expire error: %v", err) + fs.Errorf(srcParent, "dirmove: cache expire error: %v", err) + } else { + fs.Debugf(srcParent, "dirmove: cache expired") } // expire parent dir at the destination path dstParent := NewDirectory(f, cleanPath(path.Dir(dstRemote))) err = f.cache.ExpireDir(dstParent) if err != nil { - fs.Errorf(dstRemote, "dirmove: expire error: %v", err) + fs.Errorf(dstParent, "dirmove: cache expire error: %v", err) + } else { + fs.Debugf(dstParent, "dirmove: cache expired") } // TODO: precache dst dir and save the chunks - // clean cache - go f.CleanUpCache(false) return nil } @@ -746,32 +920,60 @@ type putFn func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.O func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) { var err error var obj fs.Object - if f.cacheWrites { + + // queue for upload and store in temp fs if configured + if f.tempWritePath != "" { + obj, err = f.tempFs.Put(in, src, options...) + if err != nil { + fs.Errorf(obj, "put: failed to upload in temp fs: %v", err) + return nil, err + } + fs.Infof(obj, "put: uploaded in temp fs") + err = f.cache.addPendingUpload(path.Join(f.Root(), src.Remote()), false) + if err != nil { + fs.Errorf(obj, "put: failed to queue for upload: %v", err) + return nil, err + } + fs.Infof(obj, "put: queued for upload") + // if cache writes is enabled write it first through cache + } else if f.cacheWrites { f.cacheReader(in, src, func(inn io.Reader) { obj, err = put(inn, src, options...) }) + if err == nil { + fs.Debugf(obj, "put: uploaded to remote fs and saved in cache") + } + // last option: save it directly in remote fs } else { obj, err = put(in, src, options...) + if err == nil { + fs.Debugf(obj, "put: uploaded to remote fs") + } } + // validate and stop if errors are found if err != nil { - fs.Errorf(src, "error saving in cache: %v", err) + fs.Errorf(src, "put: error uploading: %v", err) return nil, err } + + // cache the new file cachedObj := ObjectFromOriginal(f, obj).persist() + fs.Debugf(cachedObj, "put: added to cache") // expire parent - err = f.cache.ExpireDir(cachedObj.parentDir()) + parentCd := NewDirectory(f, cleanPath(path.Dir(cachedObj.Remote()))) + err = f.cache.ExpireDir(parentCd) if err != nil { - fs.Errorf(cachedObj, "put: expire error: %v", err) + fs.Errorf(cachedObj, "put: cache expire error: %v", err) + } else { + fs.Infof(parentCd, "put: cache expired") } - // clean cache - go f.CleanUpCache(false) return cachedObj, nil } // Put in to the remote path with the modTime given of the given size func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { - fs.Infof(f, "put data at '%s'", src.Remote()) + fs.Debugf(f, "put data at '%s'", src.Remote()) return f.put(in, src, options, f.Fs.Put) } @@ -781,7 +983,7 @@ func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOpt if do == nil { return nil, errors.New("can't PutUnchecked") } - fs.Infof(f, "put data unchecked in '%s'", src.Remote()) + fs.Debugf(f, "put data unchecked in '%s'", src.Remote()) return f.put(in, src, options, do) } @@ -791,111 +993,172 @@ func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption if do == nil { return nil, errors.New("can't PutStream") } - fs.Infof(f, "put data streaming in '%s'", src.Remote()) + fs.Debugf(f, "put data streaming in '%s'", src.Remote()) return f.put(in, src, options, do) } // Copy src to this remote using server side copy operations. func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + fs.Debugf(f, "copy obj '%s' -> '%s'", src, remote) + do := f.Fs.Features().Copy if do == nil { fs.Errorf(src, "source remote (%v) doesn't support Copy", src.Fs()) return nil, fs.ErrorCantCopy } - + // the source must be a cached object or we abort srcObj, ok := src.(*Object) if !ok { fs.Errorf(srcObj, "can't copy - not same remote type") return nil, fs.ErrorCantCopy } + // both the source cache fs and this cache fs need to wrap the same remote if srcObj.CacheFs.Fs.Name() != f.Fs.Name() { - fs.Errorf(srcObj, "can't copy - not wrapping same remote types") + fs.Errorf(srcObj, "can't copy - not wrapping same remotes") + return nil, fs.ErrorCantCopy + } + // refresh from source or abort + if err := srcObj.refreshFromSource(false); err != nil { + fs.Errorf(f, "can't copy %v - %v", src, err) return nil, fs.ErrorCantCopy } - fs.Infof(f, "copy obj '%s' -> '%s'", srcObj.abs(), remote) - // store in cache - if err := srcObj.refreshFromSource(); err != nil { - fs.Errorf(f, "can't move %v - %v", src, err) - return nil, fs.ErrorCantCopy + if srcObj.isTempFile() { + // we check if the feature is stil active + if f.tempWritePath == "" { + fs.Errorf(srcObj, "can't copy - this is a local cached file but this feature is turned off this run") + return nil, fs.ErrorCantCopy + } + + do = srcObj.ParentFs.Features().Copy + if do == nil { + fs.Errorf(src, "parent remote (%v) doesn't support Copy", srcObj.ParentFs) + return nil, fs.ErrorCantCopy + } } + obj, err := do(srcObj.Object, remote) if err != nil { fs.Errorf(srcObj, "error moving in cache: %v", err) return nil, err } + fs.Debugf(obj, "copy: file copied") // persist new co := ObjectFromOriginal(f, obj).persist() + fs.Debugf(co, "copy: added to cache") // expire the destination path - err = f.cache.ExpireDir(co.parentDir()) + parentCd := NewDirectory(f, cleanPath(path.Dir(co.Remote()))) + err = f.cache.ExpireDir(parentCd) if err != nil { - fs.Errorf(co, "copy: expire error: %v", err) + fs.Errorf(parentCd, "copy: cache expire error: %v", err) + } else { + fs.Infof(parentCd, "copy: cache expired") } - // expire src parent srcParent := NewDirectory(f, cleanPath(path.Dir(src.Remote()))) err = f.cache.ExpireDir(srcParent) if err != nil { - fs.Errorf(src, "copy: expire error: %v", err) + fs.Errorf(srcParent, "copy: cache expire error: %v", err) + } else { + fs.Infof(srcParent, "copy: cache expired") } - // clean cache - go f.CleanUpCache(false) return co, nil } // Move src to this remote using server side move operations. func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { + fs.Debugf(f, "moving obj '%s' -> %s", src, remote) + + // if source fs doesn't support move abort do := f.Fs.Features().Move if do == nil { fs.Errorf(src, "source remote (%v) doesn't support Move", src.Fs()) return nil, fs.ErrorCantMove } - + // the source must be a cached object or we abort srcObj, ok := src.(*Object) if !ok { fs.Errorf(srcObj, "can't move - not same remote type") return nil, fs.ErrorCantMove } + // both the source cache fs and this cache fs need to wrap the same remote if srcObj.CacheFs.Fs.Name() != f.Fs.Name() { fs.Errorf(srcObj, "can't move - not wrapping same remote types") return nil, fs.ErrorCantMove } - fs.Infof(f, "moving obj '%s' -> %s", srcObj.abs(), remote) - - // save in cache - if err := srcObj.refreshFromSource(); err != nil { + // refresh from source or abort + if err := srcObj.refreshFromSource(false); err != nil { fs.Errorf(f, "can't move %v - %v", src, err) return nil, fs.ErrorCantMove } + + // if this is a temp object then we perform the changes locally + if srcObj.isTempFile() { + // we check if the feature is stil active + if f.tempWritePath == "" { + fs.Errorf(srcObj, "can't move - this is a local cached file but this feature is turned off this run") + return nil, fs.ErrorCantMove + } + // pause background uploads + f.backgroundRunner.pause() + defer f.backgroundRunner.play() + + // started uploads can't be moved until they complete + if srcObj.tempFileStartedUpload() { + fs.Errorf(srcObj, "can't move - upload has already started. need to finish that") + return nil, fs.ErrorCantMove + } + do = f.tempFs.Features().Move + + // we must also update the pending queue + err := f.cache.updatePendingUpload(srcObj.abs(), func(item *tempUploadInfo) error { + item.DestPath = path.Join(f.Root(), remote) + item.AddedOn = time.Now() + return nil + }) + if err != nil { + fs.Errorf(srcObj, "failed to rename queued file for upload: %v", err) + return nil, fs.ErrorCantMove + } + fs.Debugf(srcObj, "move: queued file moved to %v", remote) + } + obj, err := do(srcObj.Object, remote) if err != nil { - fs.Errorf(srcObj, "error moving in cache: %v", err) + fs.Errorf(srcObj, "error moving: %v", err) return nil, err } + fs.Debugf(obj, "move: file moved") // remove old err = f.cache.RemoveObject(srcObj.abs()) if err != nil { fs.Errorf(srcObj, "move: remove error: %v", err) + } else { + fs.Debugf(srcObj, "move: removed from cache") } // expire old parent - err = f.cache.ExpireDir(srcObj.parentDir()) + parentCd := NewDirectory(f, cleanPath(path.Dir(srcObj.Remote()))) + err = f.cache.ExpireDir(parentCd) if err != nil { - fs.Errorf(srcObj, "move: expire error: %v", err) + fs.Errorf(parentCd, "move: parent cache expire error: %v", err) + } else { + fs.Infof(parentCd, "move: cache expired") } - // persist new cachedObj := ObjectFromOriginal(f, obj).persist() + fs.Debugf(cachedObj, "move: added to cache") // expire new parent - err = f.cache.ExpireDir(cachedObj.parentDir()) + parentCd = NewDirectory(f, cleanPath(path.Dir(cachedObj.Remote()))) + err = f.cache.ExpireDir(parentCd) if err != nil { - fs.Errorf(cachedObj, "move: expire error: %v", err) + fs.Errorf(parentCd, "move: expire error: %v", err) + } else { + fs.Infof(parentCd, "move: cache expired") } - // clean cache - go f.CleanUpCache(false) return cachedObj, nil } @@ -939,8 +1202,8 @@ func (f *Fs) Stats() (map[string]map[string]interface{}, error) { return f.cache.Stats() } -// OpenRateLimited will execute a closure under a rate limiter watch -func (f *Fs) OpenRateLimited(fn func() (io.ReadCloser, error)) (io.ReadCloser, error) { +// openRateLimited will execute a closure under a rate limiter watch +func (f *Fs) openRateLimited(fn func() (io.ReadCloser, error)) (io.ReadCloser, error) { var err error ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() @@ -968,6 +1231,17 @@ func (f *Fs) CleanUpCache(ignoreLastTs bool) { } } +// StopBackgroundRunners will signall all the runners to stop their work +// can be triggered from a terminate signal or from testing between runs +func (f *Fs) StopBackgroundRunners() { + f.cleanupChan <- false + if f.tempWritePath != "" { + f.backgroundRunner.close() + } + f.cache.Close() + fs.Debugf(f, "Services stopped") +} + // UnWrap returns the Fs that this Fs is wrapping func (f *Fs) UnWrap() fs.Fs { return f.Fs @@ -983,7 +1257,7 @@ func (f *Fs) SetWrapper(wrapper fs.Fs) { f.wrapper = wrapper } -// Wrap returns the Fs that is wrapping this Fs +// isWrappedByCrypt checks if this is wrapped by a crypt remote func (f *Fs) isWrappedByCrypt() (*crypt.Fs, bool) { if f.wrapper == nil { return nil, false @@ -992,11 +1266,39 @@ func (f *Fs) isWrappedByCrypt() (*crypt.Fs, bool) { return c, ok } +// cleanRootFromPath trims the root of the current fs from a path +func (f *Fs) cleanRootFromPath(p string) string { + if f.Root() != "" { + p = p[len(f.Root()):] // trim out root + if len(p) > 0 { // remove first separator + p = p[1:] + } + } + + return p +} + +func (f *Fs) isRootInPath(p string) bool { + if f.Root() == "" { + return true + } + return strings.HasPrefix(p, f.Root()+"/") +} + // DirCacheFlush flushes the dir cache func (f *Fs) DirCacheFlush() { _ = f.cache.RemoveDir("") } +// GetBackgroundUploadChannel returns a channel that can be listened to for remote activities that happen +// in the background +func (f *Fs) GetBackgroundUploadChannel() chan BackgroundUploadState { + if f.tempWritePath != "" { + return f.backgroundRunner.notifyCh + } + return nil +} + func cleanPath(p string) string { p = path.Clean(p) if p == "." || p == "/" { diff --git a/backend/cache/cache_internal_test.go b/backend/cache/cache_internal_test.go index f9523b62e..cb7ca5d03 100644 --- a/backend/cache/cache_internal_test.go +++ b/backend/cache/cache_internal_test.go @@ -4,188 +4,267 @@ package cache_test import ( "bytes" - "fmt" "io" "io/ioutil" + "log" "math/rand" + "os" "path" "path/filepath" "runtime" "strconv" + "strings" "testing" "time" - //"os" - "os/exec" - //"strings" + "github.com/pkg/errors" + + "encoding/base64" + goflag "flag" + "fmt" + "runtime/debug" "github.com/ncw/rclone/backend/cache" - "github.com/ncw/rclone/fs/config" - "github.com/ncw/rclone/fs/object" - //"github.com/ncw/rclone/cmd/mount" - //_ "github.com/ncw/rclone/cmd/cmount" - //"github.com/ncw/rclone/cmd/mountlib" + "github.com/ncw/rclone/backend/crypt" _ "github.com/ncw/rclone/backend/drive" "github.com/ncw/rclone/backend/local" "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/config" + "github.com/ncw/rclone/fs/object" "github.com/ncw/rclone/fstest" + "github.com/ncw/rclone/vfs" + "github.com/ncw/rclone/vfs/vfsflags" flag "github.com/spf13/pflag" "github.com/stretchr/testify/require" ) -var ( - infoAge = time.Second * 10 - chunkClean = time.Second - okDiff = time.Second * 9 // really big diff here but the build machines seem to be slow. need a different way for this - workers = 2 +const ( + // these 2 passwords are test random + cryptPassword1 = "3XcvMMdsV3d-HGAReTMdNH-5FcX5q32_lUeA" // oGJdUbQc7s8 + cryptPassword2 = "NlgTBEIe-qibA7v-FoMfuX6Cw8KlLai_aMvV" // mv4mZW572HM + cryptedTextBase64 = "UkNMT05FAAC320i2xIee0BiNyknSPBn+Qcw3q9FhIFp3tvq6qlqvbsno3PnxmEFeJG3jDBnR/wku2gHWeQ==" // one content + cryptedText2Base64 = "UkNMT05FAAATcQkVsgjBh8KafCKcr0wdTa1fMmV0U8hsCLGFoqcvxKVmvv7wx3Hf5EXxFcki2FFV4sdpmSrb9Q==" // updated content ) +var ( + remoteName string + mountDir string + uploadDir string + useMount bool + runInstance *run + errNotSupported = errors.New("not supported") + decryptedToEncryptedRemotes = map[string]string{ + "one": "lm4u7jjt3c85bf56vjqgeenuno", + "second": "qvt1ochrkcfbptp5mu9ugb2l14", + "test": "jn4tegjtpqro30t3o11thb4b5s", + "test2": "qakvqnh8ttei89e0gc76crpql4", + "data.bin": "0q2847tfko6mhj3dag3r809qbc", + "ticw/data.bin": "5mv97b0ule6pht33srae5pice8/0q2847tfko6mhj3dag3r809qbc", + "tiutfo/test/one": "legd371aa8ol36tjfklt347qnc/jn4tegjtpqro30t3o11thb4b5s/lm4u7jjt3c85bf56vjqgeenuno", + "tiuufo/test/one": "vi6u1olqhirqv14cd8qlej1mgo/jn4tegjtpqro30t3o11thb4b5s/lm4u7jjt3c85bf56vjqgeenuno", + "tiutfo/second/one": "legd371aa8ol36tjfklt347qnc/qvt1ochrkcfbptp5mu9ugb2l14/lm4u7jjt3c85bf56vjqgeenuno", + "second/one": "qvt1ochrkcfbptp5mu9ugb2l14/lm4u7jjt3c85bf56vjqgeenuno", + "test/one": "jn4tegjtpqro30t3o11thb4b5s/lm4u7jjt3c85bf56vjqgeenuno", + "test/second": "jn4tegjtpqro30t3o11thb4b5s/qvt1ochrkcfbptp5mu9ugb2l14", + "test/third": "jn4tegjtpqro30t3o11thb4b5s/2nd7fjiop5h3ihfj1vl953aa5g", + "test/0.bin": "jn4tegjtpqro30t3o11thb4b5s/e6frddt058b6kvbpmlstlndmtk", + "test/1.bin": "jn4tegjtpqro30t3o11thb4b5s/kck472nt1k7qbmob0mt1p1crgc", + "test/2.bin": "jn4tegjtpqro30t3o11thb4b5s/744oe9ven2rmak4u27if51qk24", + "test/3.bin": "jn4tegjtpqro30t3o11thb4b5s/2bjd8kef0u5lmsu6qhqll34bcs", + "test/4.bin": "jn4tegjtpqro30t3o11thb4b5s/cvjs73iv0a82v0c7r67avllh7s", + "test/5.bin": "jn4tegjtpqro30t3o11thb4b5s/0plkdo790b6bnmt33qsdqmhv9c", + "test/6.bin": "jn4tegjtpqro30t3o11thb4b5s/s5r633srnjtbh83893jovjt5d0", + "test/7.bin": "jn4tegjtpqro30t3o11thb4b5s/6rq45tr9bjsammku622flmqsu4", + "test/8.bin": "jn4tegjtpqro30t3o11thb4b5s/37bc6tcl3e31qb8cadvjb749vk", + "test/9.bin": "jn4tegjtpqro30t3o11thb4b5s/t4pr35hnls32789o8fk0chk1ec", + } +) + +func init() { + goflag.StringVar(&remoteName, "remote-internal", "TestCache", "Remote to test with, defaults to local filesystem") + goflag.StringVar(&mountDir, "mount-dir-internal", "", "") + goflag.StringVar(&uploadDir, "upload-dir-internal", "", "") + goflag.BoolVar(&useMount, "cache-use-mount", false, "Test only with mount") +} + +// TestMain drives the tests +func TestMain(m *testing.M) { + goflag.Parse() + var rc int + + runInstance = newRun() + rc = m.Run() + os.Exit(rc) +} + func TestInternalListRootAndInnerRemotes(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "tilrair-local", "tilrair-cache", nil) - defer cleanupFs(t, rootFs, boltDb) + id := fmt.Sprintf("tilrair%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) // Instantiate inner fs innerFolder := "inner" - err := rootFs.Mkdir(innerFolder) - require.NoError(t, err) - innerFs, err := fs.NewFs("tilrair-cache:" + innerFolder) + runInstance.mkdir(t, rootFs, innerFolder) + rootFs2, boltDb2 := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs2, boltDb2) + + runInstance.writeObjectString(t, rootFs2, "one", "content") + listRoot := runInstance.list(t, rootFs, "") + listRootInner := runInstance.list(t, rootFs, innerFolder) + listInner, err := rootFs2.List("") require.NoError(t, err) - obj := writeObjectString(t, innerFs, "one", "content") - - listRoot, err := rootFs.List("") - require.NoError(t, err) - listRootInner, err := rootFs.List(innerFolder) - require.NoError(t, err) - listInner, err := innerFs.List("") - require.NoError(t, err) - - require.Lenf(t, listRoot, 1, "remote %v should have 1 entry", rootFs.Root()) - require.Lenf(t, listRootInner, 1, "remote %v should have 1 entry in %v", rootFs.Root(), innerFolder) - require.Lenf(t, listInner, 1, "remote %v should have 1 entry", innerFs.Root()) - - err = obj.Remove() - require.NoError(t, err) + require.Len(t, listRoot, 1) + require.Len(t, listRootInner, 1) + require.Len(t, listInner, 1) } func TestInternalObjWrapFsFound(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "tiowff-local", "tiowff-cache", nil) - defer cleanupFs(t, rootFs, boltDb) + id := fmt.Sprintf("tiowff%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) - cfs, err := getCacheFs(rootFs) + cfs, err := runInstance.getCacheFs(rootFs) require.NoError(t, err) wrappedFs := cfs.UnWrap() - data := "content" - writeObjectString(t, wrappedFs, "second", data) - listRoot, err := rootFs.List("") - require.NoError(t, err) - require.Lenf(t, listRoot, 1, "remote %v should have 1 entry", rootFs.Root()) + var testData []byte + if runInstance.rootIsCrypt { + testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64) + require.NoError(t, err) + } else { + testData = []byte("test content") + } - co, err := rootFs.NewObject("second") - require.NoError(t, err) - r, err := co.Open() - require.NoError(t, err) - cachedData, err := ioutil.ReadAll(r) - require.NoError(t, err) - err = r.Close() - require.NoError(t, err) + runInstance.writeObjectBytes(t, wrappedFs, runInstance.encryptRemoteIfNeeded(t, "test"), testData) + listRoot := runInstance.list(t, rootFs, "") + require.Len(t, listRoot, 1) - strCached := string(cachedData) - require.Equal(t, data, strCached) + cachedData := runInstance.readDataFromRemote(t, rootFs, "test", 0, int64(len([]byte("test content"))), false) + require.Equal(t, "test content", string(cachedData)) - err = co.Remove() + err = runInstance.rm(t, rootFs, "test") require.NoError(t, err) - - listRoot, err = wrappedFs.List("") - require.NoError(t, err) - require.Lenf(t, listRoot, 0, "remote %v should have 0 entries: %v", wrappedFs.Root(), listRoot) + listRoot = runInstance.list(t, rootFs, "") + require.Len(t, listRoot, 0) } func TestInternalObjNotFound(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "tionf-local", "tionf-cache", nil) - defer cleanupFs(t, rootFs, boltDb) + id := fmt.Sprintf("tionf%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) obj, err := rootFs.NewObject("404") require.Error(t, err) require.Nil(t, obj) } -func TestInternalCachedWrittenContentMatches(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "ticwcm-local", "ticwcm-cache", nil) - defer cleanupFs(t, rootFs, boltDb) +func TestInternalRemoteWrittenFileFoundInMount(t *testing.T) { + if !runInstance.useMount { + t.Skip("test needs mount mode") + } + id := fmt.Sprintf("tirwffim%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) - cfs, err := getCacheFs(rootFs) + cfs, err := runInstance.getCacheFs(rootFs) + require.NoError(t, err) + + var testData []byte + if runInstance.rootIsCrypt { + testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64) + require.NoError(t, err) + } else { + testData = []byte("test content") + } + + runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test"), testData) + data := runInstance.readDataFromRemote(t, rootFs, "test", 0, int64(len([]byte("test content"))), false) + require.Equal(t, "test content", string(data)) +} + +func TestInternalCachedWrittenContentMatches(t *testing.T) { + id := fmt.Sprintf("ticwcm%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + cfs, err := runInstance.getCacheFs(rootFs) require.NoError(t, err) chunkSize := cfs.ChunkSize() // create some rand test data - testData := make([]byte, (chunkSize*4 + chunkSize/2)) - testSize, err := rand.Read(testData) - require.Equal(t, len(testData), testSize, "data size doesn't match") - require.NoError(t, err) + testData := runInstance.randomBytes(t, chunkSize*4+chunkSize/2) // write the object - o := writeObjectBytes(t, rootFs, "data.bin", testData) - require.Equal(t, o.Size(), int64(testSize)) + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) // check sample of data from in-file sampleStart := chunkSize / 2 sampleEnd := chunkSize testSample := testData[sampleStart:sampleEnd] - checkSample := readDataFromObj(t, o, sampleStart, sampleEnd, false) + checkSample := runInstance.readDataFromRemote(t, rootFs, "data.bin", sampleStart, sampleEnd, false) require.Equal(t, int64(len(checkSample)), sampleEnd-sampleStart) require.Equal(t, checkSample, testSample) } func TestInternalCachedUpdatedContentMatches(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "ticucm-local", "ticucm-cache", nil) - defer cleanupFs(t, rootFs, boltDb) + id := fmt.Sprintf("ticucm%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + var err error // create some rand test data - testData1 := []byte(fstest.RandomString(100)) - testData2 := []byte(fstest.RandomString(200)) + var testData1 []byte + var testData2 []byte + if runInstance.rootIsCrypt { + testData1, err = base64.StdEncoding.DecodeString(cryptedTextBase64) + require.NoError(t, err) + testData2, err = base64.StdEncoding.DecodeString(cryptedText2Base64) + require.NoError(t, err) + } else { + testData1 = []byte(fstest.RandomString(100)) + testData2 = []byte(fstest.RandomString(200)) + } // write the object - o := updateObjectBytes(t, rootFs, "data.bin", testData1, testData2) + o := runInstance.updateObjectRemote(t, rootFs, "data.bin", testData1, testData2) require.Equal(t, o.Size(), int64(len(testData2))) // check data from in-file - reader, err := o.Open() - require.NoError(t, err) - checkSample, err := ioutil.ReadAll(reader) - _ = reader.Close() - require.NoError(t, err) + checkSample := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(len(testData2)), false) require.Equal(t, checkSample, testData2) } func TestInternalWrappedWrittenContentMatches(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "tiwwcm-local", "tiwwcm-cache", nil) - defer cleanupFs(t, rootFs, boltDb) + id := fmt.Sprintf("tiwwcm%v", time.Now().Unix()) + vfsflags.Opt.DirCacheTime = time.Second + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + if runInstance.rootIsCrypt { + t.Skip("test skipped with crypt remote") + } - cfs, err := getCacheFs(rootFs) + cfs, err := runInstance.getCacheFs(rootFs) require.NoError(t, err) chunkSize := cfs.ChunkSize() // create some rand test data - testData := make([]byte, (chunkSize*4 + chunkSize/2)) - testSize, err := rand.Read(testData) - require.Equal(t, len(testData), testSize) - require.NoError(t, err) + testSize := chunkSize*4 + chunkSize/2 + testData := runInstance.randomBytes(t, testSize) // write the object - o := writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData) + o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData) require.Equal(t, o.Size(), int64(testSize)) + time.Sleep(time.Second * 3) - o2, err := rootFs.NewObject("data.bin") - require.NoError(t, err) - require.Equal(t, o2.Size(), o.Size()) + data2 := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false) + require.Equal(t, int64(len(data2)), o.Size()) // check sample of data from in-file sampleStart := chunkSize / 2 sampleEnd := chunkSize testSample := testData[sampleStart:sampleEnd] - checkSample := readDataFromObj(t, o2, sampleStart, sampleEnd, false) + checkSample := runInstance.readDataFromRemote(t, rootFs, "data.bin", sampleStart, sampleEnd, false) require.Equal(t, len(checkSample), len(testSample)) for i := 0; i < len(checkSample); i++ { @@ -194,177 +273,130 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) { } func TestInternalLargeWrittenContentMatches(t *testing.T) { - t.Skip("FIXME disabled because it is unreliable") - rootFs, boltDb := newLocalCacheFs(t, "tilwcm-local", "tilwcm-cache", nil) - defer cleanupFs(t, rootFs, boltDb) - - cfs, err := getCacheFs(rootFs) - require.NoError(t, err) - chunkSize := cfs.ChunkSize() - - // create some rand test data - testData := make([]byte, (chunkSize*10 + chunkSize/2)) - testSize, err := rand.Read(testData) - require.Equal(t, len(testData), testSize) - require.NoError(t, err) - - // write the object - o := writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData) - require.Equal(t, o.Size(), int64(testSize)) - - o2, err := rootFs.NewObject("data.bin") - require.NoError(t, err) - require.Equal(t, o2.Size(), o.Size()) - - // check data from in-file - checkSample := readDataFromObj(t, o2, int64(0), int64(testSize), false) - require.Equal(t, len(checkSample), len(testData)) - - for i := 0; i < len(checkSample); i++ { - require.Equal(t, testData[i], checkSample[i], "byte: %d (%d), chunk: %d", int64(i)%chunkSize, i, int64(i)/chunkSize) + id := fmt.Sprintf("tilwcm%v", time.Now().Unix()) + vfsflags.Opt.DirCacheTime = time.Second + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + if runInstance.rootIsCrypt { + t.Skip("test skipped with crypt remote") } -} -func TestInternalLargeWrittenContentMatches2(t *testing.T) { - cryptFs, boltDb := newLocalCacheCryptFs(t, "tilwcm2-local", "tilwcm2-cache", "tilwcm2-crypt", true, nil) - defer cleanupFs(t, cryptFs, boltDb) - - cfs, err := getCacheFs(cryptFs) + cfs, err := runInstance.getCacheFs(rootFs) require.NoError(t, err) chunkSize := cfs.ChunkSize() - fileSize := 87197196 - readOffset := 87195648 // create some rand test data - testData := make([]byte, fileSize) - testSize, err := rand.Read(testData) - require.Equal(t, len(testData), testSize) - require.NoError(t, err) + testSize := chunkSize*10 + chunkSize/2 + testData := runInstance.randomBytes(t, testSize) // write the object - o := writeObjectBytes(t, cryptFs, "data.bin", testData) - require.Equal(t, o.Size(), int64(testSize)) + runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData) + time.Sleep(time.Second * 3) - o2, err := cryptFs.NewObject("data.bin") - require.NoError(t, err) - require.Equal(t, o2.Size(), o.Size()) - - // check data from in-file - reader, err := o2.Open(&fs.SeekOption{Offset: int64(readOffset)}) - require.NoError(t, err) - rs, ok := reader.(io.Seeker) - require.True(t, ok) - checkOffset, err := rs.Seek(int64(readOffset), 0) - require.NoError(t, err) - require.Equal(t, checkOffset, int64(readOffset)) - checkSample, err := ioutil.ReadAll(reader) - require.NoError(t, err) - _ = reader.Close() - - require.Equal(t, len(checkSample), fileSize-readOffset) - for i := 0; i < fileSize-readOffset; i++ { - require.Equal(t, testData[readOffset+i], checkSample[i], "byte: %d (%d), chunk: %d", int64(i)%chunkSize, i, int64(i)/chunkSize) + readData := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false) + for i := 0; i < len(readData); i++ { + require.Equalf(t, testData[i], readData[i], "at byte %v", i) } } func TestInternalWrappedFsChangeNotSeen(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "tiwfcns-local", "tiwfcns-cache", nil) - defer cleanupFs(t, rootFs, boltDb) + id := fmt.Sprintf("tiwfcns%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) - cfs, err := getCacheFs(rootFs) + cfs, err := runInstance.getCacheFs(rootFs) require.NoError(t, err) chunkSize := cfs.ChunkSize() // create some rand test data - co := writeObjectRandomBytes(t, rootFs, (chunkSize*4 + chunkSize/2)) + testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2)) + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) // update in the wrapped fs - o, err := cfs.UnWrap().NewObject(co.Remote()) + o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin")) require.NoError(t, err) - err = o.SetModTime(co.ModTime().Truncate(time.Hour)) + wrappedTime := time.Now().Add(time.Hour * -1) + err = o.SetModTime(wrappedTime) require.NoError(t, err) // get a new instance from the cache - co2, err := rootFs.NewObject(o.Remote()) + co, err := rootFs.NewObject("data.bin") require.NoError(t, err) - - require.NotEqual(t, o.ModTime().String(), co.ModTime().String()) - require.NotEqual(t, o.ModTime().String(), co2.ModTime().String()) - require.Equal(t, co.ModTime().String(), co2.ModTime().String()) + require.NotEqual(t, co.ModTime().String(), o.ModTime().String()) } func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "ticsadcf-local", "ticsadcf-cache", nil) - defer cleanupFs(t, rootFs, boltDb) + id := fmt.Sprintf("ticsadcf%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) - cfs, err := getCacheFs(rootFs) + cfs, err := runInstance.getCacheFs(rootFs) require.NoError(t, err) chunkSize := cfs.ChunkSize() // create some rand test data - co := writeObjectRandomBytes(t, rootFs, (chunkSize*4 + chunkSize/2)) + testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2)) + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) // update in the wrapped fs - o, err := cfs.UnWrap().NewObject(co.Remote()) + o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin")) require.NoError(t, err) - err = o.SetModTime(co.ModTime().Add(-1 * time.Hour)) + wrappedTime := time.Now().Add(-1 * time.Hour) + err = o.SetModTime(wrappedTime) require.NoError(t, err) // get a new instance from the cache - co2, err := rootFs.NewObject(o.Remote()) + co, err := rootFs.NewObject("data.bin") require.NoError(t, err) - require.NotEqual(t, o.ModTime().String(), co.ModTime().String()) - require.NotEqual(t, o.ModTime().String(), co2.ModTime().String()) - require.Equal(t, co.ModTime().String(), co2.ModTime().String()) cfs.DirCacheFlush() // flush the cache - l, err := cfs.UnWrap().List("") - require.NoError(t, err) - require.Len(t, l, 1) - o2 := l[0] - // get a new instance from the cache - co, err = rootFs.NewObject(o.Remote()) + co, err = rootFs.NewObject("data.bin") require.NoError(t, err) - require.Equal(t, o2.ModTime().String(), co.ModTime().String()) + require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix()) } func TestInternalCacheWrites(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "ticw-local", "ticw-cache", map[string]string{"cache-writes": "true"}) - defer cleanupFs(t, rootFs, boltDb) + id := "ticw" + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"cache-writes": "true"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) - cfs, err := getCacheFs(rootFs) + cfs, err := runInstance.getCacheFs(rootFs) require.NoError(t, err) chunkSize := cfs.ChunkSize() // create some rand test data - co := writeObjectRandomBytes(t, rootFs, (chunkSize*4 + chunkSize/2)) + earliestTime := time.Now() + testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2)) + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) expectedTs := time.Now() - ts, err := boltDb.GetChunkTs(path.Join(rootFs.Root(), co.Remote()), 0) + ts, err := boltDb.GetChunkTs(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "data.bin")), 0) require.NoError(t, err) - require.WithinDuration(t, expectedTs, ts, okDiff) + require.WithinDuration(t, expectedTs, ts, expectedTs.Sub(earliestTime)) } func TestInternalMaxChunkSizeRespected(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "timcsr-local", "timcsr-cache", map[string]string{"cache-workers": "1"}) - defer cleanupFs(t, rootFs, boltDb) + id := fmt.Sprintf("timcsr%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"cache-workers": "1"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) - cfs, err := getCacheFs(rootFs) + cfs, err := runInstance.getCacheFs(rootFs) require.NoError(t, err) chunkSize := cfs.ChunkSize() totalChunks := 20 // create some rand test data - obj := writeObjectRandomBytes(t, cfs, (int64(totalChunks-1)*chunkSize + chunkSize/2)) - o, err := rootFs.NewObject(obj.Remote()) + testData := runInstance.randomBytes(t, (int64(totalChunks-1)*chunkSize + chunkSize/2)) + runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) + o, err := cfs.NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin")) require.NoError(t, err) co, ok := o.(*cache.Object) require.True(t, ok) for i := 0; i < 4; i++ { // read first 4 - _ = readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false) + _ = runInstance.readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false) } cfs.CleanUpCache(true) // the last 2 **must** be in the cache @@ -372,7 +404,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) { require.True(t, boltDb.HasChunk(co, chunkSize*3)) for i := 4; i < 6; i++ { // read next 2 - _ = readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false) + _ = runInstance.readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false) } cfs.CleanUpCache(true) // the last 2 **must** be in the cache @@ -381,36 +413,382 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) { } func TestInternalExpiredEntriesRemoved(t *testing.T) { - rootFs, boltDb := newLocalCacheFs(t, "tieer-local", "tieer-cache", map[string]string{"info_age": "5s"}) - defer cleanupFs(t, rootFs, boltDb) - - cfs, err := getCacheFs(rootFs) + id := fmt.Sprintf("tieer%v", time.Now().Unix()) + vfsflags.Opt.DirCacheTime = time.Second * 4 // needs to be lower than the defined + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, map[string]string{"info_age": "5s"}, nil) + defer runInstance.cleanupFs(t, rootFs, boltDb) + cfs, err := runInstance.getCacheFs(rootFs) require.NoError(t, err) // create some rand test data - _ = writeObjectString(t, cfs, "one", "one content") - err = cfs.Mkdir("test") - require.NoError(t, err) - _ = writeObjectString(t, cfs, "test/second", "second content") + runInstance.writeRemoteString(t, rootFs, "one", "one content") + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/second", "second content") - l, err := cfs.List("test") - require.NoError(t, err) + l := runInstance.list(t, rootFs, "test") require.Len(t, l, 1) - err = cfs.UnWrap().Mkdir("test/test2") + err = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/third")) require.NoError(t, err) - l, err = cfs.List("test") - require.NoError(t, err) + l = runInstance.list(t, rootFs, "test") require.Len(t, l, 1) - waitTime := time.Second * 5 - t.Logf("Waiting %v seconds for cache to expire\n", waitTime) - time.Sleep(waitTime) - - l, err = cfs.List("test") + err = runInstance.retryBlock(func() error { + l = runInstance.list(t, rootFs, "test") + if len(l) != 2 { + return errors.New("list is not 2") + } + return nil + }, 10, time.Second) require.NoError(t, err) - require.Len(t, l, 2) +} + +func TestInternalUploadTempDirCreated(t *testing.T) { + id := fmt.Sprintf("tiutdc%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id)}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + _, err := os.Stat(path.Join(runInstance.tmpUploadDir, id)) + require.NoError(t, err) +} + +func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltDb *cache.Persistent) { + // create some rand test data + testSize := int64(524288000) + testReader := runInstance.randomReader(t, testSize) + bu := runInstance.listenForBackgroundUpload(t, rootFs, "one") + runInstance.writeRemoteReader(t, rootFs, "one", testReader) + // validate that it exists in temp fs + ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one"))) + require.NoError(t, err) + + if runInstance.rootIsCrypt { + require.Equal(t, int64(524416032), ti.Size()) + } else { + require.Equal(t, testSize, ti.Size()) + } + de1 := runInstance.list(t, rootFs, "") + require.Len(t, de1, 1) + + runInstance.completeBackgroundUpload(t, "one", bu) + // check if it was removed from temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one"))) + require.True(t, os.IsNotExist(err)) + + // check if it can be read + data2 := runInstance.readDataFromRemote(t, rootFs, "one", 0, int64(1024), false) + require.Len(t, data2, 1024) +} + +func TestInternalUploadQueueOneFileNoRest(t *testing.T) { + id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "0s"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + testInternalUploadQueueOneFile(t, id, rootFs, boltDb) +} + +func TestInternalUploadQueueOneFileWithRest(t *testing.T) { + id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1m"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + testInternalUploadQueueOneFile(t, id, rootFs, boltDb) +} + +func TestInternalUploadQueueMoreFiles(t *testing.T) { + id := fmt.Sprintf("tiuqmf%v", time.Now().Unix()) + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1s"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + err := rootFs.Mkdir("test") + require.NoError(t, err) + minSize := 5242880 + maxSize := 10485760 + totalFiles := 10 + rand.Seed(time.Now().Unix()) + + lastFile := "" + for i := 0; i < totalFiles; i++ { + size := int64(rand.Intn(maxSize-minSize) + minSize) + testReader := runInstance.randomReader(t, size) + remote := "test/" + strconv.Itoa(i) + ".bin" + runInstance.writeRemoteReader(t, rootFs, remote, testReader) + + // validate that it exists in temp fs + ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, remote))) + require.NoError(t, err) + require.Equal(t, size, runInstance.cleanSize(t, ti.Size())) + + if runInstance.wrappedIsExternal && i < totalFiles-1 { + time.Sleep(time.Second * 3) + } + lastFile = remote + } + + // check if cache lists all files, likely temp upload didn't finish yet + de1 := runInstance.list(t, rootFs, "test") + require.Len(t, de1, totalFiles) + + // wait for background uploader to do its thing + runInstance.completeAllBackgroundUploads(t, rootFs, lastFile) + + // retry until we have no more temp files and fail if they don't go down to 0 + tf, err := ioutil.ReadDir(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test"))) + require.NoError(t, err) + require.Len(t, tf, 0) + + // check if cache lists all files + de1 = runInstance.list(t, rootFs, "test") + require.Len(t, de1, totalFiles) +} + +func TestInternalUploadTempFileOperations(t *testing.T) { + id := "tiutfo" + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + boltDb.PurgeTempUploads() + + // create some rand test data + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // check if it can be read + data1 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false) + require.Equal(t, []byte("one content"), data1) + // validate that it exists in temp fs + _, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + + // test DirMove - allowed + err = runInstance.dirMove(t, rootFs, "test", "second") + if err != errNotSupported { + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.Error(t, err) + _, err = rootFs.NewObject("second/one") + require.NoError(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.Error(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one"))) + require.NoError(t, err) + started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one"))) + require.Error(t, err) + started, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "second/one"))) + require.NoError(t, err) + require.False(t, started) + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + } + + // test Rmdir - allowed + err = runInstance.rm(t, rootFs, "test") + require.Error(t, err) + require.Contains(t, err.Error(), "directory not empty") + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one"))) + require.False(t, started) + require.NoError(t, err) + + // test Move/Rename -- allowed + err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second")) + if err != errNotSupported { + require.NoError(t, err) + // try to read from it + _, err = rootFs.NewObject("test/one") + require.Error(t, err) + _, err = rootFs.NewObject("test/second") + require.NoError(t, err) + data2 := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false) + require.Equal(t, []byte("one content"), data2) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.Error(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second"))) + require.NoError(t, err) + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + } + + // test Copy -- allowed + err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third")) + if err != errNotSupported { + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/third") + require.NoError(t, err) + data2 := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false) + require.Equal(t, []byte("one content"), data2) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third"))) + require.NoError(t, err) + } + + // test Remove -- allowed + err = runInstance.rm(t, rootFs, "test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.Error(t, err) + // validate that it doesn't exist in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.Error(t, err) + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // test Update -- allowed + firstModTime, err := runInstance.modTime(t, rootFs, "test/one") + require.NoError(t, err) + err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated") + require.NoError(t, err) + obj2, err := rootFs.NewObject("test/one") + require.NoError(t, err) + data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false) + require.Equal(t, "one content updated", string(data2)) + tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + if runInstance.rootIsCrypt { + require.Equal(t, int64(67), tmpInfo.Size()) + } else { + require.Equal(t, int64(len(data2)), tmpInfo.Size()) + } + + // test SetModTime -- allowed + secondModTime, err := runInstance.modTime(t, rootFs, "test/one") + require.NoError(t, err) + require.NotEqual(t, secondModTime, firstModTime) + require.NotEqual(t, time.Time{}, firstModTime) + require.NotEqual(t, time.Time{}, secondModTime) +} + +func TestInternalUploadUploadingFileOperations(t *testing.T) { + id := "tiuufo" + rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, + nil, + map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"}) + defer runInstance.cleanupFs(t, rootFs, boltDb) + + boltDb.PurgeTempUploads() + + // create some rand test data + runInstance.mkdir(t, rootFs, "test") + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // check if it can be read + data1 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false) + require.Equal(t, []byte("one content"), data1) + // validate that it exists in temp fs + _, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + + err = boltDb.SetPendingUploadToStarted(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "test/one"))) + require.NoError(t, err) + + // test DirMove + err = runInstance.dirMove(t, rootFs, "test", "second") + if err != errNotSupported { + require.Error(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one"))) + require.Error(t, err) + } + + // test Rmdir + err = runInstance.rm(t, rootFs, "test") + require.Error(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it doesn't exist in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + + // test Move/Rename + err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second")) + if err != errNotSupported { + require.Error(t, err) + // try to read from it + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/second") + require.Error(t, err) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second"))) + require.Error(t, err) + } + + // test Copy -- allowed + err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third")) + if err != errNotSupported { + require.NoError(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + _, err = rootFs.NewObject("test/third") + require.NoError(t, err) + data2 := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false) + require.Equal(t, []byte("one content"), data2) + // validate that it exists in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third"))) + require.NoError(t, err) + } + + // test Remove + err = runInstance.rm(t, rootFs, "test/one") + require.Error(t, err) + _, err = rootFs.NewObject("test/one") + require.NoError(t, err) + // validate that it doesn't exist in temp fs + _, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + require.NoError(t, err) + runInstance.writeRemoteString(t, rootFs, "test/one", "one content") + + // test Update - this seems to work. Why? FIXME + //firstModTime, err := runInstance.modTime(t, rootFs, "test/one") + //require.NoError(t, err) + //err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated", func() { + // data2 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len("one content updated")), true) + // require.Equal(t, "one content", string(data2)) + // + // tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one"))) + // require.NoError(t, err) + // if runInstance.rootIsCrypt { + // require.Equal(t, int64(67), tmpInfo.Size()) + // } else { + // require.Equal(t, int64(len(data2)), tmpInfo.Size()) + // } + //}) + //require.Error(t, err) + + // test SetModTime -- seems to work cause of previous + //secondModTime, err := runInstance.modTime(t, rootFs, "test/one") + //require.NoError(t, err) + //require.Equal(t, secondModTime, firstModTime) + //require.NotEqual(t, time.Time{}, firstModTime) + //require.NotEqual(t, time.Time{}, secondModTime) } // FIXME, enable this when mount is sorted out @@ -420,7 +798,7 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) { // t.Skip("Not yet") // } // id := "tifm1904" -// rootFs, _ := newLocalCacheCryptFs(t, "test-local", "test-cache", "test-crypt", false, +// rootFs, _ := newCacheFs(t, RemoteName, id, false, // map[string]string{"chunk_size": "5M", "info_age": "1m", "chunk_total_size": "500M", "cache-writes": "true"}) // mntPoint := path.Join("/tmp", "tifm1904-mnt") // testPoint := path.Join(mntPoint, id) @@ -472,273 +850,764 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) { // } //} -func writeObjectRandomBytes(t *testing.T, f fs.Fs, size int64) fs.Object { - remote := strconv.Itoa(rand.Int()) + ".bin" - // create some rand test data - testData := make([]byte, size) - testSize, err := rand.Read(testData) - require.Equal(t, size, int64(len(testData))) - require.Equal(t, size, int64(testSize)) - require.NoError(t, err) - - o := writeObjectBytes(t, f, remote, testData) - require.Equal(t, size, o.Size()) - - return o +// run holds the remotes for a test run +type run struct { + okDiff time.Duration + allCfgMap map[string]string + allFlagMap map[string]string + runDefaultCfgMap map[string]string + runDefaultFlagMap map[string]string + mntDir string + tmpUploadDir string + useMount bool + isMounted bool + rootIsCrypt bool + wrappedIsExternal bool + unmountFn func() error + unmountRes chan error + vfs *vfs.VFS + tempFiles []*os.File } -func writeObjectString(t *testing.T, f fs.Fs, remote, content string) fs.Object { - return writeObjectBytes(t, f, remote, []byte(content)) -} - -func writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte) fs.Object { - in := bytes.NewReader(data) - modTime := time.Now() - objInfo := object.NewStaticObjectInfo(remote, modTime, int64(len(data)), true, nil, f) - - obj, err := f.Put(in, objInfo) - require.NoError(t, err) - - return obj -} - -func updateObjectBytes(t *testing.T, f fs.Fs, remote string, data1 []byte, data2 []byte) fs.Object { - in1 := bytes.NewReader(data1) - in2 := bytes.NewReader(data2) - objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f) - objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f) - - obj, err := f.Put(in1, objInfo1) - require.NoError(t, err) - obj, err = f.NewObject(remote) - require.NoError(t, err) - err = obj.Update(in2, objInfo2) - - return obj -} - -func readDataFromObj(t *testing.T, co fs.Object, offset, end int64, useSeek bool) []byte { - var reader io.ReadCloser +func newRun() *run { var err error - size := end - offset - checkSample := make([]byte, size) + r := &run{ + okDiff: time.Second * 9, // really big diff here but the build machines seem to be slow. need a different way for this + useMount: useMount, + isMounted: false, + } - reader, err = co.Open(&fs.SeekOption{Offset: offset}) - require.NoError(t, err) + r.allCfgMap = map[string]string{ + "plex_url": "", + "plex_username": "", + "plex_password": "", + "chunk_size": cache.DefCacheChunkSize, + "info_age": cache.DefCacheInfoAge, + "chunk_total_size": cache.DefCacheTotalChunkSize, + } + r.allFlagMap = map[string]string{ + "cache-db-path": filepath.Join(config.CacheDir, "cache-backend"), + "cache-chunk-path": filepath.Join(config.CacheDir, "cache-backend"), + "cache-db-purge": "true", + "cache-chunk-size": cache.DefCacheChunkSize, + "cache-total-chunk-size": cache.DefCacheTotalChunkSize, + "cache-chunk-clean-interval": cache.DefCacheChunkCleanInterval, + "cache-info-age": cache.DefCacheInfoAge, + "cache-read-retries": strconv.Itoa(cache.DefCacheReadRetries), + "cache-workers": strconv.Itoa(cache.DefCacheTotalWorkers), + "cache-chunk-no-memory": "false", + "cache-rps": strconv.Itoa(cache.DefCacheRps), + "cache-writes": "false", + "cache-tmp-upload-path": "", + "cache-tmp-wait-time": cache.DefCacheTmpWaitTime, + } + r.runDefaultCfgMap = make(map[string]string) + for key, value := range r.allCfgMap { + r.runDefaultCfgMap[key] = value + } + r.runDefaultFlagMap = make(map[string]string) + for key, value := range r.allFlagMap { + r.runDefaultFlagMap[key] = value + } + if mountDir == "" { + if runtime.GOOS != "windows" { + r.mntDir, err = ioutil.TempDir("", "rclonecache-mount") + if err != nil { + log.Fatalf("Failed to create mount dir: %v", err) + return nil + } + } else { + // Find a free drive letter + drive := "" + for letter := 'E'; letter <= 'Z'; letter++ { + drive = string(letter) + ":" + _, err := os.Stat(drive + "\\") + if os.IsNotExist(err) { + goto found + } + } + log.Print("Couldn't find free drive letter for test") + found: + r.mntDir = drive + } + } else { + r.mntDir = mountDir + } + log.Printf("Mount Dir: %v", r.mntDir) - totalRead, err := io.ReadFull(reader, checkSample) - require.NoError(t, err) - _ = reader.Close() - require.Equal(t, int64(totalRead), size, "wrong data read size from file") + if uploadDir == "" { + r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp") + if err != nil { + log.Fatalf("Failed to create temp dir: %v", err) + } + } else { + r.tmpUploadDir = uploadDir + } + log.Printf("Temp Upload Dir: %v", r.tmpUploadDir) - return checkSample + return r } -func cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) { - err := f.Features().Purge() - require.NoError(t, err) - b.Close() +func (r *run) encryptRemoteIfNeeded(t *testing.T, remote string) string { + if !runInstance.rootIsCrypt || len(decryptedToEncryptedRemotes) == 0 { + return remote + } + + enc, ok := decryptedToEncryptedRemotes[remote] + if !ok { + t.Fatalf("Failed to find decrypted -> encrypted mapping for '%v'", remote) + return remote + } + return enc } -func newLocalCacheCryptFs(t *testing.T, localRemote, cacheRemote, cryptRemote string, purge bool, cfg map[string]string) (fs.Fs, *cache.Persistent) { +func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, cfg map[string]string, flags map[string]string) (fs.Fs, *cache.Persistent) { fstest.Initialise() + remoteExists := false + for _, s := range config.FileSections() { + if s == remote { + remoteExists = true + } + } + if !remoteExists && needRemote { + t.Skipf("Need remote (%v) to exist", remote) + return nil, nil + } + + // if the remote doesn't exist, create a new one with a local one for it + // identify which is the cache remote (it can be wrapped by a crypt too) + rootIsCrypt := false + cacheRemote := remote + if !remoteExists { + localRemote := remote + "-local" + config.FileSet(localRemote, "type", "local") + config.FileSet(localRemote, "nounc", "true") + config.FileSet(remote, "type", "cache") + config.FileSet(remote, "remote", localRemote+":/var/tmp/"+localRemote) + } else { + remoteType := fs.ConfigFileGet(remote, "type", "") + if remoteType == "" { + t.Skipf("skipped due to invalid remote type for %v", remote) + return nil, nil + } + if remoteType != "cache" { + if remoteType == "crypt" { + rootIsCrypt = true + config.FileSet(remote, "password", cryptPassword1) + config.FileSet(remote, "password2", cryptPassword2) + } + remoteRemote := fs.ConfigFileGet(remote, "remote", "") + if remoteRemote == "" { + t.Skipf("skipped due to invalid remote wrapper for %v", remote) + return nil, nil + } + remoteRemoteParts := strings.Split(remoteRemote, ":") + remoteWrapping := remoteRemoteParts[0] + remoteType := fs.ConfigFileGet(remoteWrapping, "type", "") + if remoteType != "cache" { + t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType) + return nil, nil + } + cacheRemote = remoteWrapping + } + } + runInstance.rootIsCrypt = rootIsCrypt dbPath := filepath.Join(config.CacheDir, "cache-backend", cacheRemote+".db") chunkPath := filepath.Join(config.CacheDir, "cache-backend", cacheRemote) boltDb, err := cache.GetPersistent(dbPath, chunkPath, &cache.Features{PurgeDb: true}) require.NoError(t, err) - localExists := false - cacheExists := false - cryptExists := false - for _, s := range config.FileSections() { - if s == localRemote { - localExists = true - } - if s == cacheRemote { - cacheExists = true - } - if s == cryptRemote { - cryptExists = true + for k, v := range r.runDefaultCfgMap { + if c, ok := cfg[k]; ok { + config.FileSet(cacheRemote, k, c) + } else { + config.FileSet(cacheRemote, k, v) } } - - localRemoteWrap := "" - if !localExists { - localRemoteWrap = localRemote + ":/var/tmp/" + localRemote - config.FileSet(localRemote, "type", "local") - config.FileSet(localRemote, "nounc", "true") - } - - if !cacheExists { - config.FileSet(cacheRemote, "type", "cache") - config.FileSet(cacheRemote, "remote", localRemoteWrap) - } - if c, ok := cfg["chunk_size"]; ok { - config.FileSet(cacheRemote, "chunk_size", c) - } else { - config.FileSet(cacheRemote, "chunk_size", "1m") - } - if c, ok := cfg["chunk_total_size"]; ok { - config.FileSet(cacheRemote, "chunk_total_size", c) - } else { - config.FileSet(cacheRemote, "chunk_total_size", "2m") - } - if c, ok := cfg["info_age"]; ok { - config.FileSet(cacheRemote, "info_age", c) - } else { - config.FileSet(cacheRemote, "info_age", infoAge.String()) - } - - if !cryptExists { - t.Skipf("Skipping due to missing crypt remote: %v", cryptRemote) - } - - if c, ok := cfg["cache-chunk-no-memory"]; ok { - _ = flag.Set("cache-chunk-no-memory", c) - } else { - _ = flag.Set("cache-chunk-no-memory", "true") - } - if c, ok := cfg["cache-workers"]; ok { - _ = flag.Set("cache-workers", c) - } else { - _ = flag.Set("cache-workers", strconv.Itoa(workers)) - } - if c, ok := cfg["cache-chunk-clean-interval"]; ok { - _ = flag.Set("cache-chunk-clean-interval", c) - } else { - _ = flag.Set("cache-chunk-clean-interval", chunkClean.String()) - } - if c, ok := cfg["cache-writes"]; ok { - _ = flag.Set("cache-writes", c) - } else { - _ = flag.Set("cache-writes", strconv.FormatBool(cache.DefCacheWrites)) + for k, v := range r.runDefaultFlagMap { + if c, ok := flags[k]; ok { + _ = flag.Set(k, c) + } else { + _ = flag.Set(k, v) + } } + fs.Config.LowLevelRetries = 1 // Instantiate root - f, err := fs.NewFs(cryptRemote + ":") + if purge { + boltDb.PurgeTempUploads() + _ = os.RemoveAll(path.Join(runInstance.tmpUploadDir, id)) + } + f, err := fs.NewFs(remote + ":" + id) require.NoError(t, err) + cfs, err := r.getCacheFs(f) + require.NoError(t, err) + _, isCache := cfs.Features().UnWrap().(*cache.Fs) + _, isCrypt := cfs.Features().UnWrap().(*crypt.Fs) + _, isLocal := cfs.Features().UnWrap().(*local.Fs) + if isCache || isCrypt || isLocal { + r.wrappedIsExternal = true + } else { + r.wrappedIsExternal = true + } + if purge { _ = f.Features().Purge() require.NoError(t, err) } err = f.Mkdir("") require.NoError(t, err) + if r.useMount && !r.isMounted { + r.mountFs(t, f) + } return f, boltDb } -func newLocalCacheFs(t *testing.T, localRemote, cacheRemote string, cfg map[string]string) (fs.Fs, *cache.Persistent) { - fstest.Initialise() - dbPath := filepath.Join(config.CacheDir, "cache-backend", cacheRemote+".db") - chunkPath := filepath.Join(config.CacheDir, "cache-backend", cacheRemote) - boltDb, err := cache.GetPersistent(dbPath, chunkPath, &cache.Features{PurgeDb: true}) +func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) { + if r.useMount && r.isMounted { + r.unmountFs(t, f) + } + + err := f.Features().Purge() + require.NoError(t, err) + cfs, err := r.getCacheFs(f) + require.NoError(t, err) + cfs.StopBackgroundRunners() + + if r.useMount && runtime.GOOS != "windows" { + err = os.RemoveAll(r.mntDir) + require.NoError(t, err) + } + err = os.RemoveAll(r.tmpUploadDir) require.NoError(t, err) - localExists := false - cacheExists := false - for _, s := range config.FileSections() { - if s == localRemote { - localExists = true - } - if s == cacheRemote { - cacheExists = true - } + for _, f := range r.tempFiles { + _ = f.Close() + _ = os.Remove(f.Name()) } - - localRemoteWrap := "" - if !localExists { - localRemoteWrap = localRemote + ":/var/tmp/" + localRemote - config.FileSet(localRemote, "type", "local") - config.FileSet(localRemote, "nounc", "true") + r.tempFiles = nil + debug.FreeOSMemory() + for k, v := range r.runDefaultFlagMap { + _ = flag.Set(k, v) } - - if !cacheExists { - config.FileSet(cacheRemote, "type", "cache") - config.FileSet(cacheRemote, "remote", localRemoteWrap) - } - if c, ok := cfg["chunk_size"]; ok { - config.FileSet(cacheRemote, "chunk_size", c) - } else { - config.FileSet(cacheRemote, "chunk_size", "1m") - } - if c, ok := cfg["chunk_total_size"]; ok { - config.FileSet(cacheRemote, "chunk_total_size", c) - } else { - config.FileSet(cacheRemote, "chunk_total_size", "2m") - } - if c, ok := cfg["info_age"]; ok { - config.FileSet(cacheRemote, "info_age", c) - } else { - config.FileSet(cacheRemote, "info_age", infoAge.String()) - } - - if c, ok := cfg["cache-chunk-no-memory"]; ok { - _ = flag.Set("cache-chunk-no-memory", c) - } else { - _ = flag.Set("cache-chunk-no-memory", "true") - } - if c, ok := cfg["cache-workers"]; ok { - _ = flag.Set("cache-workers", c) - } else { - _ = flag.Set("cache-workers", strconv.Itoa(workers)) - } - if c, ok := cfg["cache-chunk-clean-interval"]; ok { - _ = flag.Set("cache-chunk-clean-interval", c) - } else { - _ = flag.Set("cache-chunk-clean-interval", chunkClean.String()) - } - if c, ok := cfg["cache-writes"]; ok { - _ = flag.Set("cache-writes", c) - } else { - _ = flag.Set("cache-writes", strconv.FormatBool(cache.DefCacheWrites)) - } - - // Instantiate root - f, err := fs.NewFs(cacheRemote + ":") - require.NoError(t, err) - _ = f.Features().Purge() - require.NoError(t, err) - err = f.Mkdir("") - require.NoError(t, err) - - return f, boltDb } -//func mountFs(t *testing.T, f fs.Fs, mntPoint string) { -// if runtime.GOOS == "windows" { -// t.Skip("Skipping test cause on windows") -// return -// } -// -// _ = flag.Set("debug-fuse", "false") -// -// go func() { -// mountlib.DebugFUSE = false -// mountlib.AllowOther = true -// mount.Mount(f, mntPoint) -// }() -// -// time.Sleep(time.Second * 3) -//} +func (r *run) randomBytes(t *testing.T, size int64) []byte { + testData := make([]byte, size) + testSize, err := rand.Read(testData) + require.Equal(t, size, int64(len(testData))) + require.Equal(t, size, int64(testSize)) + require.NoError(t, err) + return testData +} -func unmountFs(t *testing.T, mntPoint string) { - var out []byte +func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser { + chunk := int64(1024) + cnt := size / chunk + left := size % chunk + f, err := ioutil.TempFile("", "rclonecache-tempfile") + require.NoError(t, err) + + for i := 0; i < int(cnt); i++ { + data := r.randomBytes(t, chunk) + _, _ = f.Write(data) + } + data := r.randomBytes(t, int64(left)) + _, _ = f.Write(data) + _, _ = f.Seek(int64(0), 0) + r.tempFiles = append(r.tempFiles, f) + + return f +} + +func (r *run) writeRemoteRandomBytes(t *testing.T, f fs.Fs, p string, size int64) string { + remote := path.Join(p, strconv.Itoa(rand.Int())+".bin") + // create some rand test data + testData := r.randomBytes(t, size) + + r.writeRemoteBytes(t, f, remote, testData) + return remote +} + +func (r *run) writeObjectRandomBytes(t *testing.T, f fs.Fs, p string, size int64) fs.Object { + remote := path.Join(p, strconv.Itoa(rand.Int())+".bin") + // create some rand test data + testData := r.randomBytes(t, size) + + return r.writeObjectBytes(t, f, remote, testData) +} + +func (r *run) writeRemoteString(t *testing.T, f fs.Fs, remote, content string) { + r.writeRemoteBytes(t, f, remote, []byte(content)) +} + +func (r *run) writeObjectString(t *testing.T, f fs.Fs, remote, content string) fs.Object { + return r.writeObjectBytes(t, f, remote, []byte(content)) +} + +func (r *run) writeRemoteBytes(t *testing.T, f fs.Fs, remote string, data []byte) { var err error - if runtime.GOOS == "windows" { - t.Skip("Skipping test cause on windows") - return - } else if runtime.GOOS == "linux" { - out, err = exec.Command("fusermount", "-u", mntPoint).Output() - } else if runtime.GOOS == "darwin" { - out, err = exec.Command("diskutil", "unmount", mntPoint).Output() + if r.useMount { + err = r.retryBlock(func() error { + return ioutil.WriteFile(path.Join(r.mntDir, remote), data, 0600) + }, 3, time.Second*3) + require.NoError(t, err) + r.vfs.WaitForWriters(10 * time.Second) + } else { + r.writeObjectBytes(t, f, remote, data) + } +} + +func (r *run) writeRemoteReader(t *testing.T, f fs.Fs, remote string, in io.ReadCloser) { + defer func() { + _ = in.Close() + }() + + if r.useMount { + out, err := os.Create(path.Join(r.mntDir, remote)) + require.NoError(t, err) + defer func() { + _ = out.Close() + }() + + _, err = io.Copy(out, in) + require.NoError(t, err) + r.vfs.WaitForWriters(10 * time.Second) + } else { + r.writeObjectReader(t, f, remote, in) + } +} + +func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte) fs.Object { + in := bytes.NewReader(data) + _ = r.writeObjectReader(t, f, remote, in) + o, err := f.NewObject(remote) + require.NoError(t, err) + require.Equal(t, int64(len(data)), o.Size()) + return o +} + +func (r *run) writeObjectReader(t *testing.T, f fs.Fs, remote string, in io.Reader) fs.Object { + modTime := time.Now() + objInfo := object.NewStaticObjectInfo(remote, modTime, -1, true, nil, f) + obj, err := f.Put(in, objInfo) + require.NoError(t, err) + if r.useMount { + r.vfs.WaitForWriters(10 * time.Second) } - t.Logf("Unmount output: %v", string(out)) + return obj +} + +func (r *run) updateObjectRemote(t *testing.T, f fs.Fs, remote string, data1 []byte, data2 []byte) fs.Object { + var err error + var obj fs.Object + + if r.useMount { + err = ioutil.WriteFile(path.Join(r.mntDir, remote), data1, 0600) + require.NoError(t, err) + r.vfs.WaitForWriters(10 * time.Second) + err = ioutil.WriteFile(path.Join(r.mntDir, remote), data2, 0600) + require.NoError(t, err) + r.vfs.WaitForWriters(10 * time.Second) + obj, err = f.NewObject(remote) + } else { + in1 := bytes.NewReader(data1) + in2 := bytes.NewReader(data2) + objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f) + objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f) + + obj, err = f.Put(in1, objInfo1) + require.NoError(t, err) + obj, err = f.NewObject(remote) + require.NoError(t, err) + err = obj.Update(in2, objInfo2) + } + require.NoError(t, err) + + return obj +} + +func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, end int64, noLengthCheck bool) []byte { + size := end - offset + checkSample := make([]byte, size) + + if r.useMount { + f, err := os.Open(path.Join(r.mntDir, remote)) + defer func() { + _ = f.Close() + }() + require.NoError(t, err) + _, _ = f.Seek(offset, 0) + totalRead, err := io.ReadFull(f, checkSample) + checkSample = checkSample[:totalRead] + if err == io.EOF || err == io.ErrUnexpectedEOF { + err = nil + } + require.NoError(t, err) + if !noLengthCheck { + require.Equal(t, size, int64(totalRead)) + } + require.NoError(t, err) + } else { + co, err := f.NewObject(remote) + require.NoError(t, err) + checkSample = r.readDataFromObj(t, co, offset, end, noLengthCheck) + } + if !noLengthCheck { + require.Equal(t, size, int64(len(checkSample)), "wrong data read size from file") + } + return checkSample +} + +func (r *run) readDataFromObj(t *testing.T, o fs.Object, offset, end int64, noLengthCheck bool) []byte { + size := end - offset + checkSample := make([]byte, size) + reader, err := o.Open(&fs.SeekOption{Offset: offset}) + require.NoError(t, err) + totalRead, err := io.ReadFull(reader, checkSample) + if (err == io.EOF || err == io.ErrUnexpectedEOF) && noLengthCheck { + err = nil + checkSample = checkSample[:totalRead] + } + require.NoError(t, err) + _ = reader.Close() + return checkSample +} + +func (r *run) mkdir(t *testing.T, f fs.Fs, remote string) { + var err error + if r.useMount { + err = os.Mkdir(path.Join(r.mntDir, remote), 0700) + } else { + err = f.Mkdir(remote) + } require.NoError(t, err) } -func getCacheFs(f fs.Fs) (*cache.Fs, error) { +func (r *run) rm(t *testing.T, f fs.Fs, remote string) error { + var err error + + if r.useMount { + err = os.Remove(path.Join(r.mntDir, remote)) + } else { + var obj fs.Object + obj, err = f.NewObject(remote) + if err != nil { + err = f.Rmdir(remote) + } else { + err = obj.Remove() + } + } + + return err +} + +func (r *run) list(t *testing.T, f fs.Fs, remote string) []interface{} { + var err error + var l []interface{} + if r.useMount { + var list []os.FileInfo + list, err = ioutil.ReadDir(path.Join(r.mntDir, remote)) + for _, ll := range list { + l = append(l, ll) + } + } else { + var list fs.DirEntries + list, err = f.List(remote) + for _, ll := range list { + l = append(l, ll) + } + } + require.NoError(t, err) + return l +} + +func (r *run) listPath(t *testing.T, f fs.Fs, remote string) []string { + var err error + var l []string + if r.useMount { + var list []os.FileInfo + list, err = ioutil.ReadDir(path.Join(r.mntDir, remote)) + for _, ll := range list { + l = append(l, ll.Name()) + } + } else { + var list fs.DirEntries + list, err = f.List(remote) + for _, ll := range list { + l = append(l, ll.Remote()) + } + } + require.NoError(t, err) + return l +} + +func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { + _ = in.Close() + }() + + out, err := os.Create(dst) + if err != nil { + return err + } + defer func() { + _ = out.Close() + }() + + _, err = io.Copy(out, in) + return err +} + +func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error { + var err error + + if runInstance.useMount { + err = os.Rename(path.Join(runInstance.mntDir, src), path.Join(runInstance.mntDir, dst)) + if err != nil { + return err + } + r.vfs.WaitForWriters(10 * time.Second) + } else if rootFs.Features().DirMove != nil { + err = rootFs.Features().DirMove(rootFs, src, dst) + if err != nil { + return err + } + } else { + t.Logf("DirMove not supported by %v", rootFs) + return errNotSupported + } + + return err +} + +func (r *run) move(t *testing.T, rootFs fs.Fs, src, dst string) error { + var err error + + if runInstance.useMount { + err = os.Rename(path.Join(runInstance.mntDir, src), path.Join(runInstance.mntDir, dst)) + if err != nil { + return err + } + r.vfs.WaitForWriters(10 * time.Second) + } else if rootFs.Features().Move != nil { + obj1, err := rootFs.NewObject(src) + if err != nil { + return err + } + _, err = rootFs.Features().Move(obj1, dst) + if err != nil { + return err + } + } else { + t.Logf("Move not supported by %v", rootFs) + return errNotSupported + } + + return err +} + +func (r *run) copy(t *testing.T, rootFs fs.Fs, src, dst string) error { + var err error + + if r.useMount { + err = r.copyFile(t, rootFs, path.Join(r.mntDir, src), path.Join(r.mntDir, dst)) + if err != nil { + return err + } + r.vfs.WaitForWriters(10 * time.Second) + } else if rootFs.Features().Copy != nil { + obj, err := rootFs.NewObject(src) + if err != nil { + return err + } + _, err = rootFs.Features().Copy(obj, dst) + if err != nil { + return err + } + } else { + t.Logf("Copy not supported by %v", rootFs) + return errNotSupported + } + + return err +} + +func (r *run) modTime(t *testing.T, rootFs fs.Fs, src string) (time.Time, error) { + var err error + + if r.useMount { + fi, err := os.Stat(path.Join(runInstance.mntDir, src)) + if err != nil { + return time.Time{}, err + } + return fi.ModTime(), nil + } + obj1, err := rootFs.NewObject(src) + if err != nil { + return time.Time{}, err + } + return obj1.ModTime(), nil +} + +func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) error { + var err error + + if r.useMount { + f, err := os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + _, err = f.WriteString(data + append) + if err != nil { + _ = f.Close() + return err + } + err = f.Close() + if err != nil { + return err + } + r.vfs.WaitForWriters(10 * time.Second) + } else { + obj1, err := rootFs.NewObject(src) + if err != nil { + return err + } + data1 := []byte(data + append) + r := bytes.NewReader(data1) + objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs) + err = obj1.Update(r, objInfo1) + if err != nil { + return err + } + } + + return err +} + +func (r *run) cleanSize(t *testing.T, size int64) int64 { + if r.rootIsCrypt { + denominator := int64(65536 + 16) + size = size - 32 + quotient := size / denominator + remainder := size % denominator + return (quotient*65536 + remainder - 16) + } + + return size +} + +func (r *run) listenForBackgroundUpload(t *testing.T, f fs.Fs, remote string) chan error { + cfs, err := r.getCacheFs(f) + require.NoError(t, err) + buCh := cfs.GetBackgroundUploadChannel() + require.NotNil(t, buCh) + maxDuration := time.Minute * 3 + if r.wrappedIsExternal { + maxDuration = time.Minute * 10 + } + + waitCh := make(chan error) + go func() { + var err error + var state cache.BackgroundUploadState + + for i := 0; i < 2; i++ { + select { + case state = <-buCh: + // continue + case <-time.After(maxDuration): + waitCh <- errors.Errorf("Timed out waiting for background upload: %v", remote) + return + } + checkRemote := state.Remote + if r.rootIsCrypt { + cryptFs := f.(*crypt.Fs) + checkRemote, err = cryptFs.DecryptFileName(checkRemote) + if err != nil { + waitCh <- err + return + } + } + if checkRemote == remote && cache.BackgroundUploadStarted != state.Status { + waitCh <- state.Error + return + } + } + waitCh <- errors.Errorf("Too many attempts to wait for the background upload: %v", remote) + }() + return waitCh +} + +func (r *run) completeBackgroundUpload(t *testing.T, remote string, waitCh chan error) { + var err error + maxDuration := time.Minute * 3 + if r.wrappedIsExternal { + maxDuration = time.Minute * 10 + } + select { + case err = <-waitCh: + // continue + case <-time.After(maxDuration): + t.Fatalf("Timed out waiting to complete the background upload %v", remote) + return + } + require.NoError(t, err) +} + +func (r *run) completeAllBackgroundUploads(t *testing.T, f fs.Fs, lastRemote string) { + var state cache.BackgroundUploadState + var err error + + maxDuration := time.Minute * 5 + if r.wrappedIsExternal { + maxDuration = time.Minute * 15 + } + cfs, err := r.getCacheFs(f) + require.NoError(t, err) + buCh := cfs.GetBackgroundUploadChannel() + require.NotNil(t, buCh) + + for { + select { + case state = <-buCh: + checkRemote := state.Remote + if r.rootIsCrypt { + cryptFs := f.(*crypt.Fs) + checkRemote, err = cryptFs.DecryptFileName(checkRemote) + require.NoError(t, err) + } + if checkRemote == lastRemote && cache.BackgroundUploadCompleted == state.Status { + require.NoError(t, state.Error) + return + } + case <-time.After(maxDuration): + t.Fatalf("Timed out waiting to complete the background upload %v", lastRemote) + return + } + } +} + +func (r *run) retryBlock(block func() error, maxRetries int, rate time.Duration) error { + var err error + for i := 0; i < maxRetries; i++ { + err = block() + if err == nil { + return nil + } + time.Sleep(rate) + } + return err +} + +func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) { cfs, ok := f.(*cache.Fs) if ok { return cfs, nil @@ -751,7 +1620,7 @@ func getCacheFs(f fs.Fs) (*cache.Fs, error) { } } - return nil, fmt.Errorf("didn't found a cache fs") + return nil, errors.New("didn't found a cache fs") } var ( diff --git a/backend/cache/cache_mount_unix_test.go b/backend/cache/cache_mount_unix_test.go new file mode 100644 index 000000000..dfc478d52 --- /dev/null +++ b/backend/cache/cache_mount_unix_test.go @@ -0,0 +1,78 @@ +// +build !plan9,!windows,go1.7 + +package cache_test + +import ( + "os" + "testing" + "time" + + "bazil.org/fuse" + fusefs "bazil.org/fuse/fs" + "github.com/ncw/rclone/cmd/mount" + "github.com/ncw/rclone/cmd/mountlib" + "github.com/ncw/rclone/fs" + "github.com/stretchr/testify/require" +) + +func (r *run) mountFs(t *testing.T, f fs.Fs) { + device := f.Name() + ":" + f.Root() + var options = []fuse.MountOption{ + fuse.MaxReadahead(uint32(mountlib.MaxReadAhead)), + fuse.Subtype("rclone"), + fuse.FSName(device), fuse.VolumeName(device), + fuse.NoAppleDouble(), + fuse.NoAppleXattr(), + fuse.AllowOther(), + } + err := os.MkdirAll(r.mntDir, os.ModePerm) + require.NoError(t, err) + c, err := fuse.Mount(r.mntDir, options...) + require.NoError(t, err) + filesys := mount.NewFS(f) + server := fusefs.New(c, nil) + + // Serve the mount point in the background returning error to errChan + r.unmountRes = make(chan error, 1) + go func() { + err := server.Serve(filesys) + closeErr := c.Close() + if err == nil { + err = closeErr + } + r.unmountRes <- err + }() + + // check if the mount process has an error to report + <-c.Ready + require.NoError(t, c.MountError) + + r.unmountFn = func() error { + // Shutdown the VFS + filesys.VFS.Shutdown() + return fuse.Unmount(r.mntDir) + } + + r.vfs = filesys.VFS + r.isMounted = true +} + +func (r *run) unmountFs(t *testing.T, f fs.Fs) { + var err error + + for i := 0; i < 4; i++ { + err = r.unmountFn() + if err != nil { + //log.Printf("signal to umount failed - retrying: %v", err) + time.Sleep(3 * time.Second) + continue + } + break + } + require.NoError(t, err) + err = <-r.unmountRes + require.NoError(t, err) + err = r.vfs.CleanUp() + require.NoError(t, err) + r.isMounted = false +} diff --git a/backend/cache/cache_mount_windows_test.go b/backend/cache/cache_mount_windows_test.go new file mode 100644 index 000000000..b0f16e49e --- /dev/null +++ b/backend/cache/cache_mount_windows_test.go @@ -0,0 +1,124 @@ +// +build windows,go1.7 + +package cache_test + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/billziss-gh/cgofuse/fuse" + "github.com/ncw/rclone/cmd/cmount" + "github.com/ncw/rclone/cmd/mountlib" + "github.com/ncw/rclone/fs" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +// waitFor runs fn() until it returns true or the timeout expires +func waitFor(fn func() bool) (ok bool) { + const totalWait = 10 * time.Second + const individualWait = 10 * time.Millisecond + for i := 0; i < int(totalWait/individualWait); i++ { + ok = fn() + if ok { + return ok + } + time.Sleep(individualWait) + } + return false +} + +func (r *run) mountFs(t *testing.T, f fs.Fs) { + // FIXME implement cmount + t.Skip("windows not supported yet") + + device := f.Name() + ":" + f.Root() + options := []string{ + "-o", "fsname=" + device, + "-o", "subtype=rclone", + "-o", fmt.Sprintf("max_readahead=%d", mountlib.MaxReadAhead), + "-o", "uid=-1", + "-o", "gid=-1", + "-o", "allow_other", + // This causes FUSE to supply O_TRUNC with the Open + // call which is more efficient for cmount. However + // it does not work with cgofuse on Windows with + // WinFSP so cmount must work with or without it. + "-o", "atomic_o_trunc", + "--FileSystemName=rclone", + } + + fsys := cmount.NewFS(f) + host := fuse.NewFileSystemHost(fsys) + + // Serve the mount point in the background returning error to errChan + r.unmountRes = make(chan error, 1) + go func() { + var err error + ok := host.Mount(r.mntDir, options) + if !ok { + err = errors.New("mount failed") + } + r.unmountRes <- err + }() + + // unmount + r.unmountFn = func() error { + // Shutdown the VFS + fsys.VFS.Shutdown() + if host.Unmount() { + if !waitFor(func() bool { + _, err := os.Stat(r.mntDir) + return err != nil + }) { + t.Fatalf("mountpoint %q didn't disappear after unmount - continuing anyway", r.mntDir) + } + return nil + } + return errors.New("host unmount failed") + } + + // Wait for the filesystem to become ready, checking the file + // system didn't blow up before starting + select { + case err := <-r.unmountRes: + require.NoError(t, err) + case <-time.After(time.Second * 3): + } + + // Wait for the mount point to be available on Windows + // On Windows the Init signal comes slightly before the mount is ready + if !waitFor(func() bool { + _, err := os.Stat(r.mntDir) + return err == nil + }) { + t.Errorf("mountpoint %q didn't became available on mount", r.mntDir) + } + + r.vfs = fsys.VFS + r.isMounted = true +} + +func (r *run) unmountFs(t *testing.T, f fs.Fs) { + // FIXME implement cmount + t.Skip("windows not supported yet") + var err error + + for i := 0; i < 4; i++ { + err = r.unmountFn() + if err != nil { + //log.Printf("signal to umount failed - retrying: %v", err) + time.Sleep(3 * time.Second) + continue + } + break + } + require.NoError(t, err) + err = <-r.unmountRes + require.NoError(t, err) + err = r.vfs.CleanUp() + require.NoError(t, err) + r.isMounted = false +} diff --git a/backend/cache/directory.go b/backend/cache/directory.go index 3457aa2ad..b429ae7a7 100644 --- a/backend/cache/directory.go +++ b/backend/cache/directory.go @@ -93,15 +93,7 @@ func (d *Directory) String() string { // Remote returns the remote path func (d *Directory) Remote() string { - p := cleanPath(path.Join(d.Dir, d.Name)) - if d.CacheFs.Root() != "" { - p = p[len(d.CacheFs.Root()):] // trim out root - if len(p) > 0 { // remove first separator - p = p[1:] - } - } - - return p + return d.CacheFs.cleanRootFromPath(d.abs()) } // abs returns the absolute path to the dir diff --git a/backend/cache/handle.go b/backend/cache/handle.go index f3200aa64..3fc84d070 100644 --- a/backend/cache/handle.go +++ b/backend/cache/handle.go @@ -9,14 +9,42 @@ import ( "sync" "time" + "path" + "runtime" + "strings" + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/operations" "github.com/pkg/errors" ) +var uploaderMap = make(map[string]*backgroundWriter) +var uploaderMapMx sync.Mutex + +// initBackgroundUploader returns a single instance +func initBackgroundUploader(fs *Fs) (*backgroundWriter, error) { + // write lock to create one + uploaderMapMx.Lock() + defer uploaderMapMx.Unlock() + if b, ok := uploaderMap[fs.String()]; ok { + // if it was already started we close it so that it can be started again + if b.running { + b.close() + } else { + return b, nil + } + } + + bb := newBackgroundWriter(fs) + uploaderMap[fs.String()] = bb + return uploaderMap[fs.String()], nil +} + // Handle is managing the read/write/seek operations on an open handle type Handle struct { cachedObject *Object - memory ChunkStorage + cfs *Fs + memory *Memory preloadQueue chan int64 preloadOffset int64 offset int64 @@ -31,20 +59,21 @@ type Handle struct { } // NewObjectHandle returns a new Handle for an existing Object -func NewObjectHandle(o *Object) *Handle { +func NewObjectHandle(o *Object, cfs *Fs) *Handle { r := &Handle{ cachedObject: o, + cfs: cfs, offset: 0, preloadOffset: -1, // -1 to trigger the first preload - UseMemory: o.CacheFs.chunkMemory, + UseMemory: cfs.chunkMemory, reading: false, } r.seenOffsets = make(map[int64]bool) r.memory = NewMemory(-1) // create a larger buffer to queue up requests - r.preloadQueue = make(chan int64, o.CacheFs.totalWorkers*10) + r.preloadQueue = make(chan int64, r.cfs.totalWorkers*10) r.confirmReading = make(chan bool) r.startReadWorkers() return r @@ -52,11 +81,11 @@ func NewObjectHandle(o *Object) *Handle { // cacheFs is a convenience method to get the parent cache FS of the object's manager func (r *Handle) cacheFs() *Fs { - return r.cachedObject.CacheFs + return r.cfs } // storage is a convenience method to get the persistent storage of the object's manager -func (r *Handle) storage() Storage { +func (r *Handle) storage() *Persistent { return r.cacheFs().cache } @@ -76,7 +105,7 @@ func (r *Handle) startReadWorkers() { if !r.cacheFs().plexConnector.isConnected() { err := r.cacheFs().plexConnector.authenticate() if err != nil { - fs.Infof(r, "failed to authenticate to Plex: %v", err) + fs.Errorf(r, "failed to authenticate to Plex: %v", err) } } if r.cacheFs().plexConnector.isConnected() { @@ -113,7 +142,7 @@ func (r *Handle) scaleWorkers(desired int) { } // ignore first scale out from 0 if current != 0 { - fs.Infof(r, "scale workers to %v", desired) + fs.Debugf(r, "scale workers to %v", desired) } } @@ -156,7 +185,6 @@ func (r *Handle) queueOffset(offset int64) { if r.UseMemory { go r.memory.CleanChunksByNeed(offset) } - go r.cacheFs().CleanUpCache(false) r.confirmExternalReading() r.preloadOffset = offset @@ -305,7 +333,6 @@ func (r *Handle) Close() error { } } - go r.cacheFs().CleanUpCache(false) fs.Debugf(r, "cache reader closed %v", r.offset) return nil } @@ -357,11 +384,11 @@ func (w *worker) String() string { // - if it supports seeking it will seek to the desired offset and return the same reader // - if it doesn't support seeking it will close a possible existing one and open at the desired offset // - if there's no reader associated with this worker, it will create one -func (w *worker) reader(offset, end int64) (io.ReadCloser, error) { +func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error) { var err error r := w.rc if w.rc == nil { - r, err = w.r.cacheFs().OpenRateLimited(func() (io.ReadCloser, error) { + r, err = w.r.cacheFs().openRateLimited(func() (io.ReadCloser, error) { return w.r.cachedObject.Object.Open(&fs.SeekOption{Offset: offset}, &fs.RangeOption{Start: offset, End: end}) }) if err != nil { @@ -370,14 +397,16 @@ func (w *worker) reader(offset, end int64) (io.ReadCloser, error) { return r, nil } - seekerObj, ok := r.(io.Seeker) - if ok { - _, err = seekerObj.Seek(offset, os.SEEK_SET) - return r, err + if !closeOpen { + seekerObj, ok := r.(io.Seeker) + if ok { + _, err = seekerObj.Seek(offset, os.SEEK_SET) + return r, err + } } _ = w.rc.Close() - return w.r.cacheFs().OpenRateLimited(func() (io.ReadCloser, error) { + return w.r.cacheFs().openRateLimited(func() (io.ReadCloser, error) { r, err = w.r.cachedObject.Object.Open(&fs.SeekOption{Offset: offset}, &fs.RangeOption{Start: offset, End: end}) if err != nil { return nil, err @@ -463,10 +492,18 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) { time.Sleep(time.Second * time.Duration(retry)) } - w.rc, err = w.reader(chunkStart, chunkEnd) + closeOpen := false + if retry > 0 { + closeOpen = true + } + w.rc, err = w.reader(chunkStart, chunkEnd, closeOpen) // we seem to be getting only errors so we abort if err != nil { fs.Errorf(w, "object open failed %v: %v", chunkStart, err) + err = w.r.cachedObject.refreshFromSource(true) + if err != nil { + fs.Errorf(w, "%v", err) + } w.download(chunkStart, chunkEnd, retry+1) return } @@ -476,6 +513,10 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) { sourceRead, err = io.ReadFull(w.rc, data) if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { fs.Errorf(w, "failed to read chunk %v: %v", chunkStart, err) + err = w.r.cachedObject.refreshFromSource(true) + if err != nil { + fs.Errorf(w, "%v", err) + } w.download(chunkStart, chunkEnd, retry+1) return } @@ -483,7 +524,7 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) { if err == io.ErrUnexpectedEOF { fs.Debugf(w, "partial downloaded chunk %v", fs.SizeSuffix(chunkStart)) } else { - fs.Debugf(w, "downloaded chunk %v", fs.SizeSuffix(chunkStart)) + fs.Debugf(w, "downloaded chunk %v", chunkStart) } if w.r.UseMemory { @@ -499,6 +540,115 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) { } } +const ( + // BackgroundUploadStarted is a state for a temp file that has started upload + BackgroundUploadStarted = iota + // BackgroundUploadCompleted is a state for a temp file that has completed upload + BackgroundUploadCompleted + // BackgroundUploadError is a state for a temp file that has an error upload + BackgroundUploadError +) + +// BackgroundUploadState is an entity that maps to an existing file which is stored on the temp fs +type BackgroundUploadState struct { + Remote string + Status int + Error error +} + +type backgroundWriter struct { + fs *Fs + stateCh chan int + running bool + notifyCh chan BackgroundUploadState +} + +func newBackgroundWriter(f *Fs) *backgroundWriter { + b := &backgroundWriter{ + fs: f, + stateCh: make(chan int), + notifyCh: make(chan BackgroundUploadState), + } + + return b +} + +func (b *backgroundWriter) close() { + b.stateCh <- 2 +} + +func (b *backgroundWriter) pause() { + b.stateCh <- 1 +} + +func (b *backgroundWriter) play() { + b.stateCh <- 0 +} + +func (b *backgroundWriter) notify(remote string, status int, err error) { + state := BackgroundUploadState{ + Remote: remote, + Status: status, + Error: err, + } + select { + case b.notifyCh <- state: + fs.Debugf(remote, "notified background upload state: %v", state.Status) + default: + } +} + +func (b *backgroundWriter) run() { + state := 0 + for { + b.running = true + select { + case s := <-b.stateCh: + state = s + default: + // + } + switch state { + case 1: + runtime.Gosched() + time.Sleep(time.Millisecond * 500) + continue + case 2: + b.running = false + return + } + + absPath, err := b.fs.cache.getPendingUpload(b.fs.Root(), b.fs.tempWriteWait) + if err != nil || absPath == "" || !b.fs.isRootInPath(absPath) { + time.Sleep(time.Second) + continue + } + + remote := b.fs.cleanRootFromPath(absPath) + b.notify(remote, BackgroundUploadStarted, nil) + fs.Infof(remote, "background upload: started upload") + err = operations.MoveFile(b.fs.UnWrap(), b.fs.tempFs, remote, remote) + if err != nil { + b.notify(remote, BackgroundUploadError, err) + _ = b.fs.cache.rollbackPendingUpload(absPath) + fs.Errorf(remote, "background upload: %v", err) + continue + } + fs.Infof(remote, "background upload: uploaded entry") + err = b.fs.cache.removePendingUpload(absPath) + if err != nil && !strings.Contains(err.Error(), "pending upload not found") { + fs.Errorf(remote, "background upload: %v", err) + } + parentCd := NewDirectory(b.fs, cleanPath(path.Dir(remote))) + err = b.fs.cache.ExpireDir(parentCd) + if err != nil { + fs.Errorf(parentCd, "background upload: cache expire error: %v", err) + } + fs.Infof(remote, "finished background upload") + b.notify(remote, BackgroundUploadCompleted, nil) + } +} + // Check the interfaces are satisfied var ( _ io.ReadCloser = (*Handle)(nil) diff --git a/backend/cache/object.go b/backend/cache/object.go index c17b458fb..68c0becec 100644 --- a/backend/cache/object.go +++ b/backend/cache/object.go @@ -3,24 +3,28 @@ package cache import ( - "encoding/json" "io" "os" "path" "sync" "time" - "strconv" - "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs/hash" "github.com/ncw/rclone/lib/readers" + "github.com/pkg/errors" +) + +const ( + objectInCache = "Object" + objectPendingUpload = "TempObject" ) // Object is a generic file like object that stores basic information about it type Object struct { fs.Object `json:"-"` + ParentFs fs.Fs `json:"-"` // parent fs CacheFs *Fs `json:"-"` // cache fs Name string `json:"name"` // name of the directory Dir string `json:"dir"` // abs path of the object @@ -29,79 +33,64 @@ type Object struct { CacheStorable bool `json:"storable"` // says whether this object can be stored CacheType string `json:"cacheType"` CacheTs time.Time `json:"cacheTs"` - cacheHashes map[hash.Type]string // all supported hashes cached + CacheHashes map[hash.Type]string // all supported hashes cached refreshMutex sync.Mutex } // NewObject builds one from a generic fs.Object -func NewObject(f *Fs, remote string) *Object { //0745 379 768 +func NewObject(f *Fs, remote string) *Object { fullRemote := path.Join(f.Root(), remote) dir, name := path.Split(fullRemote) + cacheType := objectInCache + parentFs := f.UnWrap() + if f.tempWritePath != "" { + _, err := f.cache.SearchPendingUpload(fullRemote) + if err == nil { // queued for upload + cacheType = objectPendingUpload + parentFs = f.tempFs + fs.Debugf(fullRemote, "pending upload found") + } + } + co := &Object{ + ParentFs: parentFs, CacheFs: f, Name: cleanPath(name), Dir: cleanPath(dir), CacheModTime: time.Now().UnixNano(), CacheSize: 0, CacheStorable: false, - CacheType: "Object", + CacheType: cacheType, CacheTs: time.Now(), } return co } -// MarshalJSON is needed to override the hashes map (needed to support older versions of Go) -func (o *Object) MarshalJSON() ([]byte, error) { - hashes := make(map[string]string) - for k, v := range o.cacheHashes { - hashes[strconv.Itoa(int(k))] = v - } - - type Alias Object - return json.Marshal(&struct { - Hashes map[string]string `json:"hashes"` - *Alias - }{ - Alias: (*Alias)(o), - Hashes: hashes, - }) -} - -// UnmarshalJSON is needed to override the CacheHashes map (needed to support older versions of Go) -func (o *Object) UnmarshalJSON(b []byte) error { - type Alias Object - aux := &struct { - Hashes map[string]string `json:"hashes"` - *Alias - }{ - Alias: (*Alias)(o), - } - if err := json.Unmarshal(b, &aux); err != nil { - return err - } - - o.cacheHashes = make(map[hash.Type]string) - for k, v := range aux.Hashes { - ht, _ := strconv.Atoi(k) - o.cacheHashes[hash.Type(ht)] = v - } - - return nil -} - // ObjectFromOriginal builds one from a generic fs.Object func ObjectFromOriginal(f *Fs, o fs.Object) *Object { var co *Object fullRemote := cleanPath(path.Join(f.Root(), o.Remote())) - dir, name := path.Split(fullRemote) + + cacheType := objectInCache + parentFs := f.UnWrap() + if f.tempWritePath != "" { + _, err := f.cache.SearchPendingUpload(fullRemote) + if err == nil { // queued for upload + cacheType = objectPendingUpload + parentFs = f.tempFs + fs.Debugf(fullRemote, "pending upload found") + } + } + co = &Object{ + ParentFs: parentFs, CacheFs: f, Name: cleanPath(name), Dir: cleanPath(dir), - CacheType: "Object", + CacheType: cacheType, CacheTs: time.Now(), } co.updateData(o) @@ -114,7 +103,7 @@ func (o *Object) updateData(source fs.Object) { o.CacheSize = source.Size() o.CacheStorable = source.Storable() o.CacheTs = time.Now() - o.cacheHashes = make(map[hash.Type]string) + o.CacheHashes = make(map[hash.Type]string) } // Fs returns its FS info @@ -133,14 +122,7 @@ func (o *Object) String() string { // Remote returns the remote path func (o *Object) Remote() string { p := path.Join(o.Dir, o.Name) - if o.CacheFs.Root() != "" { - p = p[len(o.CacheFs.Root()):] // trim out root - if len(p) > 0 { // remove first separator - p = p[1:] - } - } - - return p + return o.CacheFs.cleanRootFromPath(p) } // abs returns the absolute path to the object @@ -148,17 +130,6 @@ func (o *Object) abs() string { return path.Join(o.Dir, o.Name) } -// parentRemote returns the absolute path parent remote -func (o *Object) parentRemote() string { - absPath := o.abs() - return cleanPath(path.Dir(absPath)) -} - -// parentDir returns the absolute path parent remote -func (o *Object) parentDir() *Directory { - return NewDirectory(o.CacheFs, cleanPath(path.Dir(o.Remote()))) -} - // ModTime returns the cached ModTime func (o *Object) ModTime() time.Time { return time.Unix(0, o.CacheModTime) @@ -175,17 +146,24 @@ func (o *Object) Storable() bool { } // refreshFromSource requests the original FS for the object in case it comes from a cached entry -func (o *Object) refreshFromSource() error { +func (o *Object) refreshFromSource(force bool) error { o.refreshMutex.Lock() defer o.refreshMutex.Unlock() + var err error + var liveObject fs.Object - if o.Object != nil { + if o.Object != nil && !force { return nil } - - liveObject, err := o.CacheFs.Fs.NewObject(o.Remote()) + if o.isTempFile() { + liveObject, err = o.ParentFs.NewObject(o.Remote()) + err = errors.Wrapf(err, "in parent fs %v", o.ParentFs) + } else { + liveObject, err = o.CacheFs.Fs.NewObject(o.Remote()) + err = errors.Wrapf(err, "in cache fs %v", o.CacheFs.Fs) + } if err != nil { - fs.Errorf(o, "error refreshing object: %v", err) + fs.Errorf(o, "error refreshing object in : %v", err) return err } o.updateData(liveObject) @@ -196,7 +174,7 @@ func (o *Object) refreshFromSource() error { // SetModTime sets the ModTime of this object func (o *Object) SetModTime(t time.Time) error { - if err := o.refreshFromSource(); err != nil { + if err := o.refreshFromSource(false); err != nil { return err } @@ -207,19 +185,19 @@ func (o *Object) SetModTime(t time.Time) error { o.CacheModTime = t.UnixNano() o.persist() - fs.Debugf(o.Fs(), "updated ModTime %v: %v", o, t) + fs.Debugf(o, "updated ModTime: %v", t) return nil } // Open is used to request a specific part of the file using fs.RangeOption func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) { - if err := o.refreshFromSource(); err != nil { + if err := o.refreshFromSource(true); err != nil { return nil, err } var err error - cacheReader := NewObjectHandle(o) + cacheReader := NewObjectHandle(o, o.CacheFs) var offset, limit int64 = 0, -1 for _, option := range options { switch x := option.(type) { @@ -239,23 +217,34 @@ func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) { // Update will change the object data func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { - if err := o.refreshFromSource(); err != nil { + if err := o.refreshFromSource(false); err != nil { return err } - fs.Infof(o, "updating object contents with size %v", src.Size()) - - // deleting cached chunks and info to be replaced with new ones - _ = o.CacheFs.cache.RemoveObject(o.abs()) + // pause background uploads if active + if o.CacheFs.tempWritePath != "" { + o.CacheFs.backgroundRunner.pause() + defer o.CacheFs.backgroundRunner.play() + // don't allow started uploads + if o.isTempFile() && o.tempFileStartedUpload() { + return errors.Errorf("%v is currently uploading, can't update", o) + } + } + fs.Debugf(o, "updating object contents with size %v", src.Size()) + // FIXME use reliable upload err := o.Object.Update(in, src, options...) if err != nil { fs.Errorf(o, "error updating source: %v", err) return err } + // deleting cached chunks and info to be replaced with new ones + _ = o.CacheFs.cache.RemoveObject(o.abs()) + o.CacheModTime = src.ModTime().UnixNano() o.CacheSize = src.Size() - o.cacheHashes = make(map[hash.Type]string) + o.CacheHashes = make(map[hash.Type]string) + o.CacheTs = time.Now() o.persist() return nil @@ -263,41 +252,50 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio // Remove deletes the object from both the cache and the source func (o *Object) Remove() error { - if err := o.refreshFromSource(); err != nil { + if err := o.refreshFromSource(false); err != nil { return err } + // pause background uploads if active + if o.CacheFs.tempWritePath != "" { + o.CacheFs.backgroundRunner.pause() + defer o.CacheFs.backgroundRunner.play() + // don't allow started uploads + if o.isTempFile() && o.tempFileStartedUpload() { + return errors.Errorf("%v is currently uploading, can't delete", o) + } + } err := o.Object.Remove() if err != nil { return err } - fs.Infof(o, "removing object") + fs.Debugf(o, "removing object") _ = o.CacheFs.cache.RemoveObject(o.abs()) - return err + _ = o.CacheFs.cache.removePendingUpload(o.abs()) + _ = o.CacheFs.cache.ExpireDir(NewDirectory(o.CacheFs, cleanPath(path.Dir(o.Remote())))) + + return nil } // Hash requests a hash of the object and stores in the cache // since it might or might not be called, this is lazy loaded func (o *Object) Hash(ht hash.Type) (string, error) { - if o.cacheHashes == nil { - o.cacheHashes = make(map[hash.Type]string) + if o.CacheHashes == nil { + o.CacheHashes = make(map[hash.Type]string) } - cachedHash, found := o.cacheHashes[ht] + cachedHash, found := o.CacheHashes[ht] if found { return cachedHash, nil } - - if err := o.refreshFromSource(); err != nil { + if err := o.refreshFromSource(false); err != nil { return "", err } - liveHash, err := o.Object.Hash(ht) if err != nil { return "", err } - - o.cacheHashes[ht] = liveHash + o.CacheHashes[ht] = liveHash o.persist() fs.Debugf(o, "object hash cached: %v", liveHash) @@ -314,6 +312,25 @@ func (o *Object) persist() *Object { return o } +func (o *Object) isTempFile() bool { + _, err := o.CacheFs.cache.SearchPendingUpload(o.abs()) + if err != nil { + o.CacheType = objectInCache + return false + } + + o.CacheType = objectPendingUpload + return true +} + +func (o *Object) tempFileStartedUpload() bool { + started, err := o.CacheFs.cache.SearchPendingUpload(o.abs()) + if err != nil { + return false + } + return started +} + var ( _ fs.Object = (*Object)(nil) ) diff --git a/backend/cache/plex.go b/backend/cache/plex.go index 5b2b7f095..0bc40901a 100644 --- a/backend/cache/plex.go +++ b/backend/cache/plex.go @@ -12,6 +12,9 @@ import ( "sync" + "bytes" + "io/ioutil" + "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs/config" ) @@ -127,6 +130,17 @@ func (p *plexConnector) isConfigured() bool { } func (p *plexConnector) isPlaying(co *Object) bool { + var err error + + remote := co.Remote() + if cr, yes := p.f.isWrappedByCrypt(); yes { + remote, err = cr.DecryptFileName(co.Remote()) + if err != nil { + fs.Errorf("plex", "can not decrypt wrapped file: %v", err) + return false + } + } + isPlaying := false req, err := http.NewRequest("GET", fmt.Sprintf("%s/status/sessions", p.url.String()), nil) if err != nil { @@ -180,31 +194,12 @@ func (p *plexConnector) isPlaying(co *Object) bool { if err != nil { return false } - var data map[string]interface{} - err = json.NewDecoder(resp.Body).Decode(&data) + var data []byte + data, err = ioutil.ReadAll(resp.Body) if err != nil { return false } - - remote := co.Remote() - if cr, yes := co.CacheFs.isWrappedByCrypt(); yes { - remote, err = cr.DecryptFileName(co.Remote()) - if err != nil { - fs.Errorf("plex", "can not decrypt wrapped file: %v", err) - continue - } - } - fpGen, ok := get(data, "MediaContainer", "Metadata", 0, "Media", 0, "Part", 0, "file") - if !ok { - fs.Errorf("plex", "failed to understand: %v", data) - continue - } - fp, ok := fpGen.(string) - if !ok { - fs.Errorf("plex", "failed to understand: %v", fp) - continue - } - if strings.Contains(fp, remote) { + if bytes.Contains(data, []byte(remote)) { isPlaying = true break } diff --git a/backend/cache/storage_memory.go b/backend/cache/storage_memory.go index ddb7825f3..f66b405e3 100644 --- a/backend/cache/storage_memory.go +++ b/backend/cache/storage_memory.go @@ -14,8 +14,6 @@ import ( // Memory is a wrapper of transient storage for a go-cache store type Memory struct { - ChunkStorage - db *cache.Cache } diff --git a/backend/cache/storage_persistent.go b/backend/cache/storage_persistent.go index b997c813e..bbc77c2a4 100644 --- a/backend/cache/storage_persistent.go +++ b/backend/cache/storage_persistent.go @@ -16,8 +16,11 @@ import ( "io/ioutil" + "fmt" + bolt "github.com/coreos/bbolt" "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/walk" "github.com/pkg/errors" ) @@ -26,6 +29,7 @@ const ( RootBucket = "root" RootTsBucket = "rootTs" DataTsBucket = "dataTs" + tempBucket = "pending" ) // Features flags for this storage type @@ -42,6 +46,12 @@ func GetPersistent(dbPath, chunkPath string, f *Features) (*Persistent, error) { boltMapMx.Lock() defer boltMapMx.Unlock() if b, ok := boltMap[dbPath]; ok { + if !b.open { + err := b.connect() + if err != nil { + return nil, err + } + } return b, nil } @@ -59,15 +69,26 @@ type chunkInfo struct { Size int64 } +type tempUploadInfo struct { + DestPath string + AddedOn time.Time + Started bool +} + +// String representation of a tempUploadInfo +func (t *tempUploadInfo) String() string { + return fmt.Sprintf("%v - %v (%v)", t.DestPath, t.Started, t.AddedOn) +} + // Persistent is a wrapper of persistent storage for a bolt.DB file type Persistent struct { - Storage - - dbPath string - dataPath string - db *bolt.DB - cleanupMux sync.Mutex - features *Features + dbPath string + dataPath string + open bool + db *bolt.DB + cleanupMux sync.Mutex + tempQueueMux sync.Mutex + features *Features } // newPersistent builds a new wrapper and connects to the bolt.DB file @@ -78,7 +99,7 @@ func newPersistent(dbPath, chunkPath string, f *Features) (*Persistent, error) { features: f, } - err := b.Connect() + err := b.connect() if err != nil { fs.Errorf(dbPath, "Error opening storage cache. Is there another rclone running on the same remote? %v", err) return nil, err @@ -92,41 +113,32 @@ func (b *Persistent) String() string { return " " + b.dbPath } -// Connect creates a connection to the configured file +// connect creates a connection to the configured file // refreshDb will delete the file before to create an empty DB if it's set to true -func (b *Persistent) Connect() error { - var db *bolt.DB +func (b *Persistent) connect() error { var err error - if b.features.PurgeDb { - err := os.Remove(b.dbPath) - if err != nil { - fs.Errorf(b, "failed to remove cache file: %v", err) - } - err = os.RemoveAll(b.dataPath) - if err != nil { - fs.Errorf(b, "failed to remove cache data: %v", err) - } - } - err = os.MkdirAll(b.dataPath, os.ModePerm) if err != nil { return errors.Wrapf(err, "failed to create a data directory %q", b.dataPath) } - db, err = bolt.Open(b.dbPath, 0644, &bolt.Options{Timeout: 1 * time.Second}) + b.db, err = bolt.Open(b.dbPath, 0644, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { return errors.Wrapf(err, "failed to open a cache connection to %q", b.dbPath) } - - _ = db.Update(func(tx *bolt.Tx) error { + if b.features.PurgeDb { + b.Purge() + } + _ = b.db.Update(func(tx *bolt.Tx) error { _, _ = tx.CreateBucketIfNotExists([]byte(RootBucket)) _, _ = tx.CreateBucketIfNotExists([]byte(RootTsBucket)) _, _ = tx.CreateBucketIfNotExists([]byte(DataTsBucket)) + _, _ = tx.CreateBucketIfNotExists([]byte(tempBucket)) return nil }) - b.db = db + b.open = true return nil } @@ -136,7 +148,9 @@ func (b *Persistent) getBucket(dir string, createIfMissing bool, tx *bolt.Tx) *b cleanPath(dir) entries := strings.FieldsFunc(dir, func(c rune) bool { - return os.PathSeparator == c + // cover Windows where rclone still uses '/' as path separator + // this should be safe as '/' is not a valid Windows character + return (os.PathSeparator == c || c == rune('/')) }) bucket := tx.Bucket([]byte(RootBucket)) @@ -478,6 +492,7 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) { b.cleanupMux.Lock() defer b.cleanupMux.Unlock() var cntChunks int + var roughlyCleaned fs.SizeSuffix err := b.db.Update(func(tx *bolt.Tx) error { dataTsBucket := tx.Bucket([]byte(DataTsBucket)) @@ -499,6 +514,7 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) { if totalSize > maxSize { needToClean := totalSize - maxSize + roughlyCleaned = fs.SizeSuffix(needToClean) for k, v := c.First(); k != nil; k, v = c.Next() { var ci chunkInfo err := json.Unmarshal(v, &ci) @@ -521,7 +537,10 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) { } } } - fs.Infof("cache", "deleted (%v) chunks", cntChunks) + if cntChunks > 0 { + fs.Infof("cache-cleanup", "chunks %v, est. size: %v", cntChunks, roughlyCleaned.String()) + + } return nil }) @@ -691,6 +710,313 @@ func (b *Persistent) iterateBuckets(buk *bolt.Bucket, bucketFn func(name string) return err } +func (b *Persistent) dumpRoot() string { + var itBuckets func(buk *bolt.Bucket) map[string]interface{} + + itBuckets = func(buk *bolt.Bucket) map[string]interface{} { + m := make(map[string]interface{}) + c := buk.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + if v == nil { + buk2 := buk.Bucket(k) + m[string(k)] = itBuckets(buk2) + } else { + m[string(k)] = "-" + } + } + return m + } + var mm map[string]interface{} + _ = b.db.View(func(tx *bolt.Tx) error { + mm = itBuckets(tx.Bucket([]byte(RootBucket))) + return nil + }) + raw, _ := json.MarshalIndent(mm, "", " ") + return string(raw) +} + +// addPendingUpload adds a new file to the pending queue of uploads +func (b *Persistent) addPendingUpload(destPath string, started bool) error { + return b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + tempObj := &tempUploadInfo{ + DestPath: destPath, + AddedOn: time.Now(), + Started: started, + } + + // cache Object Info + encoded, err := json.Marshal(tempObj) + if err != nil { + return errors.Errorf("couldn't marshal object (%v) info: %v", destPath, err) + } + err = bucket.Put([]byte(destPath), []byte(encoded)) + if err != nil { + return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err) + } + + return nil + }) +} + +// getPendingUpload returns the next file from the pending queue of uploads +func (b *Persistent) getPendingUpload(inRoot string, waitTime time.Duration) (destPath string, err error) { + b.tempQueueMux.Lock() + defer b.tempQueueMux.Unlock() + + err = b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + + c := bucket.Cursor() + for k, v := c.Seek([]byte(inRoot)); k != nil && bytes.HasPrefix(k, []byte(inRoot)); k, v = c.Next() { + //for k, v := c.First(); k != nil; k, v = c.Next() { + var tempObj = &tempUploadInfo{} + err = json.Unmarshal(v, tempObj) + if err != nil { + fs.Errorf(b, "failed to read pending upload: %v", err) + continue + } + // skip over started uploads + if tempObj.Started || time.Now().Before(tempObj.AddedOn.Add(waitTime)) { + continue + } + + tempObj.Started = true + v2, err := json.Marshal(tempObj) + if err != nil { + fs.Errorf(b, "failed to update pending upload: %v", err) + continue + } + err = bucket.Put(k, v2) + if err != nil { + fs.Errorf(b, "failed to update pending upload: %v", err) + continue + } + + destPath = tempObj.DestPath + return nil + } + + return errors.Errorf("no pending upload found") + }) + + return destPath, err +} + +// SearchPendingUpload returns the file info from the pending queue of uploads +func (b *Persistent) SearchPendingUpload(remote string) (started bool, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(tempBucket)) + if bucket == nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + + var tempObj = &tempUploadInfo{} + v := bucket.Get([]byte(remote)) + err = json.Unmarshal(v, tempObj) + if err != nil { + return errors.Errorf("pending upload (%v) not found %v", remote, err) + } + + started = tempObj.Started + return nil + }) + + return started, err +} + +// searchPendingUploadFromDir files currently pending upload from a single dir +func (b *Persistent) searchPendingUploadFromDir(dir string) (remotes []string, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(tempBucket)) + if bucket == nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + + c := bucket.Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + var tempObj = &tempUploadInfo{} + err = json.Unmarshal(v, tempObj) + if err != nil { + fs.Errorf(b, "failed to read pending upload: %v", err) + continue + } + parentDir := cleanPath(path.Dir(tempObj.DestPath)) + if dir == parentDir { + remotes = append(remotes, tempObj.DestPath) + } + } + + return nil + }) + + return remotes, err +} + +func (b *Persistent) rollbackPendingUpload(remote string) error { + b.tempQueueMux.Lock() + defer b.tempQueueMux.Unlock() + + return b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + var tempObj = &tempUploadInfo{} + v := bucket.Get([]byte(remote)) + err = json.Unmarshal(v, tempObj) + if err != nil { + return errors.Errorf("pending upload (%v) not found %v", remote, err) + } + tempObj.Started = false + v2, err := json.Marshal(tempObj) + if err != nil { + return errors.Errorf("pending upload not updated %v", err) + } + err = bucket.Put([]byte(tempObj.DestPath), v2) + if err != nil { + return errors.Errorf("pending upload not updated %v", err) + } + return nil + }) +} + +func (b *Persistent) removePendingUpload(remote string) error { + b.tempQueueMux.Lock() + defer b.tempQueueMux.Unlock() + + return b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + return bucket.Delete([]byte(remote)) + }) +} + +// updatePendingUpload allows to update an existing item in the queue while checking if it's not started in the same +// transaction. If it is started, it will not allow the update +func (b *Persistent) updatePendingUpload(remote string, fn func(item *tempUploadInfo) error) error { + b.tempQueueMux.Lock() + defer b.tempQueueMux.Unlock() + + return b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return errors.Errorf("couldn't bucket for %v", tempBucket) + } + + var tempObj = &tempUploadInfo{} + v := bucket.Get([]byte(remote)) + err = json.Unmarshal(v, tempObj) + if err != nil { + return errors.Errorf("pending upload (%v) not found %v", remote, err) + } + if tempObj.Started { + return errors.Errorf("pending upload already started %v", remote) + } + err = fn(tempObj) + if err != nil { + return err + } + if remote != tempObj.DestPath { + err := bucket.Delete([]byte(remote)) + if err != nil { + return err + } + // if this is removed then the entry can be removed too + if tempObj.DestPath == "" { + return nil + } + } + v2, err := json.Marshal(tempObj) + if err != nil { + return errors.Errorf("pending upload not updated %v", err) + } + err = bucket.Put([]byte(tempObj.DestPath), v2) + if err != nil { + return errors.Errorf("pending upload not updated %v", err) + } + + return nil + }) +} + +// SetPendingUploadToStarted is a way to mark an entry as started (even if it's not already) +// TO BE USED IN TESTING ONLY +func (b *Persistent) SetPendingUploadToStarted(remote string) error { + return b.updatePendingUpload(remote, func(item *tempUploadInfo) error { + item.Started = true + return nil + }) +} + +// ReconcileTempUploads will recursively look for all the files in the temp directory and add them to the queue +func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error { + return b.db.Update(func(tx *bolt.Tx) error { + _ = tx.DeleteBucket([]byte(tempBucket)) + bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket)) + if err != nil { + return err + } + + var queuedEntries []fs.Object + err = walk.Walk(cacheFs.tempFs, "", true, -1, func(path string, entries fs.DirEntries, err error) error { + for _, o := range entries { + if oo, ok := o.(fs.Object); ok { + queuedEntries = append(queuedEntries, oo) + } + } + return nil + }) + if err != nil { + return err + } + + fs.Debugf(cacheFs, "reconciling temporary uploads") + for _, queuedEntry := range queuedEntries { + destPath := path.Join(cacheFs.Root(), queuedEntry.Remote()) + tempObj := &tempUploadInfo{ + DestPath: destPath, + AddedOn: time.Now(), + Started: false, + } + + // cache Object Info + encoded, err := json.Marshal(tempObj) + if err != nil { + return errors.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err) + } + err = bucket.Put([]byte(destPath), []byte(encoded)) + if err != nil { + return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err) + } + fs.Debugf(cacheFs, "reconciled temporary upload: %v", destPath) + } + + return nil + }) +} + +// PurgeTempUploads will remove all the pending uploads from the queue +// TO BE USED IN TESTING ONLY +func (b *Persistent) PurgeTempUploads() { + b.tempQueueMux.Lock() + defer b.tempQueueMux.Unlock() + + _ = b.db.Update(func(tx *bolt.Tx) error { + _ = tx.DeleteBucket([]byte(tempBucket)) + _, _ = tx.CreateBucketIfNotExists([]byte(tempBucket)) + return nil + }) +} + // Close should be called when the program ends gracefully func (b *Persistent) Close() { b.cleanupMux.Lock() @@ -700,6 +1026,7 @@ func (b *Persistent) Close() { if err != nil { fs.Errorf(b, "closing handle: %v", err) } + b.open = false } // itob returns an 8-byte big endian representation of v. diff --git a/docs/content/cache.md b/docs/content/cache.md index 3501e26c9..49cfc27a2 100644 --- a/docs/content/cache.md +++ b/docs/content/cache.md @@ -109,12 +109,38 @@ To start a cached mount rclone mount --allow-other test-cache: /var/tmp/test-cache +### Write Features ### + +### Offline uploading ### + +In an effort to make writing through cache more reliable, the backend +now supports this feature which can be activated by specifying a +`cache-tmp-upload-path`. + +A files goes through these states when using this feature: + +1. An upload is started (usually by copying a file on the cache remote) +2. When the copy to the temporary location is complete the file is part +of the cached remote and looks and behaves like any other file (reading included) +3. After `cache-tmp-wait-time` passes and the file is next in line, `rclone move` +is used to move the file to the cloud provider +4. Reading the file still works during the upload but most modifications on it will be prohibited +5. Once the move is complete the file is unlocked for modifications as it +becomes as any other regular file +6. If the file is being read through `cache` when it's actually +deleted from the temporary path then `cache` will simply swap the source +to the cloud provider without interrupting the reading (small blip can happen though) + +Files are uploaded in sequence and only one file is uploaded at a time. +Uploads will be stored in a queue and be processed based on the order they were added. +The queue and the temporary storage is persistent across restarts and even purges of the cache. + ### Write Support ### Writes are supported through `cache`. One caveat is that a mounted cache remote does not add any retry or fallback mechanism to the upload operation. This will depend on the implementation -of the wrapped remote. +of the wrapped remote. Consider using `Offline uploading` for reliable writes. One special case is covered with `cache-writes` which will cache the file data at the same time as the upload when it is enabled making it available @@ -157,6 +183,16 @@ Affected settings: ### Known issues ### +#### Mount and --dir-cache-time #### + +--dir-cache-time controls the first layer of directory caching which works at the mount layer. +Being an independent caching mechanism from the `cache` backend, it will manage its own entries +based on the configured time. + +To avoid getting in a scenario where dir cache has obsolete data and cache would have the correct +one, try to set `--dir-cache-time` to a lower time than `--cache-info-age`. Default values are +already configured in this way. + #### Windows support - Experimental #### There are a couple of issues with Windows `mount` functionality that still require some investigations. @@ -341,3 +377,23 @@ you can enable this flag to have their data stored in the cache store at the same time during upload. **Default**: not set + +#### --cache-tmp-upload-path=PATH #### + +This is the path where `cache` will use as a temporary storage for new files +that need to be uploaded to the cloud provider. + +Specifying a value will enable this feature. Without it, it is completely disabled +and files will be uploaded directly to the cloud provider + +**Default**: empty + +#### --cache-tmp-wait-time=DURATION #### + +This is the duration that a file must wait in the temporary location +_cache-tmp-upload-path_ before it is selected for upload. + +Note that only one file is uploaded at a time and it can take longer to +start the upload if a queue formed for this purpose. + +**Default**: 15m