b2: cleanup unfinished large files

The `cleanup` command will delete unfinished large file uploads that
were started more than a day ago (to avoid deleting uploads that are
potentially still in progress).

Fixes #2617
This commit is contained in:
Garry McNulty 2018-12-02 18:05:32 +00:00 committed by Nick Craig-Wood
parent 107293c80e
commit 887834da91
2 changed files with 11 additions and 2 deletions

View File

@ -980,6 +980,12 @@ func (f *Fs) purge(oldOnly bool) error {
errReturn = err errReturn = err
} }
} }
var isUnfinishedUploadStale = func(timestamp api.Timestamp) bool {
if time.Since(time.Time(timestamp)).Hours() > 24 {
return true
}
return false
}
// Delete Config.Transfers in parallel // Delete Config.Transfers in parallel
toBeDeleted := make(chan *api.File, fs.Config.Transfers) toBeDeleted := make(chan *api.File, fs.Config.Transfers)
@ -1003,6 +1009,9 @@ func (f *Fs) purge(oldOnly bool) error {
if object.Action == "hide" { if object.Action == "hide" {
fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID) fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
toBeDeleted <- object toBeDeleted <- object
} else if object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())
toBeDeleted <- object
} else { } else {
fs.Debugf(remote, "Not deleting current version (id %q) %q", object.ID, object.Action) fs.Debugf(remote, "Not deleting current version (id %q) %q", object.ID, object.Action)
} }

View File

@ -181,8 +181,8 @@ versions of files, leaving the current ones intact. You can also
supply a path and only old versions under that path will be deleted, supply a path and only old versions under that path will be deleted,
eg `rclone cleanup remote:bucket/path/to/stuff`. eg `rclone cleanup remote:bucket/path/to/stuff`.
Note that `cleanup` does not remove partially uploaded files Note that `cleanup` will remove partially uploaded files from the bucket
from the bucket. if they are more than a day old.
When you `purge` a bucket, the current and the old versions will be When you `purge` a bucket, the current and the old versions will be
deleted then the bucket will be deleted. deleted then the bucket will be deleted.