From ab8c0a81fa0840f501f35954b26d746c8eaf0fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 22 Jan 2018 19:53:18 +0100 Subject: [PATCH] Add a delete threshold to sync (--max-delete) Fixes #959 --- fs/accounting/stats.go | 10 ++++++++++ fs/config.go | 2 ++ fs/config/configflags/configflags.go | 1 + fs/config/flags/flags.go | 8 ++++++++ fs/operations/operations.go | 17 ++++++++++++++++- 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go index 7decbb0dd..cd2557247 100644 --- a/fs/accounting/stats.go +++ b/fs/accounting/stats.go @@ -30,6 +30,7 @@ type StatsInfo struct { checking stringSet transfers int64 transferring stringSet + deletes int64 start time.Time inProgress *inProgress } @@ -115,6 +116,14 @@ func (s *StatsInfo) GetLastError() error { return s.lastError } +// Deletes updates the stats for deletes +func (s *StatsInfo) Deletes(deletes int64) int64 { + s.lock.Lock() + defer s.lock.Unlock() + s.deletes += deletes + return s.deletes +} + // ResetCounters sets the counters (bytes, checks, errors, transfers) to 0 func (s *StatsInfo) ResetCounters() { s.lock.RLock() @@ -123,6 +132,7 @@ func (s *StatsInfo) ResetCounters() { s.errors = 0 s.checks = 0 s.transfers = 0 + s.deletes = 0 } // ResetErrors sets the errors count to 0 diff --git a/fs/config.go b/fs/config.go index 48aff8651..a7c0e6f08 100644 --- a/fs/config.go +++ b/fs/config.go @@ -41,6 +41,7 @@ type ConfigInfo struct { Dump DumpFlags InsecureSkipVerify bool // Skip server certificate verification DeleteMode DeleteMode + MaxDelete int64 TrackRenames bool // Track file renames. LowLevelRetries int UpdateOlder bool // Skip files that are newer on the destination @@ -82,6 +83,7 @@ func NewConfig() *ConfigInfo { c.ConnectTimeout = 60 * time.Second c.Timeout = 5 * 60 * time.Second c.DeleteMode = DeleteModeDefault + c.MaxDelete = -1 c.LowLevelRetries = 10 c.MaxDepth = -1 c.DataRateUnit = "bytes" diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 5378602ee..c93232eb3 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -53,6 +53,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &deleteBefore, "delete-before", "", false, "When synchronizing, delete files on destination before transfering") flags.BoolVarP(flagSet, &deleteDuring, "delete-during", "", false, "When synchronizing, delete files during transfer (default)") flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transfering") + flags.IntVar64P(flagSet, &fs.Config.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes") flags.BoolVarP(flagSet, &fs.Config.TrackRenames, "track-renames", "", fs.Config.TrackRenames, "When synchronizing, track file renames and do a server side move if possible") flags.IntVarP(flagSet, &fs.Config.LowLevelRetries, "low-level-retries", "", fs.Config.LowLevelRetries, "Number of low level retries to do.") flags.BoolVarP(flagSet, &fs.Config.UpdateOlder, "update", "u", fs.Config.UpdateOlder, "Skip files that are newer on the destination.") diff --git a/fs/config/flags/flags.go b/fs/config/flags/flags.go index bbb4f6bdd..1618dd1fd 100644 --- a/fs/config/flags/flags.go +++ b/fs/config/flags/flags.go @@ -89,6 +89,14 @@ func Int64P(name, shorthand string, value int64, usage string) (out *int64) { return out } +// IntVar64P defines a flag which can be overridden by an environment variable +// +// It is a thin wrapper around pflag.Int64VarP +func IntVar64P(flags *pflag.FlagSet, p *int64, name, shorthand string, value int64, usage string) { + flags.Int64VarP(p, name, shorthand, value, usage) + setDefaultFromEnv(name) +} + // IntVarP defines a flag which can be overridden by an environment variable // // It is a thin wrapper around pflag.IntVarP diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 4fb21c237..19a01dd51 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -416,6 +416,10 @@ func CanServerSideMove(fdst fs.Fs) bool { // deleting func DeleteFileWithBackupDir(dst fs.Object, backupDir fs.Fs) (err error) { accounting.Stats.Checking(dst.Remote()) + numDeletes := accounting.Stats.Deletes(1) + if fs.Config.MaxDelete != -1 && numDeletes > fs.Config.MaxDelete { + return fserrors.FatalError(errors.New("--max-delete threshold reached")) + } action, actioned, actioning := "delete", "Deleted", "deleting" if backupDir != nil { action, actioned, actioning = "move into backup dir", "Moved into backup dir", "moving into backup dir" @@ -460,6 +464,8 @@ func DeleteFilesWithBackupDir(toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error var wg sync.WaitGroup wg.Add(fs.Config.Transfers) var errorCount int32 + var fatalErrorCount int32 + for i := 0; i < fs.Config.Transfers; i++ { go func() { defer wg.Done() @@ -467,6 +473,11 @@ func DeleteFilesWithBackupDir(toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error err := DeleteFileWithBackupDir(dst, backupDir) if err != nil { atomic.AddInt32(&errorCount, 1) + if fserrors.IsFatalError(err) { + fs.Errorf(nil, "Got fatal error on delete: %s", err) + atomic.AddInt32(&fatalErrorCount, 1) + return + } } } }() @@ -474,7 +485,11 @@ func DeleteFilesWithBackupDir(toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error fs.Infof(nil, "Waiting for deletions to finish") wg.Wait() if errorCount > 0 { - return errors.Errorf("failed to delete %d files", errorCount) + err := errors.Errorf("failed to delete %d files", errorCount) + if fatalErrorCount > 0 { + return fserrors.FatalError(err) + } + return err } return nil }