package pacer import ( "math/rand" "time" "golang.org/x/time/rate" ) type ( // MinSleep configures the minimum sleep time of a Calculator MinSleep time.Duration // MaxSleep configures the maximum sleep time of a Calculator MaxSleep time.Duration // DecayConstant configures the decay constant time of a Calculator DecayConstant uint // AttackConstant configures the attack constant of a Calculator AttackConstant uint // Burst configures the number of API calls to allow without sleeping Burst int ) // Default is a truncated exponential attack and decay. // // On retries the sleep time is doubled, on non errors then sleeptime decays // according to the decay constant as set with SetDecayConstant. // // The sleep never goes below that set with SetMinSleep or above that set // with SetMaxSleep. type Default struct { minSleep time.Duration // minimum sleep time maxSleep time.Duration // maximum sleep time decayConstant uint // decay constant attackConstant uint // attack constant } // DefaultOption is the interface implemented by all options for the Default Calculator type DefaultOption interface { ApplyDefault(*Default) } // NewDefault creates a Calculator used by Pacer as the default. func NewDefault(opts ...DefaultOption) *Default { c := &Default{ minSleep: 10 * time.Millisecond, maxSleep: 2 * time.Second, decayConstant: 2, attackConstant: 1, } c.Update(opts...) return c } // Update applies the Calculator options. func (c *Default) Update(opts ...DefaultOption) { for _, opt := range opts { opt.ApplyDefault(c) } } // ApplyDefault updates the value on the Calculator func (o MinSleep) ApplyDefault(c *Default) { c.minSleep = time.Duration(o) } // ApplyDefault updates the value on the Calculator func (o MaxSleep) ApplyDefault(c *Default) { c.maxSleep = time.Duration(o) } // ApplyDefault updates the value on the Calculator func (o DecayConstant) ApplyDefault(c *Default) { c.decayConstant = uint(o) } // ApplyDefault updates the value on the Calculator func (o AttackConstant) ApplyDefault(c *Default) { c.attackConstant = uint(o) } // Calculate takes the current Pacer state and return the wait time until the next try. func (c *Default) Calculate(state State) time.Duration { if t, ok := IsRetryAfter(state.LastError); ok { if t < c.minSleep { return c.minSleep } return t } if state.ConsecutiveRetries > 0 { sleepTime := c.maxSleep if c.attackConstant != 0 { sleepTime = (state.SleepTime << c.attackConstant) / ((1 << c.attackConstant) - 1) } if sleepTime > c.maxSleep { sleepTime = c.maxSleep } return sleepTime } sleepTime := (state.SleepTime<<c.decayConstant - state.SleepTime) >> c.decayConstant if sleepTime < c.minSleep { sleepTime = c.minSleep } return sleepTime } // ZeroDelayCalculator is a Calculator that never delays. type ZeroDelayCalculator struct { } // Calculate takes the current Pacer state and return the wait time until the next try. func (c *ZeroDelayCalculator) Calculate(state State) time.Duration { return 0 } // AmazonCloudDrive is a specialized pacer for Amazon Drive // // It implements a truncated exponential backoff strategy with randomization. // Normally operations are paced at the interval set with SetMinSleep. On errors // the sleep timer is set to 0..2**retries seconds. // // See https://developer.amazon.com/public/apis/experience/cloud-drive/content/restful-api-best-practices type AmazonCloudDrive struct { minSleep time.Duration // minimum sleep time } // AmazonCloudDriveOption is the interface implemented by all options for the AmazonCloudDrive Calculator type AmazonCloudDriveOption interface { ApplyAmazonCloudDrive(*AmazonCloudDrive) } // NewAmazonCloudDrive returns a new AmazonCloudDrive Calculator with default values func NewAmazonCloudDrive(opts ...AmazonCloudDriveOption) *AmazonCloudDrive { c := &AmazonCloudDrive{ minSleep: 10 * time.Millisecond, } c.Update(opts...) return c } // Update applies the Calculator options. func (c *AmazonCloudDrive) Update(opts ...AmazonCloudDriveOption) { for _, opt := range opts { opt.ApplyAmazonCloudDrive(c) } } // ApplyAmazonCloudDrive updates the value on the Calculator func (o MinSleep) ApplyAmazonCloudDrive(c *AmazonCloudDrive) { c.minSleep = time.Duration(o) } // Calculate takes the current Pacer state and return the wait time until the next try. func (c *AmazonCloudDrive) Calculate(state State) time.Duration { if t, ok := IsRetryAfter(state.LastError); ok { if t < c.minSleep { return c.minSleep } return t } consecutiveRetries := state.ConsecutiveRetries if consecutiveRetries == 0 { return c.minSleep } if consecutiveRetries > 9 { consecutiveRetries = 9 } // consecutiveRetries starts at 1 so // maxSleep is 2**(consecutiveRetries-1) seconds maxSleep := time.Second << uint(consecutiveRetries-1) // actual sleep is random from 0..maxSleep sleepTime := time.Duration(rand.Int63n(int64(maxSleep))) if sleepTime < c.minSleep { sleepTime = c.minSleep } return sleepTime } // AzureIMDS is a pacer for the Azure instance metadata service. type AzureIMDS struct { } // NewAzureIMDS returns a new Azure IMDS calculator. func NewAzureIMDS() *AzureIMDS { c := &AzureIMDS{} return c } // Calculate takes the current Pacer state and return the wait time until the next try. func (c *AzureIMDS) Calculate(state State) time.Duration { var addBackoff time.Duration if state.ConsecutiveRetries == 0 { // Initial condition: no backoff. return 0 } if state.ConsecutiveRetries > 4 { // The number of consecutive retries shouldn't exceed five. // In case it does for some reason, cap delay. addBackoff = 0 } else { addBackoff = time.Duration(2<<uint(state.ConsecutiveRetries-1)) * time.Second } return addBackoff + state.SleepTime } // GoogleDrive is a specialized pacer for Google Drive // // It implements a truncated exponential backoff strategy with randomization. // Normally operations are paced at the interval set with SetMinSleep. On errors // the sleep timer is set to (2 ^ n) + random_number_milliseconds seconds. // // See https://developers.google.com/drive/v2/web/handle-errors#exponential-backoff type GoogleDrive struct { minSleep time.Duration // minimum sleep time burst int // number of requests without sleeping limiter *rate.Limiter // rate limiter for the minSleep } // GoogleDriveOption is the interface implemented by all options for the GoogleDrive Calculator type GoogleDriveOption interface { ApplyGoogleDrive(*GoogleDrive) } // NewGoogleDrive returns a new GoogleDrive Calculator with default values func NewGoogleDrive(opts ...GoogleDriveOption) *GoogleDrive { c := &GoogleDrive{ minSleep: 10 * time.Millisecond, burst: 100, } c.Update(opts...) return c } // Update applies the Calculator options. func (c *GoogleDrive) Update(opts ...GoogleDriveOption) { for _, opt := range opts { opt.ApplyGoogleDrive(c) } if c.burst <= 0 { c.burst = 1 } c.limiter = rate.NewLimiter(rate.Every(c.minSleep), c.burst) } // ApplyGoogleDrive updates the value on the Calculator func (o MinSleep) ApplyGoogleDrive(c *GoogleDrive) { c.minSleep = time.Duration(o) } // ApplyGoogleDrive updates the value on the Calculator func (o Burst) ApplyGoogleDrive(c *GoogleDrive) { c.burst = int(o) } // Calculate takes the current Pacer state and return the wait time until the next try. func (c *GoogleDrive) Calculate(state State) time.Duration { if t, ok := IsRetryAfter(state.LastError); ok { if t < c.minSleep { return c.minSleep } return t } consecutiveRetries := state.ConsecutiveRetries if consecutiveRetries == 0 { return c.limiter.Reserve().Delay() } if consecutiveRetries > 5 { consecutiveRetries = 5 } // consecutiveRetries starts at 1 so go from 1,2,3,4,5,5 => 1,2,4,8,16,16 // maxSleep is 2**(consecutiveRetries-1) seconds + random milliseconds return time.Second<<uint(consecutiveRetries-1) + time.Duration(rand.Int63n(int64(time.Second))) } // S3 implements a pacer compatible with our expectations of S3, where it tries to not // delay at all between successful calls, but backs off in the default fashion in response // to any errors. // The assumption is that errors should be exceedingly rare (S3 seems to have largely solved // the sort of stability questions rclone is likely to run into), and in the happy case // it can handle calls with no delays between them. // // Basically defaultPacer, but with some handling of sleepTime going to/from 0ms type S3 struct { minSleep time.Duration // minimum sleep time maxSleep time.Duration // maximum sleep time decayConstant uint // decay constant attackConstant uint // attack constant } // S3Option is the interface implemented by all options for the S3 Calculator type S3Option interface { ApplyS3(*S3) } // NewS3 returns a new S3 Calculator with default values func NewS3(opts ...S3Option) *S3 { c := &S3{ maxSleep: 2 * time.Second, decayConstant: 2, attackConstant: 1, } c.Update(opts...) return c } // Update applies the Calculator options. func (c *S3) Update(opts ...S3Option) { for _, opt := range opts { opt.ApplyS3(c) } } // ApplyS3 updates the value on the Calculator func (o MaxSleep) ApplyS3(c *S3) { c.maxSleep = time.Duration(o) } // ApplyS3 updates the value on the Calculator func (o MinSleep) ApplyS3(c *S3) { c.minSleep = time.Duration(o) } // ApplyS3 updates the value on the Calculator func (o DecayConstant) ApplyS3(c *S3) { c.decayConstant = uint(o) } // ApplyS3 updates the value on the Calculator func (o AttackConstant) ApplyS3(c *S3) { c.attackConstant = uint(o) } // Calculate takes the current Pacer state and return the wait time until the next try. func (c *S3) Calculate(state State) time.Duration { if t, ok := IsRetryAfter(state.LastError); ok { if t < c.minSleep { return c.minSleep } return t } if state.ConsecutiveRetries > 0 { if c.attackConstant == 0 { return c.maxSleep } if state.SleepTime == 0 { return c.minSleep } sleepTime := (state.SleepTime << c.attackConstant) / ((1 << c.attackConstant) - 1) if sleepTime > c.maxSleep { sleepTime = c.maxSleep } return sleepTime } sleepTime := (state.SleepTime<<c.decayConstant - state.SleepTime) >> c.decayConstant if sleepTime < c.minSleep { sleepTime = 0 } return sleepTime }