mirror of
https://github.com/rclone/rclone.git
synced 2024-12-25 00:19:13 +01:00
11da2a6c9b
The purpose of this is to make it easier to maintain and eventually to allow the rclone backends to be re-used in other projects without having to use the rclone configuration system. The new code layout is documented in CONTRIBUTING.
438 lines
11 KiB
Go
438 lines
11 KiB
Go
package pacer
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/ncw/rclone/fs"
|
|
"github.com/ncw/rclone/fs/fserrors"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
func TestNew(t *testing.T) {
|
|
const expectedRetries = 7
|
|
fs.Config.LowLevelRetries = expectedRetries
|
|
p := New()
|
|
if p.minSleep != 10*time.Millisecond {
|
|
t.Errorf("minSleep")
|
|
}
|
|
if p.maxSleep != 2*time.Second {
|
|
t.Errorf("maxSleep")
|
|
}
|
|
if p.sleepTime != p.minSleep {
|
|
t.Errorf("sleepTime")
|
|
}
|
|
if p.retries != expectedRetries {
|
|
t.Errorf("retries want %v got %v", expectedRetries, p.retries)
|
|
}
|
|
if p.decayConstant != 2 {
|
|
t.Errorf("decayConstant")
|
|
}
|
|
if p.attackConstant != 1 {
|
|
t.Errorf("attackConstant")
|
|
}
|
|
if cap(p.pacer) != 1 {
|
|
t.Errorf("pacer 1")
|
|
}
|
|
if len(p.pacer) != 1 {
|
|
t.Errorf("pacer 2")
|
|
}
|
|
if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.defaultPacer) {
|
|
t.Errorf("calculatePace")
|
|
}
|
|
if p.maxConnections != fs.Config.Checkers+fs.Config.Transfers {
|
|
t.Errorf("maxConnections")
|
|
}
|
|
if cap(p.connTokens) != fs.Config.Checkers+fs.Config.Transfers {
|
|
t.Errorf("connTokens")
|
|
}
|
|
if p.consecutiveRetries != 0 {
|
|
t.Errorf("consecutiveRetries")
|
|
}
|
|
}
|
|
|
|
func TestSetSleep(t *testing.T) {
|
|
p := New().SetSleep(2 * time.Millisecond)
|
|
if p.sleepTime != 2*time.Millisecond {
|
|
t.Errorf("didn't set")
|
|
}
|
|
}
|
|
|
|
func TestGetSleep(t *testing.T) {
|
|
p := New().SetSleep(2 * time.Millisecond)
|
|
if p.GetSleep() != 2*time.Millisecond {
|
|
t.Errorf("didn't get")
|
|
}
|
|
}
|
|
|
|
func TestSetMinSleep(t *testing.T) {
|
|
p := New().SetMinSleep(1 * time.Millisecond)
|
|
if p.minSleep != 1*time.Millisecond {
|
|
t.Errorf("didn't set")
|
|
}
|
|
}
|
|
|
|
func TestSetMaxSleep(t *testing.T) {
|
|
p := New().SetMaxSleep(100 * time.Second)
|
|
if p.maxSleep != 100*time.Second {
|
|
t.Errorf("didn't set")
|
|
}
|
|
}
|
|
|
|
func TestMaxConnections(t *testing.T) {
|
|
p := New().SetMaxConnections(20)
|
|
if p.maxConnections != 20 {
|
|
t.Errorf("maxConnections")
|
|
}
|
|
if cap(p.connTokens) != 20 {
|
|
t.Errorf("connTokens")
|
|
}
|
|
p.SetMaxConnections(0)
|
|
if p.maxConnections != 0 {
|
|
t.Errorf("maxConnections is not 0")
|
|
}
|
|
if p.connTokens != nil {
|
|
t.Errorf("connTokens is not nil")
|
|
}
|
|
}
|
|
|
|
func TestSetDecayConstant(t *testing.T) {
|
|
p := New().SetDecayConstant(17)
|
|
if p.decayConstant != 17 {
|
|
t.Errorf("didn't set")
|
|
}
|
|
}
|
|
|
|
func TestDecay(t *testing.T) {
|
|
p := New().SetMinSleep(time.Microsecond).SetPacer(DefaultPacer).SetMaxSleep(time.Second)
|
|
for _, test := range []struct {
|
|
in time.Duration
|
|
attackConstant uint
|
|
want time.Duration
|
|
}{
|
|
{8 * time.Millisecond, 1, 4 * time.Millisecond},
|
|
{1 * time.Millisecond, 0, time.Microsecond},
|
|
{1 * time.Millisecond, 2, (3 * time.Millisecond) / 4},
|
|
{1 * time.Millisecond, 3, (7 * time.Millisecond) / 8},
|
|
} {
|
|
p.sleepTime = test.in
|
|
p.SetDecayConstant(test.attackConstant)
|
|
p.defaultPacer(false)
|
|
got := p.sleepTime
|
|
if got != test.want {
|
|
t.Errorf("bad sleep want %v got %v", test.want, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSetAttackConstant(t *testing.T) {
|
|
p := New().SetAttackConstant(19)
|
|
if p.attackConstant != 19 {
|
|
t.Errorf("didn't set")
|
|
}
|
|
}
|
|
|
|
func TestAttack(t *testing.T) {
|
|
p := New().SetMinSleep(time.Microsecond).SetPacer(DefaultPacer).SetMaxSleep(time.Second)
|
|
for _, test := range []struct {
|
|
in time.Duration
|
|
attackConstant uint
|
|
want time.Duration
|
|
}{
|
|
{1 * time.Millisecond, 1, 2 * time.Millisecond},
|
|
{1 * time.Millisecond, 0, time.Second},
|
|
{1 * time.Millisecond, 2, (4 * time.Millisecond) / 3},
|
|
{1 * time.Millisecond, 3, (8 * time.Millisecond) / 7},
|
|
} {
|
|
p.sleepTime = test.in
|
|
p.SetAttackConstant(test.attackConstant)
|
|
p.defaultPacer(true)
|
|
got := p.sleepTime
|
|
if got != test.want {
|
|
t.Errorf("bad sleep want %v got %v", test.want, got)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func TestSetRetries(t *testing.T) {
|
|
p := New().SetRetries(18)
|
|
if p.retries != 18 {
|
|
t.Errorf("didn't set")
|
|
}
|
|
}
|
|
|
|
func TestSetPacer(t *testing.T) {
|
|
p := New().SetPacer(AmazonCloudDrivePacer)
|
|
if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.acdPacer) {
|
|
t.Errorf("calculatePace is not acdPacer")
|
|
}
|
|
p.SetPacer(GoogleDrivePacer)
|
|
if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.drivePacer) {
|
|
t.Errorf("calculatePace is not drivePacer")
|
|
}
|
|
p.SetPacer(DefaultPacer)
|
|
if fmt.Sprintf("%p", p.calculatePace) != fmt.Sprintf("%p", p.defaultPacer) {
|
|
t.Errorf("calculatePace is not defaultPacer")
|
|
}
|
|
}
|
|
|
|
// emptyTokens empties the pacer of all its tokens
|
|
func emptyTokens(p *Pacer) {
|
|
for len(p.pacer) != 0 {
|
|
<-p.pacer
|
|
}
|
|
for len(p.connTokens) != 0 {
|
|
<-p.connTokens
|
|
}
|
|
}
|
|
|
|
// waitForPace waits for duration for the pace to arrive
|
|
// returns the time that it arrived or a zero time
|
|
func waitForPace(p *Pacer, duration time.Duration) (when time.Time) {
|
|
select {
|
|
case <-time.After(duration):
|
|
return
|
|
case <-p.pacer:
|
|
return time.Now()
|
|
}
|
|
}
|
|
|
|
func TestBeginCall(t *testing.T) {
|
|
p := New().SetMaxConnections(10).SetMinSleep(1 * time.Millisecond)
|
|
emptyTokens(p)
|
|
go p.beginCall()
|
|
if !waitForPace(p, 10*time.Millisecond).IsZero() {
|
|
t.Errorf("beginSleep fired too early #1")
|
|
}
|
|
startTime := time.Now()
|
|
p.pacer <- struct{}{}
|
|
time.Sleep(1 * time.Millisecond)
|
|
connTime := time.Now()
|
|
p.connTokens <- struct{}{}
|
|
time.Sleep(1 * time.Millisecond)
|
|
paceTime := waitForPace(p, 10*time.Millisecond)
|
|
if paceTime.IsZero() {
|
|
t.Errorf("beginSleep didn't fire")
|
|
} else if paceTime.Sub(startTime) < 0 {
|
|
t.Errorf("pace arrived before returning pace token")
|
|
} else if paceTime.Sub(connTime) < 0 {
|
|
t.Errorf("pace arrived before sending conn token")
|
|
}
|
|
}
|
|
|
|
func TestBeginCallZeroConnections(t *testing.T) {
|
|
p := New().SetMaxConnections(0).SetMinSleep(1 * time.Millisecond)
|
|
emptyTokens(p)
|
|
go p.beginCall()
|
|
if !waitForPace(p, 10*time.Millisecond).IsZero() {
|
|
t.Errorf("beginSleep fired too early #1")
|
|
}
|
|
startTime := time.Now()
|
|
p.pacer <- struct{}{}
|
|
time.Sleep(1 * time.Millisecond)
|
|
paceTime := waitForPace(p, 10*time.Millisecond)
|
|
if paceTime.IsZero() {
|
|
t.Errorf("beginSleep didn't fire")
|
|
} else if paceTime.Sub(startTime) < 0 {
|
|
t.Errorf("pace arrived before returning pace token")
|
|
}
|
|
}
|
|
|
|
func TestDefaultPacer(t *testing.T) {
|
|
p := New().SetMinSleep(time.Millisecond).SetPacer(DefaultPacer).SetMaxSleep(time.Second).SetDecayConstant(2)
|
|
for _, test := range []struct {
|
|
in time.Duration
|
|
retry bool
|
|
want time.Duration
|
|
}{
|
|
{time.Millisecond, true, 2 * time.Millisecond},
|
|
{time.Second, true, time.Second},
|
|
{(3 * time.Second) / 4, true, time.Second},
|
|
{time.Second, false, 750 * time.Millisecond},
|
|
{1000 * time.Microsecond, false, time.Millisecond},
|
|
{1200 * time.Microsecond, false, time.Millisecond},
|
|
} {
|
|
p.sleepTime = test.in
|
|
p.defaultPacer(test.retry)
|
|
got := p.sleepTime
|
|
if got != test.want {
|
|
t.Errorf("bad sleep want %v got %v", test.want, got)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func TestAmazonCloudDrivePacer(t *testing.T) {
|
|
p := New().SetMinSleep(time.Millisecond).SetPacer(AmazonCloudDrivePacer).SetMaxSleep(time.Second).SetDecayConstant(2)
|
|
// Do lots of times because of the random number!
|
|
for _, test := range []struct {
|
|
in time.Duration
|
|
consecutiveRetries int
|
|
retry bool
|
|
want time.Duration
|
|
}{
|
|
{time.Millisecond, 0, true, time.Millisecond},
|
|
{10 * time.Millisecond, 0, true, time.Millisecond},
|
|
{1 * time.Second, 1, true, 500 * time.Millisecond},
|
|
{1 * time.Second, 2, true, 1 * time.Second},
|
|
{1 * time.Second, 3, true, 2 * time.Second},
|
|
{1 * time.Second, 4, true, 4 * time.Second},
|
|
{1 * time.Second, 5, true, 8 * time.Second},
|
|
{1 * time.Second, 6, true, 16 * time.Second},
|
|
{1 * time.Second, 7, true, 32 * time.Second},
|
|
{1 * time.Second, 8, true, 64 * time.Second},
|
|
{1 * time.Second, 9, true, 128 * time.Second},
|
|
{1 * time.Second, 10, true, 128 * time.Second},
|
|
{1 * time.Second, 11, true, 128 * time.Second},
|
|
} {
|
|
const n = 1000
|
|
var sum time.Duration
|
|
// measure average time over n cycles
|
|
for i := 0; i < n; i++ {
|
|
p.sleepTime = test.in
|
|
p.consecutiveRetries = test.consecutiveRetries
|
|
p.acdPacer(test.retry)
|
|
sum += p.sleepTime
|
|
}
|
|
got := sum / n
|
|
//t.Logf("%+v: got = %v", test, got)
|
|
if got < (test.want*9)/10 || got > (test.want*11)/10 {
|
|
t.Fatalf("%+v: bad sleep want %v+/-10%% got %v", test, test.want, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGoogleDrivePacer(t *testing.T) {
|
|
p := New().SetMinSleep(time.Millisecond).SetPacer(GoogleDrivePacer).SetMaxSleep(time.Second).SetDecayConstant(2)
|
|
// Do lots of times because of the random number!
|
|
for _, test := range []struct {
|
|
in time.Duration
|
|
consecutiveRetries int
|
|
retry bool
|
|
want time.Duration
|
|
}{
|
|
{time.Millisecond, 0, true, time.Millisecond},
|
|
{10 * time.Millisecond, 0, true, time.Millisecond},
|
|
{1 * time.Second, 1, true, 1*time.Second + 500*time.Millisecond},
|
|
{1 * time.Second, 2, true, 2*time.Second + 500*time.Millisecond},
|
|
{1 * time.Second, 3, true, 4*time.Second + 500*time.Millisecond},
|
|
{1 * time.Second, 4, true, 8*time.Second + 500*time.Millisecond},
|
|
{1 * time.Second, 5, true, 16*time.Second + 500*time.Millisecond},
|
|
{1 * time.Second, 6, true, 16*time.Second + 500*time.Millisecond},
|
|
{1 * time.Second, 7, true, 16*time.Second + 500*time.Millisecond},
|
|
} {
|
|
const n = 1000
|
|
var sum time.Duration
|
|
// measure average time over n cycles
|
|
for i := 0; i < n; i++ {
|
|
p.sleepTime = test.in
|
|
p.consecutiveRetries = test.consecutiveRetries
|
|
p.drivePacer(test.retry)
|
|
sum += p.sleepTime
|
|
}
|
|
got := sum / n
|
|
//t.Logf("%+v: got = %v", test, got)
|
|
if got < (test.want*9)/10 || got > (test.want*11)/10 {
|
|
t.Fatalf("%+v: bad sleep want %v+/-10%% got %v", test, test.want, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEndCall(t *testing.T) {
|
|
p := New().SetMaxConnections(5)
|
|
emptyTokens(p)
|
|
p.consecutiveRetries = 1
|
|
p.endCall(true)
|
|
if len(p.connTokens) != 1 {
|
|
t.Errorf("Expecting 1 token")
|
|
}
|
|
if p.consecutiveRetries != 2 {
|
|
t.Errorf("Bad consecutive retries")
|
|
}
|
|
}
|
|
|
|
func TestEndCallZeroConnections(t *testing.T) {
|
|
p := New().SetMaxConnections(0)
|
|
emptyTokens(p)
|
|
p.consecutiveRetries = 1
|
|
p.endCall(false)
|
|
if len(p.connTokens) != 0 {
|
|
t.Errorf("Expecting 0 token")
|
|
}
|
|
if p.consecutiveRetries != 0 {
|
|
t.Errorf("Bad consecutive retries")
|
|
}
|
|
}
|
|
|
|
var errFoo = errors.New("foo")
|
|
|
|
type dummyPaced struct {
|
|
retry bool
|
|
called int
|
|
}
|
|
|
|
func (dp *dummyPaced) fn() (bool, error) {
|
|
dp.called++
|
|
return dp.retry, errFoo
|
|
}
|
|
|
|
func Test_callNoRetry(t *testing.T) {
|
|
p := New().SetMinSleep(time.Millisecond).SetMaxSleep(2 * time.Millisecond)
|
|
|
|
dp := &dummyPaced{retry: false}
|
|
err := p.call(dp.fn, 10)
|
|
if dp.called != 1 {
|
|
t.Errorf("called want %d got %d", 1, dp.called)
|
|
}
|
|
if err != errFoo {
|
|
t.Errorf("err want %v got %v", errFoo, err)
|
|
}
|
|
}
|
|
|
|
func Test_callRetry(t *testing.T) {
|
|
p := New().SetMinSleep(time.Millisecond).SetMaxSleep(2 * time.Millisecond)
|
|
|
|
dp := &dummyPaced{retry: true}
|
|
err := p.call(dp.fn, 10)
|
|
if dp.called != 10 {
|
|
t.Errorf("called want %d got %d", 10, dp.called)
|
|
}
|
|
if err == errFoo {
|
|
t.Errorf("err didn't want %v got %v", errFoo, err)
|
|
}
|
|
_, ok := err.(fserrors.Retrier)
|
|
if !ok {
|
|
t.Errorf("didn't return a retry error")
|
|
}
|
|
}
|
|
|
|
func TestCall(t *testing.T) {
|
|
p := New().SetMinSleep(time.Millisecond).SetMaxSleep(2 * time.Millisecond).SetRetries(20)
|
|
|
|
dp := &dummyPaced{retry: true}
|
|
err := p.Call(dp.fn)
|
|
if dp.called != 20 {
|
|
t.Errorf("called want %d got %d", 20, dp.called)
|
|
}
|
|
_, ok := err.(fserrors.Retrier)
|
|
if !ok {
|
|
t.Errorf("didn't return a retry error")
|
|
}
|
|
}
|
|
|
|
func TestCallNoRetry(t *testing.T) {
|
|
p := New().SetMinSleep(time.Millisecond).SetMaxSleep(2 * time.Millisecond).SetRetries(20)
|
|
|
|
dp := &dummyPaced{retry: true}
|
|
err := p.CallNoRetry(dp.fn)
|
|
if dp.called != 1 {
|
|
t.Errorf("called want %d got %d", 1, dp.called)
|
|
}
|
|
_, ok := err.(fserrors.Retrier)
|
|
if !ok {
|
|
t.Errorf("didn't return a retry error")
|
|
}
|
|
}
|