extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-12-23 07:29:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

fstest/test_all: limit concurrency and run tests in random order

Browse Source
This commit is contained in:
Nick Craig-Wood 2018-10-24 10:33:38 +01:00
parent 5c32b32011
commit 52c7c738ca
2 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
fstest/test_all/config.go
View File

@ -93,7 +93,7 @@ func NewConfig(configFile string) (*Config, error) {
// MakeRuns makes Run objects for each combination of Backend and Test // MakeRuns makes Run objects for each combination of Backend and Test
// in the config // in the config
func (c *Config) MakeRuns() (runs []*Run) { func (c *Config) MakeRuns() (runs Runs) {
for _, backend := range c.Backends { for _, backend := range c.Backends {
for _, test := range c.Tests { for _, test := range c.Tests {
runs = append(runs, backend.MakeRuns(&test)...) runs = append(runs, backend.MakeRuns(&test)...)

18
fstest/test_all/test_all.go
View File

@ -13,6 +13,7 @@ Make TesTrun have a []string of flags to try - that then makes it generic
import ( import (
"flag" "flag"
"log" "log"
"math/rand"
"os" "os"
"path" "path"
"regexp" "regexp"
@ -20,6 +21,7 @@ import (
"time" "time"
_ "github.com/ncw/rclone/backend/all" // import all fs _ "github.com/ncw/rclone/backend/all" // import all fs
"github.com/ncw/rclone/lib/pacer"
) )
type remoteConfig struct { type remoteConfig struct {
@ -31,6 +33,7 @@ type remoteConfig struct {
var ( var (
// Flags // Flags
maxTries = flag.Int("maxtries", 5, "Number of times to try each test") maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
maxN = flag.Int("n", 20, "Maximum number of tests to run at once")
testRemotes = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'") testRemotes = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
testBackends = flag.String("backends", "", "Comma separated list of backends to test, eg 's3,googlecloudstorage") testBackends = flag.String("backends", "", "Comma separated list of backends to test, eg 's3,googlecloudstorage")
testTests = flag.String("tests", "", "Comma separated list of tests to test, eg 'fs/sync,fs/operations'") testTests = flag.String("tests", "", "Comma separated list of tests to test, eg 'fs/sync,fs/operations'")
@ -72,6 +75,9 @@ func main() {
log.Fatal(err) log.Fatal(err)
} }
// Seed the random number generator
rand.Seed(time.Now().UTC().UnixNano())
// Filter selection // Filter selection
if *testRemotes != "" { if *testRemotes != "" {
conf.filterBackendsByRemotes(strings.Split(*testRemotes, ",")) conf.filterBackendsByRemotes(strings.Split(*testRemotes, ","))
@ -98,8 +104,9 @@ func main() {
} }
log.Printf("Testing remotes: %s", strings.Join(names, ", ")) log.Printf("Testing remotes: %s", strings.Join(names, ", "))
// Runs we will do for this test // Runs we will do for this test in random order
runs := conf.MakeRuns() runs := conf.MakeRuns()
rand.Shuffle(len(runs), runs.Swap)
// Create Report // Create Report
report := NewReport() report := NewReport()
@ -120,10 +127,15 @@ func main() {
_ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m") _ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m")
// start the tests // start the tests
results := make(chan *Run, 8) results := make(chan *Run, len(runs))
awaiting := 0 awaiting := 0
tokens := pacer.NewTokenDispenser(*maxN)
for _, run := range runs { for _, run := range runs {
go run.Run(report.LogDir, results) tokens.Get()
go func(run *Run) {
defer tokens.Put()
run.Run(report.LogDir, results)
}(run)
awaiting++ awaiting++
} }