diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b87a162b4..dc4275046 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -123,6 +123,13 @@ but they can be run against any of the remotes. cd fs/operations go test -v -remote TestDrive: +If you want to use the integration test framework to run these tests +all together with an HTML report and test retries then from the +project root: + + go install github.com/ncw/rclone/fstest/test_all + test_all -backend drive + If you want to run all the integration tests against all the remotes, then change into the project root and run @@ -343,7 +350,7 @@ Unit tests Integration tests - * Add your fs to `fstest/test_all/test_all.go` + * Add your backend to `fstest/test_all/config.yaml` * Make sure integration tests pass with * `cd fs/operations` * `go test -v -remote TestRemote:` diff --git a/Makefile b/Makefile index 8d6ab96fd..d1576a301 100644 --- a/Makefile +++ b/Makefile @@ -51,9 +51,8 @@ version: # Full suite of integration tests test: rclone go install github.com/ncw/rclone/fstest/test_all - -go test -v -count 1 -timeout 20m $(BUILDTAGS) $(GO_FILES) 2>&1 | tee test.log - -test_all github.com/ncw/rclone/fs/operations github.com/ncw/rclone/fs/sync 2>&1 | tee fs/test_all.log - @echo "Written logs in test.log and fs/test_all.log" + -test_all 2>&1 | tee test_all.log + @echo "Written logs in test_all.log" # Quick test quicktest: diff --git a/fstest/test_all/clean.go b/fstest/test_all/clean.go new file mode 100644 index 000000000..8ce451bdc --- /dev/null +++ b/fstest/test_all/clean.go @@ -0,0 +1,60 @@ +// Clean the left over test files + +package main + +import ( + "log" + "regexp" + + "github.com/ncw/rclone/fs" + "github.com/ncw/rclone/fs/list" + "github.com/ncw/rclone/fs/operations" +) + +// MatchTestRemote matches the remote names used for testing (copied +// from fstest/fstest.go so we don't have to import that and get all +// its flags) +var MatchTestRemote = regexp.MustCompile(`^rclone-test-[abcdefghijklmnopqrstuvwxyz0123456789]{24}$`) + +// cleanFs runs a single clean fs for left over directories +func cleanFs(remote string) error { + f, err := fs.NewFs(remote) + if err != nil { + return err + } + entries, err := list.DirSorted(f, true, "") + if err != nil { + return err + } + return entries.ForDirError(func(dir fs.Directory) error { + dirPath := dir.Remote() + fullPath := remote + dirPath + if MatchTestRemote.MatchString(dirPath) { + if *dryRun { + log.Printf("Not Purging %s - -dry-run", fullPath) + return nil + } + log.Printf("Purging %s", fullPath) + dir, err := fs.NewFs(fullPath) + if err != nil { + return err + } + return operations.Purge(dir, "") + } + return nil + }) +} + +// cleanRemotes cleans the list of remotes passed in +func cleanRemotes(remotes []string) error { + var lastError error + for _, remote := range remotes { + log.Printf("%q - Cleaning", remote) + err := cleanFs(remote) + if err != nil { + lastError = err + log.Printf("Failed to purge %q: %v", remote, err) + } + } + return lastError +} diff --git a/fstest/test_all/config.go b/fstest/test_all/config.go new file mode 100644 index 000000000..ecffb2753 --- /dev/null +++ b/fstest/test_all/config.go @@ -0,0 +1,159 @@ +// Config handling + +package main + +import ( + "io/ioutil" + "log" + "path" + + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +// Test describes an integration test to run with `go test` +type Test struct { + Path string // path to the source directory + SubDir bool // if it is possible to add -sub-dir to tests + FastList bool // if it is possible to add -fast-list to tests + AddBackend bool // set if Path needs the current backend appending + NoRetries bool // set if no retries should be performed +} + +// Backend describes a backend test +// +// FIXME make bucket based remotes set sub-dir automatically??? +type Backend struct { + Backend string // name of the backend directory + Remote string // name of the test remote + SubDir bool // set to test with -sub-dir + FastList bool // set to test with -fast-list +} + +// MakeRuns creates Run objects the Backend and Test +// +// There can be several created, one for each combination of SubDir +// and FastList +func (b *Backend) MakeRuns(t *Test) (runs []*Run) { + subdirs := []bool{false} + if b.SubDir && t.SubDir { + subdirs = append(subdirs, true) + } + fastlists := []bool{false} + if b.FastList && t.FastList { + fastlists = append(fastlists, true) + } + for _, subdir := range subdirs { + for _, fastlist := range fastlists { + run := &Run{ + Remote: b.Remote, + Backend: b.Backend, + Path: t.Path, + SubDir: subdir, + FastList: fastlist, + NoRetries: t.NoRetries, + } + if t.AddBackend { + run.Path = path.Join(run.Path, b.Backend) + } + runs = append(runs, run) + } + } + return runs +} + +// Config describes the config for this program +type Config struct { + Tests []Test + Backends []Backend +} + +// NewConfig reads the config file +func NewConfig(configFile string) (*Config, error) { + d, err := ioutil.ReadFile(configFile) + if err != nil { + return nil, errors.Wrap(err, "failed to read config file") + } + config := &Config{} + err = yaml.Unmarshal(d, &config) + if err != nil { + return nil, errors.Wrap(err, "failed to parse config file") + } + // d, err = yaml.Marshal(&config) + // if err != nil { + // log.Fatalf("error: %v", err) + // } + // fmt.Printf("--- m dump:\n%s\n\n", string(d)) + return config, nil +} + +// MakeRuns makes Run objects for each combination of Backend and Test +// in the config +func (c *Config) MakeRuns() (runs []*Run) { + for _, backend := range c.Backends { + for _, test := range c.Tests { + runs = append(runs, backend.MakeRuns(&test)...) + } + } + return runs +} + +// Filter the Backends with the remotes passed in. +// +// If no backend is found with a remote is found then synthesize one +func (c *Config) filterBackendsByRemotes(remotes []string) { + var newBackends []Backend + for _, name := range remotes { + found := false + for i := range c.Backends { + if c.Backends[i].Remote == name { + newBackends = append(newBackends, c.Backends[i]) + found = true + } + } + if !found { + log.Printf("Remote %q not found - inserting with default flags", name) + newBackends = append(newBackends, Backend{Remote: name}) + } + } + c.Backends = newBackends +} + +// Filter the Backends with the backendNames passed in +func (c *Config) filterBackendsByBackends(backendNames []string) { + var newBackends []Backend + for _, name := range backendNames { + for i := range c.Backends { + if c.Backends[i].Backend == name { + newBackends = append(newBackends, c.Backends[i]) + } + } + } + c.Backends = newBackends +} + +// Filter the incoming tests into the backends selected +func (c *Config) filterTests(paths []string) { + var newTests []Test + for _, path := range paths { + for i := range c.Tests { + if c.Tests[i].Path == path { + newTests = append(newTests, c.Tests[i]) + } + } + } + c.Tests = newTests +} + +// Remotes returns the unique remotes +func (c *Config) Remotes() (remotes []string) { + found := map[string]struct{}{} + for _, backend := range c.Backends { + if _, ok := found[backend.Remote]; ok { + continue + } + remotes = append(remotes, backend.Remote) + found[backend.Remote] = struct{}{} + } + return remotes +} diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml new file mode 100644 index 000000000..25738416a --- /dev/null +++ b/fstest/test_all/config.yaml @@ -0,0 +1,107 @@ +tests: + - path: backend + addbackend: true + noretries: true + - path: fs/operations + subdir: true + fastlist: true + - path: fs/sync + subdir: true + fastlist: true +backends: + # - backend: "amazonclouddrive" + # remote: "TestAmazonCloudDrive:" + # subdir: false + # fastlist: false + - backend: "b2" + remote: "TestB2:" + subdir: true + fastlist: true + - backend: "crypt" + remote: "TestCryptDrive:" + subdir: false + fastlist: true + - backend: "crypt" + remote: "TestCryptSwift:" + subdir: false + fastlist: false + - backend: "drive" + remote: "TestDrive:" + subdir: false + fastlist: true + - backend: "dropbox" + remote: "TestDropbox:" + subdir: false + fastlist: false + - backend: "googlecloudstorage" + remote: "TestGoogleCloudStorage:" + subdir: true + fastlist: true + - backend: "hubic" + remote: "TestHubic:" + subdir: false + fastlist: false + - backend: "jottacloud" + remote: "TestJottacloud:" + subdir: false + fastlist: true + - backend: "onedrive" + remote: "TestOneDrive:" + subdir: false + fastlist: false + - backend: "s3" + remote: "TestS3:" + subdir: true + fastlist: true + - backend: "sftp" + remote: "TestSftp:" + subdir: false + fastlist: false + - backend: "swift" + remote: "TestSwift:" + subdir: true + fastlist: true + - backend: "yandex" + remote: "TestYandex:" + subdir: false + fastlist: false + - backend: "ftp" + remote: "TestFTP:" + subdir: false + fastlist: false + - backend: "box" + remote: "TestBox:" + subdir: false + fastlist: false + - backend: "qingstor" + remote: "TestQingStor:" + subdir: false + fastlist: false + - backend: "azureblob" + remote: "TestAzureBlob:" + subdir: true + fastlist: true + - backend: "pcloud" + remote: "TestPcloud:" + subdir: false + fastlist: false + - backend: "webdav" + remote: "TestWebdav:" + subdir: false + fastlist: false + - backend: "cache" + remote: "TestCache:" + subdir: false + fastlist: false + - backend: "mega" + remote: "TestMega:" + subdir: false + fastlist: false + - backend: "opendrive" + remote: "TestOpenDrive:" + subdir: false + fastlist: false + - backend: "union" + remote: "TestUnion:" + subdir: false + fastlist: false diff --git a/fstest/test_all/report.go b/fstest/test_all/report.go new file mode 100644 index 000000000..a647239a2 --- /dev/null +++ b/fstest/test_all/report.go @@ -0,0 +1,260 @@ +package main + +import ( + "fmt" + "html/template" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "sort" + "time" + + "github.com/ncw/rclone/fs" + "github.com/skratchdot/open-golang/open" +) + +const timeFormat = "2006-01-02-150405" + +// Report holds the info to make a report on a series of test runs +type Report struct { + LogDir string // output directory for logs and report + StartTime time.Time // time started + DateTime string // directory name for output + Duration time.Duration // time the run took + Failed Runs // failed runs + Passed Runs // passed runs + Runs []ReportRun // runs to report + Version string // rclone version + Previous string // previous test name if known + IndexHTML string // path to the index.html file + URL string // online version +} + +// ReportRun is used in the templates to report on a test run +type ReportRun struct { + Name string + Runs Runs +} + +// NewReport initialises and returns a Report +func NewReport() *Report { + r := &Report{ + StartTime: time.Now(), + Version: fs.Version, + } + r.DateTime = r.StartTime.Format(timeFormat) + + // Find previous log directory if possible + names, err := ioutil.ReadDir(*outputDir) + if err == nil && len(names) > 0 { + r.Previous = names[len(names)-1].Name() + } + + // Create output directory for logs and report + r.LogDir = path.Join(*outputDir, r.DateTime) + err = os.MkdirAll(r.LogDir, 0777) + if err != nil { + log.Fatalf("Failed to make log directory: %v", err) + } + + // Online version + r.URL = *urlBase + r.DateTime + "/index.html" + + return r +} + +// End should be called when the tests are complete +func (r *Report) End() { + r.Duration = time.Since(r.StartTime) + sort.Sort(r.Failed) + sort.Sort(r.Passed) + r.Runs = []ReportRun{ + {Name: "Failed", Runs: r.Failed}, + {Name: "Passed", Runs: r.Passed}, + } +} + +// AllPassed returns true if there were no failed tests +func (r *Report) AllPassed() bool { + return len(r.Failed) == 0 +} + +// RecordResult should be called with a Run when it has finished to be +// recorded into the Report +func (r *Report) RecordResult(t *Run) { + if !t.passed() { + r.Failed = append(r.Failed, t) + } else { + r.Passed = append(r.Passed, t) + } +} + +// Title returns a human readable summary title for the Report +func (r *Report) Title() string { + if r.AllPassed() { + return fmt.Sprintf("PASS: All tests finished OK in %v", r.Duration) + } + return fmt.Sprintf("FAIL: %d tests failed in %v", len(r.Failed), r.Duration) +} + +// LogSummary writes the summary to the log file +func (r *Report) LogSummary() { + log.Printf("Logs in %q", r.LogDir) + + // Summarise results + log.Printf("SUMMARY") + log.Println(r.Title()) + if !r.AllPassed() { + for _, t := range r.Failed { + log.Printf(" * %s", toShell(t.nextCmdLine())) + log.Printf(" * Failed tests: %v", t.failedTests) + } + } +} + +// LogHTML writes the summary to index.html in LogDir +func (r *Report) LogHTML() { + r.IndexHTML = path.Join(r.LogDir, "index.html") + out, err := os.Create(r.IndexHTML) + if err != nil { + log.Fatalf("Failed to open index.html: %v", err) + } + defer func() { + err := out.Close() + if err != nil { + log.Fatalf("Failed to close index.html: %v", err) + } + }() + err = reportTemplate.Execute(out, r) + if err != nil { + log.Fatalf("Failed to execute template: %v", err) + } + _ = open.Start("file://" + r.IndexHTML) +} + +var reportHTML = ` + + + +{{ .Title }} + + + +

{{ .Title }}

+ + + + + +{{ if .Previous}}{{ end }} + +
Version{{ .Version }}
Date{{ .DateTime}} [online]
Duration{{ .Duration }}
Previous{{ .Previous }}
UpOlder Tests
+ +{{ range .Runs }} +{{ if .Runs }} +

{{ .Name }}: {{ len .Runs }}

+ + + + + + + + + + +{{ $prevBackend := "" }} +{{ $prevRemote := "" }} +{{ range .Runs}} + + + + + + + + + +{{ end }} +
BackendRemoteTestSubDirFastListFailedLogs
{{ if ne $prevBackend .Backend }}{{ .Backend }}{{ end }}{{ $prevBackend = .Backend }}{{ if ne $prevRemote .Remote }}{{ .Remote }}{{ end }}{{ $prevRemote = .Remote }}{{ .Path }}{{ .SubDir }}{{ .FastList }}{{ .FailedTests }}{{ range $i, $v := .Logs }}#{{ $i }} {{ end }}
+{{ end }} +{{ end }} + + +` + +var reportTemplate = template.Must(template.New("Report").Parse(reportHTML)) + +// EmailHTML sends the summary report to the email address supplied +func (r *Report) EmailHTML() { + if *emailReport == "" || r.IndexHTML == "" { + return + } + log.Printf("Sending email summary to %q", *emailReport) + cmdLine := []string{"mail", "-a", "Content-Type: text/html", *emailReport, "-s", "rclone integration tests: " + r.Title()} + cmd := exec.Command(cmdLine[0], cmdLine[1:]...) + in, err := os.Open(r.IndexHTML) + if err != nil { + log.Fatalf("Failed to open index.html: %v", err) + } + cmd.Stdin = in + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + log.Fatalf("Failed to send email: %v", err) + } + _ = in.Close() +} + +// Upload uploads a copy of the report online +func (r *Report) Upload() { + if *uploadPath == "" || r.IndexHTML == "" { + return + } + dst := path.Join(*uploadPath, r.DateTime) + log.Printf("Uploading results to %q", dst) + cmdLine := []string{"rclone", "copy", "-v", r.LogDir, dst} + cmd := exec.Command(cmdLine[0], cmdLine[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + log.Fatalf("Failed to upload results: %v", err) + } +} diff --git a/fstest/test_all/run.go b/fstest/test_all/run.go new file mode 100644 index 000000000..a3d51fbf5 --- /dev/null +++ b/fstest/test_all/run.go @@ -0,0 +1,318 @@ +// Run a test + +package main + +import ( + "bytes" + "fmt" + "go/build" + "io" + "log" + "os" + "os/exec" + "path" + "regexp" + "runtime" + "strings" + "time" + + "github.com/ncw/rclone/fs" +) + +const testBase = "github.com/ncw/rclone/" + +// Run holds info about a running test +// +// A run just runs one command line, but it can be run multiple times +// if retries are needed. +type Run struct { + // Config + Remote string // name of the test remote + Backend string // name of the backend + Path string // path to the source directory + SubDir bool // add -sub-dir to tests + FastList bool // add -fast-list to tests + NoRetries bool // don't retry if set + // Internals + cmdLine []string + cmdString string + try int + err error + output []byte + failedTests []string + runFlag string + logDir string // directory to place the logs + trialName string // name/log file name of current trial + trialNames []string // list of all the trials +} + +// Runs records multiple Run objects +type Runs []*Run + +// Sort interface +func (rs Runs) Len() int { return len(rs) } +func (rs Runs) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] } +func (rs Runs) Less(i, j int) bool { + a, b := rs[i], rs[j] + if a.Backend < b.Backend { + return true + } else if a.Backend > b.Backend { + return false + } + if a.Remote < b.Remote { + return true + } else if a.Remote > b.Remote { + return false + } + if a.Path < b.Path { + return true + } else if a.Path > b.Path { + return false + } + if !a.SubDir && b.SubDir { + return true + } else if a.SubDir && !b.SubDir { + return false + } + if !a.FastList && b.FastList { + return true + } else if a.FastList && !b.FastList { + return false + } + return false +} + +// dumpOutput prints the error output +func (r *Run) dumpOutput() { + log.Println("------------------------------------------------------------") + log.Printf("---- %q ----", r.cmdString) + log.Println(string(r.output)) + log.Println("------------------------------------------------------------") +} + +var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`) + +// findFailures looks for all the tests which failed +func (r *Run) findFailures() { + oldFailedTests := r.failedTests + r.failedTests = nil + for _, matches := range failRe.FindAllSubmatch(r.output, -1) { + r.failedTests = append(r.failedTests, string(matches[1])) + } + if len(r.failedTests) != 0 { + r.runFlag = "^(" + strings.Join(r.failedTests, "|") + ")$" + } else { + r.runFlag = "" + } + if r.passed() && len(r.failedTests) != 0 { + log.Printf("%q - Expecting no errors but got: %v", r.cmdString, r.failedTests) + r.dumpOutput() + } else if !r.passed() && len(r.failedTests) == 0 { + log.Printf("%q - Expecting errors but got none: %v", r.cmdString, r.failedTests) + r.dumpOutput() + r.failedTests = oldFailedTests + } +} + +// nextCmdLine returns the next command line +func (r *Run) nextCmdLine() []string { + cmdLine := r.cmdLine + if r.runFlag != "" { + cmdLine = append(cmdLine, "-test.run", r.runFlag) + } + return cmdLine +} + +// trial runs a single test +func (r *Run) trial() { + cmdLine := r.nextCmdLine() + cmdString := toShell(cmdLine) + msg := fmt.Sprintf("%q - Starting (try %d/%d)", cmdString, r.try, *maxTries) + log.Println(msg) + logName := path.Join(r.logDir, r.trialName) + out, err := os.Create(logName) + if err != nil { + log.Fatalf("Couldn't create log file: %v", err) + } + defer func() { + err := out.Close() + if err != nil { + log.Fatalf("Failed to close log file: %v", err) + } + }() + _, _ = fmt.Fprintln(out, msg) + + // Early exit if --try-run + if *dryRun { + log.Printf("Not executing as --dry-run: %v", cmdLine) + _, _ = fmt.Fprintln(out, "--dry-run is set - not running") + return + } + + // Internal buffer + var b bytes.Buffer + multiOut := io.MultiWriter(out, &b) + + cmd := exec.Command(cmdLine[0], cmdLine[1:]...) + cmd.Stderr = multiOut + cmd.Stdout = multiOut + start := time.Now() + r.err = cmd.Run() + r.output = b.Bytes() + duration := time.Since(start) + r.findFailures() + if r.passed() { + msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, r.try, *maxTries) + } else { + msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, r.try, *maxTries, r.err, r.failedTests) + } + log.Println(msg) + _, _ = fmt.Fprintln(out, msg) +} + +// passed returns true if the test passed +func (r *Run) passed() bool { + return r.err == nil +} + +// GOPATH returns the current GOPATH +func GOPATH() string { + gopath := os.Getenv("GOPATH") + if gopath == "" { + gopath = build.Default.GOPATH + } + return gopath +} + +// BinaryName turns a package name into a binary name +func (r *Run) BinaryName() string { + binary := path.Base(r.Path) + ".test" + if runtime.GOOS == "windows" { + binary += ".exe" + } + return binary +} + +// BinaryPath turns a package name into a binary path +func (r *Run) BinaryPath() string { + return path.Join(r.Path, r.BinaryName()) +} + +// PackagePath returns the path to the package +func (r *Run) PackagePath() string { + return path.Join(GOPATH(), "src", r.Path) +} + +// Chdir into the package directory +func (r *Run) Chdir() { + err := os.Chdir(r.PackagePath()) + if err != nil { + log.Fatalf("Failed to chdir to package %q: %v", r.Path, err) + } +} + +// MakeTestBinary makes the binary we will run +func (r *Run) MakeTestBinary() { + binary := r.BinaryPath() + binaryName := r.BinaryName() + log.Printf("%s: Making test binary %q", r.Path, binaryName) + cmdLine := []string{"go", "test", "-c", "-o", binary, testBase + r.Path} + if *dryRun { + log.Printf("Not executing: %v", cmdLine) + return + } + err := exec.Command(cmdLine[0], cmdLine[1:]...).Run() + if err != nil { + log.Fatalf("Failed to make test binary: %v", err) + } + if _, err := os.Stat(binary); err != nil { + log.Fatalf("Couldn't find test binary %q", binary) + } +} + +// RemoveTestBinary removes the binary made in makeTestBinary +func (r *Run) RemoveTestBinary() { + if *dryRun { + return + } + binary := r.BinaryPath() + err := os.Remove(binary) // Delete the binary when finished + if err != nil { + log.Printf("Error removing test binary %q: %v", binary, err) + } +} + +// Name returns the run name as a file name friendly string +func (r *Run) Name() string { + ns := []string{ + r.Backend, + strings.Replace(r.Path, "/", ".", -1), + r.Remote, + } + if r.SubDir { + ns = append(ns, "subdir") + } + if r.FastList { + ns = append(ns, "fastlist") + } + ns = append(ns, fmt.Sprintf("%d", r.try)) + s := strings.Join(ns, "-") + s = strings.Replace(s, ":", "", -1) + return s +} + +// Init the Run +func (r *Run) Init() { + binary := r.BinaryPath() + r.cmdLine = []string{binary, "-test.v", "-test.timeout", timeout.String(), "-remote", r.Remote} + r.try = 1 + if *verbose { + r.cmdLine = append(r.cmdLine, "-verbose") + fs.Config.LogLevel = fs.LogLevelDebug + } + if *runOnly != "" { + r.cmdLine = append(r.cmdLine, "-test.run", *runOnly) + } + if r.SubDir { + r.cmdLine = append(r.cmdLine, "-subdir") + } + if r.FastList { + r.cmdLine = append(r.cmdLine, "-fast-list") + } + r.cmdString = toShell(r.cmdLine) +} + +// Logs returns all the log names +func (r *Run) Logs() []string { + return r.trialNames +} + +// FailedTests returns the failed tests as a comma separated string, limiting the number +func (r *Run) FailedTests() string { + const maxTests = 5 + ts := r.failedTests + if len(ts) > maxTests { + ts = ts[:maxTests:maxTests] + ts = append(ts, fmt.Sprintf("… (%d more)", len(r.failedTests)-maxTests)) + } + return strings.Join(ts, ", ") +} + +// Run runs all the trials for this test +func (r *Run) Run(logDir string, result chan<- *Run) { + r.Init() + r.logDir = logDir + for r.try = 1; r.try <= *maxTries; r.try++ { + r.trialName = r.Name() + ".txt" + r.trialNames = append(r.trialNames, r.trialName) + log.Printf("Starting run with log %q", r.trialName) + r.trial() + if r.passed() || r.NoRetries { + break + } + } + if !r.passed() { + r.dumpOutput() + } + result <- r +} diff --git a/fstest/test_all/test_all.go b/fstest/test_all/test_all.go index c74879b05..d37179353 100644 --- a/fstest/test_all/test_all.go +++ b/fstest/test_all/test_all.go @@ -4,24 +4,22 @@ // See the `test` target in the Makefile. package main +/* FIXME + +Make TesTrun have a []string of flags to try - that then makes it generic + +*/ + import ( "flag" - "go/build" "log" "os" - "os/exec" "path" "regexp" - "runtime" "strings" "time" _ "github.com/ncw/rclone/backend/all" // import all fs - "github.com/ncw/rclone/fs" - "github.com/ncw/rclone/fs/config" - "github.com/ncw/rclone/fs/list" - "github.com/ncw/rclone/fs/operations" - "github.com/ncw/rclone/fstest" ) type remoteConfig struct { @@ -31,218 +29,23 @@ type remoteConfig struct { } var ( - remotes = []remoteConfig{ - // { - // Name: "TestAmazonCloudDrive:", - // SubDir: false, - // FastList: false, - // }, - { - Name: "TestB2:", - SubDir: true, - FastList: true, - }, - { - Name: "TestCryptDrive:", - SubDir: false, - FastList: true, - }, - { - Name: "TestCryptSwift:", - SubDir: false, - FastList: false, - }, - { - Name: "TestDrive:", - SubDir: false, - FastList: true, - }, - { - Name: "TestDropbox:", - SubDir: false, - FastList: false, - }, - { - Name: "TestGoogleCloudStorage:", - SubDir: true, - FastList: true, - }, - { - Name: "TestHubic:", - SubDir: false, - FastList: false, - }, - { - Name: "TestJottacloud:", - SubDir: false, - FastList: true, - }, - { - Name: "TestOneDrive:", - SubDir: false, - FastList: false, - }, - { - Name: "TestS3:", - SubDir: true, - FastList: true, - }, - { - Name: "TestSftp:", - SubDir: false, - FastList: false, - }, - { - Name: "TestSwift:", - SubDir: true, - FastList: true, - }, - { - Name: "TestYandex:", - SubDir: false, - FastList: false, - }, - { - Name: "TestFTP:", - SubDir: false, - FastList: false, - }, - { - Name: "TestBox:", - SubDir: false, - FastList: false, - }, - { - Name: "TestQingStor:", - SubDir: false, - FastList: false, - }, - { - Name: "TestAzureBlob:", - SubDir: true, - FastList: true, - }, - { - Name: "TestPcloud:", - SubDir: false, - FastList: false, - }, - { - Name: "TestWebdav:", - SubDir: false, - FastList: false, - }, - { - Name: "TestCache:", - SubDir: false, - FastList: false, - }, - { - Name: "TestMega:", - SubDir: false, - FastList: false, - }, - { - Name: "TestOpenDrive:", - SubDir: false, - FastList: false, - }, - { - Name: "TestUnion:", - SubDir: false, - FastList: false, - }, - } // Flags - maxTries = flag.Int("maxtries", 5, "Number of times to try each test") - runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'") - clean = flag.Bool("clean", false, "Instead of testing, clean all left over test directories") - runOnly = flag.String("run", "", "Run only those tests matching the regexp supplied") - timeout = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up") + maxTries = flag.Int("maxtries", 5, "Number of times to try each test") + testRemotes = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'") + testBackends = flag.String("backends", "", "Comma separated list of backends to test, eg 's3,googlecloudstorage") + testTests = flag.String("tests", "", "Comma separated list of tests to test, eg 'fs/sync,fs/operations'") + clean = flag.Bool("clean", false, "Instead of testing, clean all left over test directories") + runOnly = flag.String("run", "", "Run only those tests matching the regexp supplied") + timeout = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up") + configFile = flag.String("config", "fstest/test_all/config.yaml", "Path to config file") + outputDir = flag.String("output", path.Join(os.TempDir(), "rclone-integration-tests"), "Place to store results") + emailReport = flag.String("email", "", "Set to email the report to the address supplied") + dryRun = flag.Bool("dry-run", false, "Print commands which would be executed only") + urlBase = flag.String("url-base", "https://pub.rclone.org/integration-tests/", "Base for the online version") + uploadPath = flag.String("upload", "", "Set this to an rclone path to upload the results here") + verbose = flag.Bool("verbose", false, "Set to enable verbose logging in the tests") ) -// test holds info about a running test -type test struct { - pkg string - remote string - subdir bool - cmdLine []string - cmdString string - try int - err error - output []byte - failedTests []string - runFlag string -} - -// newTest creates a new test -func newTest(pkg, remote string, subdir bool, fastlist bool) *test { - binary := pkgBinary(pkg) - t := &test{ - pkg: pkg, - remote: remote, - subdir: subdir, - cmdLine: []string{binary, "-test.timeout", timeout.String(), "-remote", remote}, - try: 1, - } - if *fstest.Verbose { - t.cmdLine = append(t.cmdLine, "-test.v") - fs.Config.LogLevel = fs.LogLevelDebug - } - if *runOnly != "" { - t.cmdLine = append(t.cmdLine, "-test.run", *runOnly) - } - if subdir { - t.cmdLine = append(t.cmdLine, "-subdir") - } - if fastlist { - t.cmdLine = append(t.cmdLine, "-fast-list") - } - t.cmdString = toShell(t.cmdLine) - return t -} - -// dumpOutput prints the error output -func (t *test) dumpOutput() { - log.Println("------------------------------------------------------------") - log.Printf("---- %q ----", t.cmdString) - log.Println(string(t.output)) - log.Println("------------------------------------------------------------") -} - -var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`) - -// findFailures looks for all the tests which failed -func (t *test) findFailures() { - oldFailedTests := t.failedTests - t.failedTests = nil - for _, matches := range failRe.FindAllSubmatch(t.output, -1) { - t.failedTests = append(t.failedTests, string(matches[1])) - } - if len(t.failedTests) != 0 { - t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$" - } else { - t.runFlag = "" - } - if t.passed() && len(t.failedTests) != 0 { - log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests) - t.dumpOutput() - } else if !t.passed() && len(t.failedTests) == 0 { - log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests) - t.dumpOutput() - t.failedTests = oldFailedTests - } -} - -// nextCmdLine returns the next command line -func (t *test) nextCmdLine() []string { - cmdLine := t.cmdLine - if t.runFlag != "" { - cmdLine = append(cmdLine, "-test.run", t.runFlag) - } - return cmdLine -} - // if matches then is definitely OK in the shell var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$") @@ -261,181 +64,53 @@ func toShell(args []string) (result string) { return result } -// trial runs a single test -func (t *test) trial() { - cmdLine := t.nextCmdLine() - cmdString := toShell(cmdLine) - log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries) - cmd := exec.Command(cmdLine[0], cmdLine[1:]...) - start := time.Now() - t.output, t.err = cmd.CombinedOutput() - duration := time.Since(start) - t.findFailures() - if t.passed() { - log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries) - } else { - log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests) - } -} - -// cleanFs runs a single clean fs for left over directories -func (t *test) cleanFs() error { - f, err := fs.NewFs(t.remote) - if err != nil { - return err - } - entries, err := list.DirSorted(f, true, "") - if err != nil { - return err - } - return entries.ForDirError(func(dir fs.Directory) error { - remote := dir.Remote() - if fstest.MatchTestRemote.MatchString(remote) { - log.Printf("Purging %s%s", t.remote, remote) - dir, err := fs.NewFs(t.remote + remote) - if err != nil { - return err - } - return operations.Purge(dir, "") - } - return nil - }) -} - -// clean runs a single clean on a fs for left over directories -func (t *test) clean() { - log.Printf("%q - Starting clean (try %d/%d)", t.remote, t.try, *maxTries) - start := time.Now() - t.err = t.cleanFs() - if t.err != nil { - log.Printf("%q - Failed to purge %v", t.remote, t.err) - } - duration := time.Since(start) - if t.passed() { - log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries) - } else { - log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err) - } -} - -// passed returns true if the test passed -func (t *test) passed() bool { - return t.err == nil -} - -// run runs all the trials for this test -func (t *test) run(result chan<- *test) { - for t.try = 1; t.try <= *maxTries; t.try++ { - if *clean { - if !t.subdir { - t.clean() - } - } else { - t.trial() - } - if t.passed() { - break - } - } - if !t.passed() { - t.dumpOutput() - } - result <- t -} - -// GOPATH returns the current GOPATH -func GOPATH() string { - gopath := os.Getenv("GOPATH") - if gopath == "" { - gopath = build.Default.GOPATH - } - return gopath -} - -// turn a package name into a binary name -func pkgBinaryName(pkg string) string { - binary := path.Base(pkg) + ".test" - if runtime.GOOS == "windows" { - binary += ".exe" - } - return binary -} - -// turn a package name into a binary path -func pkgBinary(pkg string) string { - return path.Join(pkgPath(pkg), pkgBinaryName(pkg)) -} - -// returns the path to the package -func pkgPath(pkg string) string { - return path.Join(GOPATH(), "src", pkg) -} - -// cd into the package directory -func pkgChdir(pkg string) { - err := os.Chdir(pkgPath(pkg)) - if err != nil { - log.Fatalf("Failed to chdir to package %q: %v", pkg, err) - } -} - -// makeTestBinary makes the binary we will run -func makeTestBinary(pkg string) { - binaryName := pkgBinaryName(pkg) - log.Printf("%s: Making test binary %q", pkg, binaryName) - pkgChdir(pkg) - err := exec.Command("go", "test", "-c", "-o", binaryName).Run() - if err != nil { - log.Fatalf("Failed to make test binary: %v", err) - } - binary := pkgBinary(pkg) - if _, err := os.Stat(binary); err != nil { - log.Fatalf("Couldn't find test binary %q", binary) - } -} - -// removeTestBinary removes the binary made in makeTestBinary -func removeTestBinary(pkg string) { - binary := pkgBinary(pkg) - err := os.Remove(binary) // Delete the binary when finished - if err != nil { - log.Printf("Error removing test binary %q: %v", binary, err) - } -} - func main() { flag.Parse() - packages := flag.Args() - log.Printf("Testing packages: %s", strings.Join(packages, ", ")) - if *runTests != "" { - newRemotes := []remoteConfig{} - for _, name := range strings.Split(*runTests, ",") { - for i := range remotes { - if remotes[i].Name == name { - newRemotes = append(newRemotes, remotes[i]) - goto found - } - } - log.Printf("Remote %q not found - inserting with default flags", name) - newRemotes = append(newRemotes, remoteConfig{Name: name}) - found: - } - remotes = newRemotes + conf, err := NewConfig(*configFile) + if err != nil { + log.Println("test_all should be run from the root of the rclone source code") + log.Fatal(err) } + + // Filter selection + if *testRemotes != "" { + conf.filterBackendsByRemotes(strings.Split(*testRemotes, ",")) + } + if *testBackends != "" { + conf.filterBackendsByBackends(strings.Split(*testBackends, ",")) + } + if *testTests != "" { + conf.filterTests(strings.Split(*testTests, ",")) + } + + // Just clean the directories if required + if *clean { + err := cleanRemotes(conf.Remotes()) + if err != nil { + log.Fatalf("Failed to clean: %v", err) + } + return + } + var names []string - for _, remote := range remotes { - names = append(names, remote.Name) + for _, remote := range conf.Backends { + names = append(names, remote.Remote) } log.Printf("Testing remotes: %s", strings.Join(names, ", ")) - start := time.Now() - if *clean { - config.LoadConfig() - packages = []string{"clean"} - } else { - for _, pkg := range packages { - makeTestBinary(pkg) - defer removeTestBinary(pkg) + // Runs we will do for this test + runs := conf.MakeRuns() + + // Create Report + report := NewReport() + + // Make the test binaries, one per Path found in the tests + done := map[string]struct{}{} + for _, run := range runs { + if _, found := done[run.Path]; !found { + done[run.Path] = struct{}{} + run.MakeTestBinary() + defer run.RemoveTestBinary() } } @@ -443,46 +118,26 @@ func main() { _ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m") // start the tests - results := make(chan *test, 8) + results := make(chan *Run, 8) awaiting := 0 - bools := []bool{false, true} - if *clean { - // Don't run -subdir and -fast-list if -clean - bools = bools[:1] - } - for _, pkg := range packages { - for _, remote := range remotes { - for _, subdir := range bools { - for _, fastlist := range bools { - if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) { - go newTest(pkg, remote.Name, subdir, fastlist).run(results) - awaiting++ - } - } - } - } + for _, run := range runs { + go run.Run(report.LogDir, results) + awaiting++ } // Wait for the tests to finish - var failed []*test for ; awaiting > 0; awaiting-- { t := <-results - if !t.passed() { - failed = append(failed, t) - } + report.RecordResult(t) } - duration := time.Since(start) - // Summarise results - log.Printf("SUMMARY") - if len(failed) == 0 { - log.Printf("PASS: All tests finished OK in %v", duration) - } else { - log.Printf("FAIL: %d tests failed in %v", len(failed), duration) - for _, t := range failed { - log.Printf(" * %s", toShell(t.nextCmdLine())) - log.Printf(" * Failed tests: %v", t.failedTests) - } + // Log and exit + report.End() + report.LogSummary() + report.LogHTML() + report.EmailHTML() + report.Upload() + if !report.AllPassed() { os.Exit(1) } }