Fix integration tests so they can be run independently and out of order - fixes #291

* Make all integration tests start with an empty remote
  * Add an -individual flag so this can be a different bucket/container/directory
  * Fix up tests after changing the hashers
  * Add sha1sum test
  * Make directory checking in tests sleep more to fix acd inconsistencies
  * Factor integration tests to make more maintainable
  * Ensure remote writes have a fstest.CheckItems() before use
    * this fixes eventual consistency on the directory listings later
  * Call fs.Stats.ResetCounters() before every fs.Sync()

Note that the tests shouldn't be run concurrently as fs.Config is global state.

fstest/fstest.go

@@ -4,6 +4,8 @@ package fstest
 // FIXME put name of test FS in Fs structure
 
 import (
+	"bytes"
+	"io"
 	"io/ioutil"
 	"log"
 	"math/rand"
@@ -31,6 +33,23 @@ type Item struct {
 	WinPath string
 }
 
+// NewItem creates an item from a string content
+func NewItem(Path, Content string, modTime time.Time) Item {
+	i := Item{
+		Path:    Path,
+		ModTime: modTime,
+		Size:    int64(len(Content)),
+	}
+	hash := fs.NewMultiHasher()
+	buf := bytes.NewBufferString(Content)
+	_, err := io.Copy(hash, buf)
+	if err != nil {
+		log.Fatalf("Failed to create item: %v", err)
+	}
+	i.Hashes = hash.Sums()
+	return i
+}
+
 // CheckTimeEqualWithPrecision checks the times are equal within the
 // precision, returns the delta and a flag
 func CheckTimeEqualWithPrecision(t0, t1 time.Time, precision time.Duration) (time.Duration, bool) {
@@ -129,17 +148,26 @@ func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision ti
 	is := NewItems(items)
 	oldErrors := fs.Stats.GetErrors()
 	var objs []fs.Object
-	const retries = 10
+	const retries = 6
+	sleep := time.Second / 2
 	for i := 1; i <= retries; i++ {
 		objs = nil
 		for obj := range f.List() {
 			objs = append(objs, obj)
 		}
 		if len(objs) == len(items) {
+			// Put an extra sleep in if we did any retries just to make sure it really
+			// is consistent (here is looking at you Amazon Cloud Drive!)
+			if i != 1 {
+				extraSleep := 5*time.Second + sleep
+				t.Logf("Sleeping for %v just to make sure", extraSleep)
+				time.Sleep(extraSleep)
+			}
 			break
 		}
-		t.Logf("Sleeping for 1 second for list eventual consistency: %d/%d", i, retries)
-		time.Sleep(1 * time.Second)
+		sleep *= 2
+		t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)
+		time.Sleep(sleep)
 	}
 	for _, obj := range objs {
 		if obj == nil {
@@ -161,6 +189,12 @@ func CheckListing(t *testing.T, f fs.Fs, items []Item) {
 	CheckListingWithPrecision(t, f, items, precision)
 }
 
+// CheckItems checks the fs to see if it has only the items passed in
+// using a precision of fs.Config.ModifyWindow
+func CheckItems(t *testing.T, f fs.Fs, items ...Item) {
+	CheckListingWithPrecision(t, f, items, fs.Config.ModifyWindow)
+}
+
 // Time parses a time string or logs a fatal error
 func Time(timeString string) time.Time {
 	t, err := time.Parse(time.RFC3339Nano, timeString)