Move rclonetest into go tests for fs module

This commit is contained in:
Nick Craig-Wood 2014-08-01 17:58:39 +01:00
parent d3c16608e4
commit 18439cf2d7
10 changed files with 428 additions and 382 deletions

View File

@ -8,7 +8,7 @@ rclone:
test: rclone test: rclone
go test ./... go test ./...
rclonetest/test.sh fs/test_all.sh
doc: rclone.1 README.html README.txt doc: rclone.1 README.html README.txt

View File

@ -4,6 +4,7 @@ package fs
import ( import (
"fmt" "fmt"
"io"
"sync" "sync"
) )
@ -430,9 +431,9 @@ func ListFn(f Fs, fn func(Object)) error {
// Shows size and path // Shows size and path
// //
// Lists in parallel which may get them out of order // Lists in parallel which may get them out of order
func List(f Fs) error { func List(f Fs, w io.Writer) error {
return ListFn(f, func(o Object) { return ListFn(f, func(o Object) {
fmt.Printf("%9d %s\n", o.Size(), o.Remote()) fmt.Fprintf(w, "%9d %s\n", o.Size(), o.Remote())
}) })
} }
@ -441,12 +442,12 @@ func List(f Fs) error {
// Shows size, mod time and path // Shows size, mod time and path
// //
// Lists in parallel which may get them out of order // Lists in parallel which may get them out of order
func ListLong(f Fs) error { func ListLong(f Fs, w io.Writer) error {
return ListFn(f, func(o Object) { return ListFn(f, func(o Object) {
Stats.Checking(o) Stats.Checking(o)
modTime := o.ModTime() modTime := o.ModTime()
Stats.DoneChecking(o) Stats.DoneChecking(o)
fmt.Printf("%9d %19s %s\n", o.Size(), modTime.Format("2006-01-02 15:04:05.00000000"), o.Remote()) fmt.Fprintf(w, "%9d %s %s\n", o.Size(), modTime.Format("2006-01-02 15:04:05.000000000"), o.Remote())
}) })
} }
@ -455,7 +456,7 @@ func ListLong(f Fs) error {
// Produces the same output as the md5sum command // Produces the same output as the md5sum command
// //
// Lists in parallel which may get them out of order // Lists in parallel which may get them out of order
func Md5sum(f Fs) error { func Md5sum(f Fs, w io.Writer) error {
return ListFn(f, func(o Object) { return ListFn(f, func(o Object) {
Stats.Checking(o) Stats.Checking(o)
md5sum, err := o.Md5sum() md5sum, err := o.Md5sum()
@ -464,14 +465,14 @@ func Md5sum(f Fs) error {
Debug(o, "Failed to read MD5: %v", err) Debug(o, "Failed to read MD5: %v", err)
md5sum = "UNKNOWN" md5sum = "UNKNOWN"
} }
fmt.Printf("%32s %s\n", md5sum, o.Remote()) fmt.Fprintf(w, "%32s %s\n", md5sum, o.Remote())
}) })
} }
// List the directories/buckets/containers in the Fs to stdout // List the directories/buckets/containers in the Fs to stdout
func ListDir(f Fs) error { func ListDir(f Fs, w io.Writer) error {
for dir := range f.ListDir() { for dir := range f.ListDir() {
fmt.Printf("%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name) fmt.Fprintf(w, "%12d %13s %9d %s\n", dir.Bytes, dir.When.Format("2006-01-02 15:04:05"), dir.Count, dir.Name)
} }
return nil return nil
} }

332
fs/operations_test.go Normal file
View File

@ -0,0 +1,332 @@
// Test rclone by doing real transactions to a storage provider to and
// from the local disk
package fs_test
import (
"bytes"
"flag"
"io/ioutil"
"log"
"os"
"path"
"regexp"
"strings"
"testing"
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest"
// Active file systems
_ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/googlecloudstorage"
_ "github.com/ncw/rclone/local"
_ "github.com/ncw/rclone/s3"
_ "github.com/ncw/rclone/swift"
)
// Globals
var (
localName, remoteName string
flocal, fremote fs.Fs
RemoteName = flag.String("remote", "", "Remote to test with, defaults to local filesystem")
SubDir = flag.Bool("subdir", false, "Set to test with a sub directory")
finalise func()
)
// Write a file
func WriteFile(filePath, content string, t time.Time) {
// FIXME make directories?
filePath = path.Join(localName, filePath)
dirPath := path.Dir(filePath)
err := os.MkdirAll(dirPath, 0770)
if err != nil {
log.Fatalf("Failed to make directories %q: %v", dirPath, err)
}
err = ioutil.WriteFile(filePath, []byte(content), 0600)
if err != nil {
log.Fatalf("Failed to write file %q: %v", filePath, err)
}
err = os.Chtimes(filePath, t, t)
if err != nil {
log.Fatalf("Failed to chtimes file %q: %v", filePath, err)
}
}
var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
var t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")
var t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")
func TestInit(t *testing.T) {
fs.LoadConfig()
fs.Config.Verbose = false
fs.Config.Quiet = true
var err error
fremote, finalise, err = fstest.RandomRemote(*RemoteName, *SubDir)
if err != nil {
t.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
}
t.Logf("Testing with remote %v", fremote)
localName, err = ioutil.TempDir("", "rclone")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
t.Logf("Testing with local %q", localName)
flocal, err = fs.NewFs(localName)
if err != nil {
t.Fatalf("Failed to make %q: %v", remoteName, err)
}
}
func TestCalculateModifyWindow(t *testing.T) {
fs.CalculateModifyWindow(fremote, flocal)
}
func TestMkdir(t *testing.T) {
fstest.TestMkdir(t, fremote)
}
// Check dry run is working
func TestCopyWithDryRun(t *testing.T) {
WriteFile("sub dir/hello world", "hello world", t1)
fs.Config.DryRun = true
err := fs.Sync(fremote, flocal, false)
fs.Config.DryRun = false
if err != nil {
t.Fatalf("Copy failed: %v", err)
}
items := []fstest.Item{
{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
}
fstest.CheckListing(t, flocal, items)
fstest.CheckListing(t, fremote, []fstest.Item{})
}
// Now without dry run
func TestCopy(t *testing.T) {
err := fs.Sync(fremote, flocal, false)
if err != nil {
t.Fatalf("Copy failed: %v", err)
}
items := []fstest.Item{
{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
}
fstest.CheckListing(t, flocal, items)
fstest.CheckListing(t, fremote, items)
}
func TestLsd(t *testing.T) {
var buf bytes.Buffer
err := fs.ListDir(fremote, &buf)
if err != nil {
t.Fatalf("ListDir failed: %v", err)
}
res := buf.String()
if !strings.Contains(res, "sub dir\n") {
t.Fatalf("Result wrong %q", res)
}
}
// Now delete the local file and download it
func TestCopyAfterDelete(t *testing.T) {
err := os.Remove(localName + "/sub dir/hello world")
if err != nil {
t.Fatalf("Remove failed: %v", err)
}
items := []fstest.Item{
{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
}
fstest.CheckListing(t, flocal, []fstest.Item{})
fstest.CheckListing(t, fremote, items)
}
func TestCopyRedownload(t *testing.T) {
err := fs.Sync(flocal, fremote, false)
if err != nil {
t.Fatalf("Copy failed: %v", err)
}
items := []fstest.Item{
{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
}
fstest.CheckListingWithPrecision(t, flocal, items, fremote.Precision())
fstest.CheckListing(t, fremote, items)
// Clean the directory
cleanTempDir(t)
}
func TestSyncAfterChangingModtimeOnly(t *testing.T) {
WriteFile("empty space", "", t1)
err := os.Chtimes(localName+"/empty space", t2, t2)
if err != nil {
t.Fatalf("Chtimes failed: %v", err)
}
err = fs.Sync(fremote, flocal, true)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
}
fstest.CheckListing(t, flocal, items)
fstest.CheckListing(t, fremote, items)
}
func TestSyncAfterAddingAFile(t *testing.T) {
WriteFile("potato", "------------------------------------------------------------", t3)
err := fs.Sync(fremote, flocal, true)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListing(t, flocal, items)
fstest.CheckListing(t, fremote, items)
}
func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
WriteFile("potato", "smaller but same date", t3)
err := fs.Sync(fremote, flocal, true)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
}
fstest.CheckListing(t, flocal, items)
fstest.CheckListing(t, fremote, items)
}
// Sync after changing a file's contents, modtime but not length
func TestSyncAfterChangingContentsOnly(t *testing.T) {
WriteFile("potato", "SMALLER BUT SAME DATE", t2)
err := fs.Sync(fremote, flocal, true)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
}
fstest.CheckListing(t, flocal, items)
fstest.CheckListing(t, fremote, items)
}
// Sync after removing a file and adding a file --dry-run
func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
WriteFile("potato2", "------------------------------------------------------------", t1)
err := os.Remove(localName + "/potato")
if err != nil {
t.Fatalf("Remove failed: %v", err)
}
fs.Config.DryRun = true
err = fs.Sync(fremote, flocal, true)
fs.Config.DryRun = false
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
before := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListing(t, flocal, items)
fstest.CheckListing(t, fremote, before)
}
// Sync after removing a file and adding a file
func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
err := fs.Sync(fremote, flocal, true)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListing(t, flocal, items)
fstest.CheckListing(t, fremote, items)
}
func TestLs(t *testing.T) {
var buf bytes.Buffer
err := fs.List(fremote, &buf)
if err != nil {
t.Fatalf("List failed: %v", err)
}
res := buf.String()
if !strings.Contains(res, " 0 empty space\n") {
t.Errorf("empty space missing: %q", res)
}
if !strings.Contains(res, " 60 potato2\n") {
t.Errorf("potato2 missing: %q", res)
}
}
func TestLsLong(t *testing.T) {
var buf bytes.Buffer
err := fs.ListLong(fremote, &buf)
if err != nil {
t.Fatalf("List failed: %v", err)
}
res := buf.String()
m1 := regexp.MustCompile(`(?m)^ 0 2011-12-25 12:59:59\.\d{9} empty space$`)
if !m1.MatchString(res) {
t.Errorf("empty space missing: %q", res)
}
m2 := regexp.MustCompile(`(?m)^ 60 2001-02-03 04:05:06\.\d{9} potato2$`)
if !m2.MatchString(res) {
t.Errorf("potato2 missing: %q", res)
}
}
func TestMd5sum(t *testing.T) {
var buf bytes.Buffer
err := fs.Md5sum(fremote, &buf)
if err != nil {
t.Fatalf("List failed: %v", err)
}
res := buf.String()
if !strings.Contains(res, "d41d8cd98f00b204e9800998ecf8427e empty space\n") {
t.Errorf("empty space missing: %q", res)
}
if !strings.Contains(res, "6548b156ea68a4e003e786df99eee76 potato2\n") {
t.Errorf("potato2 missing: %q", res)
}
}
func TestCheck(t *testing.T) {
}
// Clean the temporary directory
func cleanTempDir(t *testing.T) {
t.Logf("Cleaning temporary directory: %q", localName)
err := os.RemoveAll(localName)
if err != nil {
t.Logf("Failed to remove %q: %v", localName, err)
}
}
func TestFinalise(t *testing.T) {
finalise()
cleanTempDir(t)
}

29
fs/test_all.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
go install
REMOTES="
TestSwift:
TestS3:
TestDrive:
TestGoogleCloudStorage:
TestDropbox:
"
function test_remote {
args=$@
echo "@go test $args"
go test $args || {
echo "*** test $args FAILED ***"
exit 1
}
}
test_remote
test_remote --subdir
for remote in $REMOTES; do
test_remote --remote $remote
test_remote --remote $remote --subdir
done
echo "All OK"

View File

@ -10,13 +10,12 @@ import (
"math/rand" "math/rand"
"os" "os"
"strings" "strings"
"testing"
"time" "time"
"github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs"
) )
var Fatalf = log.Fatalf
// Seed the random number generator // Seed the random number generator
func init() { func init() {
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
@ -32,30 +31,29 @@ type Item struct {
} }
// check the mod time to the given precision // check the mod time to the given precision
func (i *Item) CheckModTime(obj fs.Object, modTime time.Time) { func (i *Item) CheckModTime(t *testing.T, obj fs.Object, modTime time.Time, precision time.Duration) {
dt := modTime.Sub(i.ModTime) dt := modTime.Sub(i.ModTime)
precision := obj.Fs().Precision()
if dt >= precision || dt <= -precision { if dt >= precision || dt <= -precision {
Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, precision, modTime, i.ModTime) t.Errorf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", obj.Remote(), dt, precision, modTime, i.ModTime, precision)
} }
} }
func (i *Item) Check(obj fs.Object) { func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) {
if obj == nil { if obj == nil {
Fatalf("Object is nil") t.Fatalf("Object is nil")
} }
// Check attributes // Check attributes
Md5sum, err := obj.Md5sum() Md5sum, err := obj.Md5sum()
if err != nil { if err != nil {
Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err) t.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
} }
if i.Md5sum != Md5sum { if i.Md5sum != Md5sum {
Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum) t.Errorf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
} }
if i.Size != obj.Size() { if i.Size != obj.Size() {
Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size()) t.Errorf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
} }
i.CheckModTime(obj, obj.ModTime()) i.CheckModTime(t, obj, obj.ModTime(), precision)
} }
// Represents all items for checking // Represents all items for checking
@ -78,39 +76,45 @@ func NewItems(items []Item) *Items {
} }
// Check off an item // Check off an item
func (is *Items) Find(obj fs.Object) { func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) {
i, ok := is.byName[obj.Remote()] i, ok := is.byName[obj.Remote()]
if !ok { if !ok {
Fatalf("Unexpected file %q", obj.Remote()) t.Errorf("Unexpected file %q", obj.Remote())
} }
delete(is.byName, obj.Remote()) delete(is.byName, obj.Remote())
i.Check(obj) i.Check(t, obj, precision)
} }
// Check all done // Check all done
func (is *Items) Done() { func (is *Items) Done(t *testing.T) {
if len(is.byName) != 0 { if len(is.byName) != 0 {
for name := range is.byName { for name := range is.byName {
log.Printf("Not found %q", name) log.Printf("Not found %q", name)
} }
Fatalf("%d objects not found", len(is.byName)) t.Errorf("%d objects not found", len(is.byName))
} }
} }
// Checks the fs to see if it has the expected contents // Checks the fs to see if it has the expected contents
func CheckListing(f fs.Fs, items []Item) { func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, precision time.Duration) {
is := NewItems(items) is := NewItems(items)
for obj := range f.List() { for obj := range f.List() {
is.Find(obj) is.Find(t, obj, precision)
} }
is.Done() is.Done(t)
}
// Checks the fs to see if it has the expected contents
func CheckListing(t *testing.T, f fs.Fs, items []Item) {
precision := f.Precision()
CheckListingWithPrecision(t, f, items, precision)
} }
// Parse a time string or explode // Parse a time string or explode
func Time(timeString string) time.Time { func Time(timeString string) time.Time {
t, err := time.Parse(time.RFC3339Nano, timeString) t, err := time.Parse(time.RFC3339Nano, timeString)
if err != nil { if err != nil {
Fatalf("Failed to parse time %q: %v", timeString, err) log.Fatalf("Failed to parse time %q: %v", timeString, err)
} }
return t return t
} }
@ -197,25 +201,25 @@ func RandomRemote(remoteName string, subdir bool) (fs.Fs, func(), error) {
return remote, finalise, nil return remote, finalise, nil
} }
func TestMkdir(remote fs.Fs) { func TestMkdir(t *testing.T, remote fs.Fs) {
err := fs.Mkdir(remote) err := fs.Mkdir(remote)
if err != nil { if err != nil {
Fatalf("Mkdir failed: %v", err) t.Fatalf("Mkdir failed: %v", err)
} }
CheckListing(remote, []Item{}) CheckListing(t, remote, []Item{})
} }
func TestPurge(remote fs.Fs) { func TestPurge(t *testing.T, remote fs.Fs) {
err := fs.Purge(remote) err := fs.Purge(remote)
if err != nil { if err != nil {
Fatalf("Purge failed: %v", err) t.Fatalf("Purge failed: %v", err)
} }
CheckListing(remote, []Item{}) CheckListing(t, remote, []Item{})
} }
func TestRmdir(remote fs.Fs) { func TestRmdir(t *testing.T, remote fs.Fs) {
err := fs.Rmdir(remote) err := fs.Rmdir(remote)
if err != nil { if err != nil {
Fatalf("Rmdir failed: %v", err) t.Fatalf("Rmdir failed: %v", err)
} }
} }

View File

@ -1,8 +1,6 @@
// Generic tests for testing the Fs and Object interfaces // Generic tests for testing the Fs and Object interfaces
package fstests package fstests
// FIXME need to check the limited file system
import ( import (
"bytes" "bytes"
"crypto/md5" "crypto/md5"
@ -58,12 +56,10 @@ func TestInit(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Couldn't start FS: %v", err) t.Fatalf("Couldn't start FS: %v", err)
} }
fstest.Fatalf = t.Fatalf fstest.TestMkdir(t, remote)
fstest.TestMkdir(remote)
} }
func skipIfNotOk(t *testing.T) { func skipIfNotOk(t *testing.T) {
fstest.Fatalf = t.Fatalf
if remote == nil { if remote == nil {
t.Skip("FS not configured") t.Skip("FS not configured")
} }
@ -88,7 +84,7 @@ type TestFile struct {
func TestFsRmdirEmpty(t *testing.T) { func TestFsRmdirEmpty(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.TestRmdir(remote) fstest.TestRmdir(t, remote)
} }
func TestFsRmdirNotFound(t *testing.T) { func TestFsRmdirNotFound(t *testing.T) {
@ -101,13 +97,13 @@ func TestFsRmdirNotFound(t *testing.T) {
func TestFsMkdir(t *testing.T) { func TestFsMkdir(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.TestMkdir(remote) fstest.TestMkdir(t, remote)
fstest.TestMkdir(remote) fstest.TestMkdir(t, remote)
} }
func TestFsListEmpty(t *testing.T) { func TestFsListEmpty(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.CheckListing(remote, []fstest.Item{}) fstest.CheckListing(t, remote, []fstest.Item{})
} }
func TestFsListDirEmpty(t *testing.T) { func TestFsListDirEmpty(t *testing.T) {
@ -143,10 +139,10 @@ func testPut(t *testing.T, file *fstest.Item) {
t.Fatal("Put error", err) t.Fatal("Put error", err)
} }
file.Md5sum = hex.EncodeToString(hash.Sum(nil)) file.Md5sum = hex.EncodeToString(hash.Sum(nil))
file.Check(obj) file.Check(t, obj, remote.Precision())
// Re-read the object and check again // Re-read the object and check again
obj = findObject(t, file.Path) obj = findObject(t, file.Path)
file.Check(obj) file.Check(t, obj, remote.Precision())
} }
func TestFsPutFile1(t *testing.T) { func TestFsPutFile1(t *testing.T) {
@ -231,18 +227,18 @@ func TestFsListRoot(t *testing.T) {
func TestFsListFile1(t *testing.T) { func TestFsListFile1(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.CheckListing(remote, []fstest.Item{file1, file2}) fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
} }
func TestFsNewFsObject(t *testing.T) { func TestFsNewFsObject(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
file1.Check(obj) file1.Check(t, obj, remote.Precision())
} }
func TestFsListFile1and2(t *testing.T) { func TestFsListFile1and2(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.CheckListing(remote, []fstest.Item{file1, file2}) fstest.CheckListing(t, remote, []fstest.Item{file1, file2})
} }
func TestFsRmdirFull(t *testing.T) { func TestFsRmdirFull(t *testing.T) {
@ -307,7 +303,7 @@ func TestObjectMd5sum(t *testing.T) {
func TestObjectModTime(t *testing.T) { func TestObjectModTime(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
file1.CheckModTime(obj, obj.ModTime()) file1.CheckModTime(t, obj, obj.ModTime(), remote.Precision())
} }
func TestObjectSetModTime(t *testing.T) { func TestObjectSetModTime(t *testing.T) {
@ -316,7 +312,7 @@ func TestObjectSetModTime(t *testing.T) {
obj := findObject(t, file1.Path) obj := findObject(t, file1.Path)
obj.SetModTime(newModTime) obj.SetModTime(newModTime)
file1.ModTime = newModTime file1.ModTime = newModTime
file1.CheckModTime(obj, newModTime) file1.CheckModTime(t, obj, newModTime, remote.Precision())
// And make a new object and read it from there too // And make a new object and read it from there too
TestObjectModTime(t) TestObjectModTime(t)
} }
@ -367,10 +363,10 @@ func TestObjectUpdate(t *testing.T) {
t.Fatal("Update error", err) t.Fatal("Update error", err)
} }
file1.Md5sum = hex.EncodeToString(hash.Sum(nil)) file1.Md5sum = hex.EncodeToString(hash.Sum(nil))
file1.Check(obj) file1.Check(t, obj, remote.Precision())
// Re-read the object and check again // Re-read the object and check again
obj = findObject(t, file1.Path) obj = findObject(t, file1.Path)
file1.Check(obj) file1.Check(t, obj, remote.Precision())
} }
func TestObjectStorable(t *testing.T) { func TestObjectStorable(t *testing.T) {
@ -390,7 +386,7 @@ func TestLimitedFs(t *testing.T) {
if err != nil { if err != nil {
t.Fatal("Failed to make remote %q: %v", remoteName, err) t.Fatal("Failed to make remote %q: %v", remoteName, err)
} }
fstest.CheckListing(fileRemote, []fstest.Item{file2Copy}) fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy})
_, ok := fileRemote.(*fs.Limited) _, ok := fileRemote.(*fs.Limited)
if !ok { if !ok {
t.Errorf("%v is not a fs.Limited", fileRemote) t.Errorf("%v is not a fs.Limited", fileRemote)
@ -404,7 +400,7 @@ func TestLimitedFsNotFound(t *testing.T) {
if err != nil { if err != nil {
t.Fatal("Failed to make remote %q: %v", remoteName, err) t.Fatal("Failed to make remote %q: %v", remoteName, err)
} }
fstest.CheckListing(fileRemote, []fstest.Item{}) fstest.CheckListing(t, fileRemote, []fstest.Item{})
_, ok := fileRemote.(*fs.Limited) _, ok := fileRemote.(*fs.Limited)
if ok { if ok {
t.Errorf("%v is is a fs.Limited", fileRemote) t.Errorf("%v is is a fs.Limited", fileRemote)
@ -418,12 +414,12 @@ func TestObjectRemove(t *testing.T) {
if err != nil { if err != nil {
t.Fatal("Remove error", err) t.Fatal("Remove error", err)
} }
fstest.CheckListing(remote, []fstest.Item{file2}) fstest.CheckListing(t, remote, []fstest.Item{file2})
} }
func TestObjectPurge(t *testing.T) { func TestObjectPurge(t *testing.T) {
skipIfNotOk(t) skipIfNotOk(t)
fstest.TestPurge(remote) fstest.TestPurge(t, remote)
err := fs.Purge(remote) err := fs.Purge(remote)
if err == nil { if err == nil {
t.Fatal("Expecting error after on second purge") t.Fatal("Expecting error after on second purge")

View File

@ -3,14 +3,13 @@ Change lsd command so it doesn't show -1
* Make test? * Make test?
Put the TestRemote names into the Fs description Put the TestRemote names into the Fs description
Make rclonetest use the TestRemote name automatically Make test_all.sh use the TestRemote name automatically
Put rclonetest back into rclone as tests
* defaults to using local remote
* but could pass another in with a flag
Run errcheck and go vet in the make file Run errcheck and go vet in the make file
.. Also race detector? .. Also race detector?
Get rid of Storable?
Write developer manual Write developer manual
Todo Todo

View File

@ -96,7 +96,7 @@ var Commands = []Command{
Help: ` Help: `
List all the objects in the the path with size and path.`, List all the objects in the the path with size and path.`,
Run: func(fdst, fsrc fs.Fs) { Run: func(fdst, fsrc fs.Fs) {
err := fs.List(fdst) err := fs.List(fdst, os.Stdout)
if err != nil { if err != nil {
log.Fatalf("Failed to list: %v", err) log.Fatalf("Failed to list: %v", err)
} }
@ -110,7 +110,7 @@ var Commands = []Command{
Help: ` Help: `
List all directories/containers/buckets in the the path.`, List all directories/containers/buckets in the the path.`,
Run: func(fdst, fsrc fs.Fs) { Run: func(fdst, fsrc fs.Fs) {
err := fs.ListDir(fdst) err := fs.ListDir(fdst, os.Stdout)
if err != nil { if err != nil {
log.Fatalf("Failed to listdir: %v", err) log.Fatalf("Failed to listdir: %v", err)
} }
@ -124,7 +124,7 @@ var Commands = []Command{
Help: ` Help: `
List all the objects in the the path with modification time, size and path.`, List all the objects in the the path with modification time, size and path.`,
Run: func(fdst, fsrc fs.Fs) { Run: func(fdst, fsrc fs.Fs) {
err := fs.ListLong(fdst) err := fs.ListLong(fdst, os.Stdout)
if err != nil { if err != nil {
log.Fatalf("Failed to list long: %v", err) log.Fatalf("Failed to list long: %v", err)
} }
@ -138,7 +138,7 @@ var Commands = []Command{
Help: ` Help: `
Produces an md5sum file for all the objects in the path.`, Produces an md5sum file for all the objects in the path.`,
Run: func(fdst, fsrc fs.Fs) { Run: func(fdst, fsrc fs.Fs) {
err := fs.Md5sum(fdst) err := fs.Md5sum(fdst, os.Stdout)
if err != nil { if err != nil {
log.Fatalf("Failed to list: %v", err) log.Fatalf("Failed to list: %v", err)
} }

View File

@ -1,290 +0,0 @@
// Test rclone by doing real transactions to a storage provider to and
// from the local disk
package main
import (
"fmt"
"io/ioutil"
"log"
"os"
"path"
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest"
"github.com/ogier/pflag"
// Active file systems
_ "github.com/ncw/rclone/drive"
_ "github.com/ncw/rclone/dropbox"
_ "github.com/ncw/rclone/googlecloudstorage"
_ "github.com/ncw/rclone/local"
_ "github.com/ncw/rclone/s3"
_ "github.com/ncw/rclone/swift"
)
// Globals
var (
localName, remoteName string
version = pflag.BoolP("version", "V", false, "Print the version number")
subDir = pflag.BoolP("subdir", "S", false, "Test with a sub directory")
)
// Write a file
func WriteFile(filePath, content string, t time.Time) {
// FIXME make directories?
filePath = path.Join(localName, filePath)
dirPath := path.Dir(filePath)
err := os.MkdirAll(dirPath, 0770)
if err != nil {
log.Fatalf("Failed to make directories %q: %v", dirPath, err)
}
err = ioutil.WriteFile(filePath, []byte(content), 0600)
if err != nil {
log.Fatalf("Failed to write file %q: %v", filePath, err)
}
err = os.Chtimes(filePath, t, t)
if err != nil {
log.Fatalf("Failed to chtimes file %q: %v", filePath, err)
}
}
var t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
var t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")
var t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")
func TestCopy(flocal, fremote fs.Fs) {
WriteFile("sub dir/hello world", "hello world", t1)
// Check dry run is working
log.Printf("Copy with --dry-run")
fs.Config.DryRun = true
err := fs.Sync(fremote, flocal, false)
fs.Config.DryRun = false
if err != nil {
log.Fatalf("Copy failed: %v", err)
}
items := []fstest.Item{
{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
}
fstest.CheckListing(flocal, items)
fstest.CheckListing(fremote, []fstest.Item{})
// Now without dry run
log.Printf("Copy")
err = fs.Sync(fremote, flocal, false)
if err != nil {
log.Fatalf("Copy failed: %v", err)
}
fstest.CheckListing(flocal, items)
fstest.CheckListing(fremote, items)
// Now delete the local file and download it
err = os.Remove(localName + "/sub dir/hello world")
if err != nil {
log.Fatalf("Remove failed: %v", err)
}
fstest.CheckListing(flocal, []fstest.Item{})
fstest.CheckListing(fremote, items)
log.Printf("Copy - redownload")
err = fs.Sync(flocal, fremote, false)
if err != nil {
log.Fatalf("Copy failed: %v", err)
}
fstest.CheckListing(flocal, items)
fstest.CheckListing(fremote, items)
// Clean the directory
cleanTempDir()
}
func TestSync(flocal, fremote fs.Fs) {
WriteFile("empty space", "", t1)
log.Printf("Sync after changing file modtime only")
err := os.Chtimes(localName+"/empty space", t2, t2)
if err != nil {
log.Fatalf("Chtimes failed: %v", err)
}
err = fs.Sync(fremote, flocal, true)
if err != nil {
log.Fatalf("Sync failed: %v", err)
}
items := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
}
fstest.CheckListing(flocal, items)
fstest.CheckListing(fremote, items)
// ------------------------------------------------------------
log.Printf("Sync after adding a file")
WriteFile("potato", "------------------------------------------------------------", t3)
err = fs.Sync(fremote, flocal, true)
if err != nil {
log.Fatalf("Sync failed: %v", err)
}
items = []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 60, ModTime: t3, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListing(flocal, items)
fstest.CheckListing(fremote, items)
// ------------------------------------------------------------
log.Printf("Sync after changing a file's size only")
WriteFile("potato", "smaller but same date", t3)
err = fs.Sync(fremote, flocal, true)
if err != nil {
log.Fatalf("Sync failed: %v", err)
}
items = []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 21, ModTime: t3, Md5sum: "100defcf18c42a1e0dc42a789b107cd2"},
}
fstest.CheckListing(flocal, items)
fstest.CheckListing(fremote, items)
// ------------------------------------------------------------
log.Printf("Sync after changing a file's contents, modtime but not length")
WriteFile("potato", "SMALLER BUT SAME DATE", t2)
err = fs.Sync(fremote, flocal, true)
if err != nil {
log.Fatalf("Sync failed: %v", err)
}
items = []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
}
fstest.CheckListing(flocal, items)
fstest.CheckListing(fremote, items)
// ------------------------------------------------------------
log.Printf("Sync after removing a file and adding a file --dry-run")
WriteFile("potato2", "------------------------------------------------------------", t1)
err = os.Remove(localName + "/potato")
if err != nil {
log.Fatalf("Remove failed: %v", err)
}
fs.Config.DryRun = true
err = fs.Sync(fremote, flocal, true)
fs.Config.DryRun = false
if err != nil {
log.Fatalf("Sync failed: %v", err)
}
before := []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato", Size: 21, ModTime: t2, Md5sum: "e4cb6955d9106df6263c45fcfc10f163"},
}
items = []fstest.Item{
{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
}
fstest.CheckListing(flocal, items)
fstest.CheckListing(fremote, before)
log.Printf("Sync after removing a file and adding a file")
err = fs.Sync(fremote, flocal, true)
if err != nil {
log.Fatalf("Sync failed: %v", err)
}
fstest.CheckListing(flocal, items)
fstest.CheckListing(fremote, items)
}
func TestLs(flocal, fremote fs.Fs) {
// Underlying List has been tested above, so we just make sure it runs
err := fs.List(fremote)
if err != nil {
log.Fatalf("List failed: %v", err)
}
}
func TestLsd(flocal, fremote fs.Fs) {
}
func TestCheck(flocal, fremote fs.Fs) {
}
func syntaxError() {
fmt.Fprintf(os.Stderr, `Test rclone with a remote to find bugs in either - %s.
Syntax: [options] remote:
Need a remote: as argument. This will create a random container or
directory under it and perform tests on it, deleting it at the end.
Options:
`, fs.Version)
pflag.PrintDefaults()
}
// Clean the temporary directory
func cleanTempDir() {
log.Printf("Cleaning temporary directory: %q", localName)
err := os.RemoveAll(localName)
if err != nil {
log.Printf("Failed to remove %q: %v", localName, err)
}
}
func main() {
pflag.Usage = syntaxError
pflag.Parse()
if *version {
fmt.Printf("rclonetest %s\n", fs.Version)
os.Exit(0)
}
fs.LoadConfig()
args := pflag.Args()
if len(args) != 1 {
syntaxError()
os.Exit(1)
}
fremote, finalise, err := fstest.RandomRemote(args[0], *subDir)
if err != nil {
log.Fatalf("Failed to open remote %q: %v", args[0], err)
}
log.Printf("Testing with remote %v", fremote)
localName, err = ioutil.TempDir("", "rclone")
if err != nil {
log.Fatalf("Failed to create temp dir: %v", err)
}
log.Printf("Testing with local %q", localName)
flocal, err := fs.NewFs(localName)
if err != nil {
log.Fatalf("Failed to make %q: %v", remoteName, err)
}
fs.CalculateModifyWindow(fremote, flocal)
fstest.TestMkdir(fremote)
TestCopy(flocal, fremote)
TestSync(flocal, fremote)
TestLs(flocal, fremote)
TestLsd(flocal, fremote)
TestCheck(flocal, fremote)
//TestRmdir(flocal, fremote)
finalise()
cleanTempDir()
log.Printf("Tests OK")
}

View File

@ -1,25 +0,0 @@
#!/bin/bash
go install
REMOTES="
TestSwift:
TestS3:
TestDrive:
TestGoogleCloudStorage:
TestDropbox:
/tmp/z
"
function test_remote {
args=$@
rclonetest $args || {
echo "*** rclonetest $args FAILED ***"
exit 1
}
}
for remote in $REMOTES; do
test_remote $remote
test_remote --subdir $remote
done