mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 16:34:30 +01:00
march: rework testcases to better reflect real use
This commit is contained in:
buengese
Nick Craig-Wood
parent
6f4b86e569
commit
3a04d0d1a9
@ -3,16 +3,282 @@
|
||||
package march
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/mockdir"
|
||||
"github.com/rclone/rclone/fstest/mockobject"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Some times used in the tests
|
||||
var (
|
||||
t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
fstest.TestMain(m)
|
||||
}
|
||||
|
||||
type marchTester struct {
|
||||
ctx context.Context // internal context for controlling go-routines
|
||||
cancel func() // cancel the context
|
||||
srcOnly fs.DirEntries
|
||||
dstOnly fs.DirEntries
|
||||
match fs.DirEntries
|
||||
entryMutex sync.Mutex
|
||||
errorMu sync.Mutex // Mutex covering the error variables
|
||||
err error
|
||||
noRetryErr error
|
||||
fatalErr error
|
||||
noTraverse bool
|
||||
}
|
||||
|
||||
// DstOnly have an object which is in the destination only
|
||||
func (mt *marchTester) DstOnly(dst fs.DirEntry) (recurse bool) {
|
||||
mt.entryMutex.Lock()
|
||||
mt.dstOnly = append(mt.dstOnly, dst)
|
||||
mt.entryMutex.Unlock()
|
||||
|
||||
switch dst.(type) {
|
||||
case fs.Object:
|
||||
return false
|
||||
case fs.Directory:
|
||||
return true
|
||||
default:
|
||||
panic("Bad object in DirEntries")
|
||||
}
|
||||
}
|
||||
|
||||
// SrcOnly have an object which is in the source only
|
||||
func (mt *marchTester) SrcOnly(src fs.DirEntry) (recurse bool) {
|
||||
mt.entryMutex.Lock()
|
||||
mt.srcOnly = append(mt.srcOnly, src)
|
||||
mt.entryMutex.Unlock()
|
||||
|
||||
switch src.(type) {
|
||||
case fs.Object:
|
||||
return false
|
||||
case fs.Directory:
|
||||
return true
|
||||
default:
|
||||
panic("Bad object in DirEntries")
|
||||
}
|
||||
}
|
||||
|
||||
// Match is called when src and dst are present, so sync src to dst
|
||||
func (mt *marchTester) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) {
|
||||
mt.entryMutex.Lock()
|
||||
mt.match = append(mt.match, src)
|
||||
mt.entryMutex.Unlock()
|
||||
|
||||
switch src.(type) {
|
||||
case fs.Object:
|
||||
return false
|
||||
case fs.Directory:
|
||||
// Do the same thing to the entire contents of the directory
|
||||
_, ok := dst.(fs.Directory)
|
||||
if ok {
|
||||
return true
|
||||
}
|
||||
// FIXME src is dir, dst is file
|
||||
err := errors.New("can't overwrite file with directory")
|
||||
fs.Errorf(dst, "%v", err)
|
||||
mt.processError(err)
|
||||
default:
|
||||
panic("Bad object in DirEntries")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (mt *marchTester) processError(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
mt.errorMu.Lock()
|
||||
defer mt.errorMu.Unlock()
|
||||
switch {
|
||||
case fserrors.IsFatalError(err):
|
||||
if !mt.aborting() {
|
||||
fs.Errorf(nil, "Cancelling sync due to fatal error: %v", err)
|
||||
mt.cancel()
|
||||
}
|
||||
mt.fatalErr = err
|
||||
case fserrors.IsNoRetryError(err):
|
||||
mt.noRetryErr = err
|
||||
default:
|
||||
mt.err = err
|
||||
}
|
||||
}
|
||||
|
||||
func (mt *marchTester) currentError() error {
|
||||
mt.errorMu.Lock()
|
||||
defer mt.errorMu.Unlock()
|
||||
if mt.fatalErr != nil {
|
||||
return mt.fatalErr
|
||||
}
|
||||
if mt.err != nil {
|
||||
return mt.err
|
||||
}
|
||||
return mt.noRetryErr
|
||||
}
|
||||
|
||||
func (mt *marchTester) aborting() bool {
|
||||
return mt.ctx.Err() != nil
|
||||
}
|
||||
|
||||
func TestMarch(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
what string
|
||||
fileSrcOnly []string
|
||||
dirSrcOnly []string
|
||||
fileDstOnly []string
|
||||
dirDstOnly []string
|
||||
fileMatch []string
|
||||
dirMatch []string
|
||||
}{
|
||||
{
|
||||
what: "source only",
|
||||
fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"},
|
||||
dirSrcOnly: []string{"sub dir"},
|
||||
},
|
||||
{
|
||||
what: "identical",
|
||||
fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"},
|
||||
dirMatch: []string{"sub dir", "sub dir/sub sub dir"},
|
||||
},
|
||||
{
|
||||
what: "typical sync",
|
||||
fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"},
|
||||
dirSrcOnly: []string{"srcOnlyDir"},
|
||||
fileMatch: []string{"match", "matchDir/match file"},
|
||||
dirMatch: []string{"matchDir"},
|
||||
fileDstOnly: []string{"dstOnly", "dstOnlyDir/sub"},
|
||||
dirDstOnly: []string{"dstOnlyDir"},
|
||||
},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
var srcOnly []fstest.Item
|
||||
var dstOnly []fstest.Item
|
||||
var match []fstest.Item
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
for _, f := range test.fileSrcOnly {
|
||||
srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1))
|
||||
}
|
||||
for _, f := range test.fileDstOnly {
|
||||
dstOnly = append(dstOnly, r.WriteObject(ctx, f, "hello world", t1))
|
||||
}
|
||||
for _, f := range test.fileMatch {
|
||||
match = append(match, r.WriteBoth(ctx, f, "hello world", t1))
|
||||
}
|
||||
|
||||
mt := &marchTester{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
noTraverse: false,
|
||||
}
|
||||
m := &March{
|
||||
Ctx: ctx,
|
||||
Fdst: r.Fremote,
|
||||
Fsrc: r.Flocal,
|
||||
Dir: "",
|
||||
NoTraverse: mt.noTraverse,
|
||||
Callback: mt,
|
||||
DstIncludeAll: filter.Active.Opt.DeleteExcluded,
|
||||
}
|
||||
|
||||
mt.processError(m.Run())
|
||||
mt.cancel()
|
||||
err := mt.currentError()
|
||||
require.NoError(t, err)
|
||||
|
||||
fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, "srcOnly")
|
||||
fstest.CompareItems(t, mt.dstOnly, dstOnly, test.dirDstOnly, "dstOnly")
|
||||
fstest.CompareItems(t, mt.match, match, test.dirMatch, "match")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarchNoTraverse(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
what string
|
||||
fileSrcOnly []string
|
||||
dirSrcOnly []string
|
||||
fileMatch []string
|
||||
dirMatch []string
|
||||
}{
|
||||
{
|
||||
what: "source only",
|
||||
fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"},
|
||||
dirSrcOnly: []string{"sub dir"},
|
||||
},
|
||||
{
|
||||
what: "identical",
|
||||
fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"},
|
||||
},
|
||||
{
|
||||
what: "typical sync",
|
||||
fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"},
|
||||
fileMatch: []string{"match", "matchDir/match file"},
|
||||
},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
var srcOnly []fstest.Item
|
||||
var match []fstest.Item
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
for _, f := range test.fileSrcOnly {
|
||||
srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1))
|
||||
}
|
||||
for _, f := range test.fileMatch {
|
||||
match = append(match, r.WriteBoth(ctx, f, "hello world", t1))
|
||||
}
|
||||
|
||||
mt := &marchTester{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
noTraverse: true,
|
||||
}
|
||||
m := &March{
|
||||
Ctx: ctx,
|
||||
Fdst: r.Fremote,
|
||||
Fsrc: r.Flocal,
|
||||
Dir: "",
|
||||
NoTraverse: mt.noTraverse,
|
||||
Callback: mt,
|
||||
DstIncludeAll: filter.Active.Opt.DeleteExcluded,
|
||||
}
|
||||
|
||||
mt.processError(m.Run())
|
||||
mt.cancel()
|
||||
err := mt.currentError()
|
||||
require.NoError(t, err)
|
||||
|
||||
fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, "srcOnly")
|
||||
fstest.CompareItems(t, mt.match, match, test.dirMatch, "match")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewMatchEntries(t *testing.T) {
|
||||
var (
|
||||
a = mockobject.Object("path/a")
|
||||
|
||||
@ -349,6 +349,49 @@ func CheckItems(t *testing.T, f fs.Fs, items ...Item) {
|
||||
CheckListingWithPrecision(t, f, items, nil, fs.GetModifyWindow(f))
|
||||
}
|
||||
|
||||
// CompareItems compares a set of DirEntries to a slice of items and a lit of dirs
|
||||
func CompareItems(t *testing.T, entries fs.DirEntries, items []Item, expectedDirs []string, what string) {
|
||||
is := NewItems(items)
|
||||
precision, _ := time.ParseDuration("1s")
|
||||
var objs []fs.Object
|
||||
var dirs []fs.Directory
|
||||
wantListing1, wantListing2 := makeListingFromItems(items)
|
||||
for _, entry := range entries {
|
||||
switch x := entry.(type) {
|
||||
case fs.Directory:
|
||||
dirs = append(dirs, x)
|
||||
case fs.Object:
|
||||
objs = append(objs, x)
|
||||
// do nothing
|
||||
default:
|
||||
t.Fatalf("unknown object type %T", entry)
|
||||
}
|
||||
}
|
||||
|
||||
gotListing := makeListingFromObjects(objs)
|
||||
listingOK := wantListing1 == gotListing || wantListing2 == gotListing
|
||||
assert.True(t, listingOK, fmt.Sprintf("%s not equal, want\n %s or\n %s got\n %s", what, wantListing1, wantListing2, gotListing))
|
||||
for _, obj := range objs {
|
||||
require.NotNil(t, obj)
|
||||
is.Find(t, obj, precision)
|
||||
}
|
||||
is.Done(t)
|
||||
// Check the directories
|
||||
if expectedDirs != nil {
|
||||
expectedDirsCopy := make([]string, len(expectedDirs))
|
||||
for i, dir := range expectedDirs {
|
||||
expectedDirsCopy[i] = WinPath(Normalize(dir))
|
||||
}
|
||||
actualDirs := []string{}
|
||||
for _, dir := range dirs {
|
||||
actualDirs = append(actualDirs, WinPath(Normalize(dir.Remote())))
|
||||
}
|
||||
sort.Strings(actualDirs)
|
||||
sort.Strings(expectedDirsCopy)
|
||||
assert.Equal(t, expectedDirsCopy, actualDirs, "directories not equal")
|
||||
}
|
||||
}
|
||||
|
||||
// Time parses a time string or logs a fatal error
|
||||
func Time(timeString string) time.Time {
|
||||
t, err := time.Parse(time.RFC3339Nano, timeString)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user