mirror of
https://github.com/rclone/rclone.git
synced 2025-01-31 02:32:52 +01:00
d523f00d31
This changes the syncing method to take callbacks for directory listings rather than being passed the entire directory listing at once. This will enable out of memory syncing.
543 lines
12 KiB
Go
543 lines
12 KiB
Go
// Internal tests for march
|
|
|
|
package march
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
_ "github.com/rclone/rclone/backend/local"
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/filter"
|
|
"github.com/rclone/rclone/fs/fserrors"
|
|
"github.com/rclone/rclone/fs/list"
|
|
"github.com/rclone/rclone/fs/walk"
|
|
"github.com/rclone/rclone/fstest"
|
|
"github.com/rclone/rclone/fstest/mockdir"
|
|
"github.com/rclone/rclone/fstest/mockobject"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/text/unicode/norm"
|
|
)
|
|
|
|
// Some times used in the tests
|
|
var (
|
|
t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
|
|
)
|
|
|
|
func TestMain(m *testing.M) {
|
|
fstest.TestMain(m)
|
|
}
|
|
|
|
type marchTester struct {
|
|
ctx context.Context // internal context for controlling go-routines
|
|
cancel func() // cancel the context
|
|
srcOnly fs.DirEntries
|
|
dstOnly fs.DirEntries
|
|
match fs.DirEntries
|
|
entryMutex sync.Mutex
|
|
errorMu sync.Mutex // Mutex covering the error variables
|
|
err error
|
|
noRetryErr error
|
|
fatalErr error
|
|
noTraverse bool
|
|
}
|
|
|
|
// DstOnly have an object which is in the destination only
|
|
func (mt *marchTester) DstOnly(dst fs.DirEntry) (recurse bool) {
|
|
mt.entryMutex.Lock()
|
|
mt.dstOnly = append(mt.dstOnly, dst)
|
|
mt.entryMutex.Unlock()
|
|
|
|
switch dst.(type) {
|
|
case fs.Object:
|
|
return false
|
|
case fs.Directory:
|
|
return true
|
|
default:
|
|
panic("Bad object in DirEntries")
|
|
}
|
|
}
|
|
|
|
// SrcOnly have an object which is in the source only
|
|
func (mt *marchTester) SrcOnly(src fs.DirEntry) (recurse bool) {
|
|
mt.entryMutex.Lock()
|
|
mt.srcOnly = append(mt.srcOnly, src)
|
|
mt.entryMutex.Unlock()
|
|
|
|
switch src.(type) {
|
|
case fs.Object:
|
|
return false
|
|
case fs.Directory:
|
|
return true
|
|
default:
|
|
panic("Bad object in DirEntries")
|
|
}
|
|
}
|
|
|
|
// Match is called when src and dst are present, so sync src to dst
|
|
func (mt *marchTester) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) {
|
|
mt.entryMutex.Lock()
|
|
mt.match = append(mt.match, src)
|
|
mt.entryMutex.Unlock()
|
|
|
|
switch src.(type) {
|
|
case fs.Object:
|
|
return false
|
|
case fs.Directory:
|
|
// Do the same thing to the entire contents of the directory
|
|
_, ok := dst.(fs.Directory)
|
|
if ok {
|
|
return true
|
|
}
|
|
// FIXME src is dir, dst is file
|
|
err := errors.New("can't overwrite file with directory")
|
|
fs.Errorf(dst, "%v", err)
|
|
mt.processError(err)
|
|
default:
|
|
panic("Bad object in DirEntries")
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (mt *marchTester) processError(err error) {
|
|
if err == nil {
|
|
return
|
|
}
|
|
mt.errorMu.Lock()
|
|
defer mt.errorMu.Unlock()
|
|
switch {
|
|
case fserrors.IsFatalError(err):
|
|
if !mt.aborting() {
|
|
fs.Errorf(nil, "Cancelling sync due to fatal error: %v", err)
|
|
mt.cancel()
|
|
}
|
|
mt.fatalErr = err
|
|
case fserrors.IsNoRetryError(err):
|
|
mt.noRetryErr = err
|
|
default:
|
|
mt.err = err
|
|
}
|
|
}
|
|
|
|
func (mt *marchTester) currentError() error {
|
|
mt.errorMu.Lock()
|
|
defer mt.errorMu.Unlock()
|
|
if mt.fatalErr != nil {
|
|
return mt.fatalErr
|
|
}
|
|
if mt.err != nil {
|
|
return mt.err
|
|
}
|
|
return mt.noRetryErr
|
|
}
|
|
|
|
func (mt *marchTester) aborting() bool {
|
|
return mt.ctx.Err() != nil
|
|
}
|
|
|
|
func TestMarch(t *testing.T) {
|
|
for _, test := range []struct {
|
|
what string
|
|
fileSrcOnly []string
|
|
dirSrcOnly []string
|
|
fileDstOnly []string
|
|
dirDstOnly []string
|
|
fileMatch []string
|
|
dirMatch []string
|
|
noTraverse bool
|
|
fastList bool
|
|
}{
|
|
{
|
|
what: "source only",
|
|
fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"},
|
|
dirSrcOnly: []string{"sub dir"},
|
|
},
|
|
{
|
|
what: "identical",
|
|
fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"},
|
|
dirMatch: []string{"sub dir", "sub dir/sub sub dir"},
|
|
},
|
|
{
|
|
what: "typical sync",
|
|
fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"},
|
|
dirSrcOnly: []string{"srcOnlyDir"},
|
|
fileMatch: []string{"match", "matchDir/match file"},
|
|
dirMatch: []string{"matchDir"},
|
|
fileDstOnly: []string{"dstOnly", "dstOnlyDir/sub"},
|
|
dirDstOnly: []string{"dstOnlyDir"},
|
|
},
|
|
{
|
|
what: "no traverse source only",
|
|
fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"},
|
|
dirSrcOnly: []string{"sub dir"},
|
|
noTraverse: true,
|
|
},
|
|
{
|
|
what: "no traverse identical",
|
|
fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"},
|
|
noTraverse: true,
|
|
},
|
|
{
|
|
what: "no traverse typical sync",
|
|
fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"},
|
|
fileMatch: []string{"match", "matchDir/match file"},
|
|
noTraverse: true,
|
|
},
|
|
{
|
|
what: "fast list source only",
|
|
fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"},
|
|
dirSrcOnly: []string{"sub dir"},
|
|
fastList: true,
|
|
},
|
|
{
|
|
what: "fast list identical",
|
|
fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"},
|
|
dirMatch: []string{"sub dir", "sub dir/sub sub dir"},
|
|
fastList: true,
|
|
},
|
|
{
|
|
what: "fast list typical sync",
|
|
fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"},
|
|
dirSrcOnly: []string{"srcOnlyDir"},
|
|
fileMatch: []string{"match", "matchDir/match file"},
|
|
dirMatch: []string{"matchDir"},
|
|
fileDstOnly: []string{"dstOnly", "dstOnlyDir/sub"},
|
|
dirDstOnly: []string{"dstOnlyDir"},
|
|
fastList: true,
|
|
},
|
|
} {
|
|
t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
|
|
var srcOnly []fstest.Item
|
|
var dstOnly []fstest.Item
|
|
var match []fstest.Item
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
for _, f := range test.fileSrcOnly {
|
|
srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1))
|
|
}
|
|
for _, f := range test.fileDstOnly {
|
|
dstOnly = append(dstOnly, r.WriteObject(ctx, f, "hello world", t1))
|
|
}
|
|
for _, f := range test.fileMatch {
|
|
match = append(match, r.WriteBoth(ctx, f, "hello world", t1))
|
|
}
|
|
|
|
ctx, ci := fs.AddConfig(ctx)
|
|
ci.UseListR = test.fastList
|
|
|
|
fi := filter.GetConfig(ctx)
|
|
|
|
// Local backend doesn't implement ListR, so monkey patch it for this test
|
|
if test.fastList && r.Flocal.Features().ListR == nil {
|
|
r.Flocal.Features().ListR = func(ctx context.Context, dir string, callback fs.ListRCallback) error {
|
|
r.Flocal.Features().ListR = nil // disable ListR to avoid infinite recursion
|
|
return walk.ListR(ctx, r.Flocal, dir, true, -1, walk.ListAll, callback)
|
|
}
|
|
defer func() {
|
|
r.Flocal.Features().ListR = nil
|
|
}()
|
|
}
|
|
|
|
mt := &marchTester{
|
|
ctx: ctx,
|
|
cancel: cancel,
|
|
noTraverse: test.noTraverse,
|
|
}
|
|
m := &March{
|
|
Ctx: ctx,
|
|
Fdst: r.Fremote,
|
|
Fsrc: r.Flocal,
|
|
Dir: "",
|
|
NoTraverse: test.noTraverse,
|
|
Callback: mt,
|
|
DstIncludeAll: fi.Opt.DeleteExcluded,
|
|
}
|
|
|
|
mt.processError(m.Run(ctx))
|
|
mt.cancel()
|
|
err := mt.currentError()
|
|
require.NoError(t, err)
|
|
|
|
precision := fs.GetModifyWindow(ctx, r.Fremote, r.Flocal)
|
|
fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, precision, "srcOnly")
|
|
fstest.CompareItems(t, mt.dstOnly, dstOnly, test.dirDstOnly, precision, "dstOnly")
|
|
fstest.CompareItems(t, mt.match, match, test.dirMatch, precision, "match")
|
|
})
|
|
}
|
|
}
|
|
|
|
// matchPair is a matched pair of direntries returned by matchListings
|
|
type matchPair struct {
|
|
src, dst fs.DirEntry
|
|
}
|
|
|
|
func TestMatchListings(t *testing.T) {
|
|
var (
|
|
a = mockobject.Object("a")
|
|
A = mockobject.Object("A")
|
|
b = mockobject.Object("b")
|
|
c = mockobject.Object("c")
|
|
d = mockobject.Object("d")
|
|
uE1 = mockobject.Object("é") // one of the unicode E characters
|
|
uE2 = mockobject.Object("é") // a different unicode E character
|
|
dirA = mockdir.New("A")
|
|
dirb = mockdir.New("b")
|
|
)
|
|
|
|
for _, test := range []struct {
|
|
what string
|
|
input fs.DirEntries // pairs of input src, dst
|
|
srcOnly fs.DirEntries
|
|
dstOnly fs.DirEntries
|
|
matches []matchPair // pairs of output
|
|
transforms []matchTransformFn
|
|
}{
|
|
{
|
|
what: "only src or dst",
|
|
input: fs.DirEntries{
|
|
a, nil,
|
|
b, nil,
|
|
c, nil,
|
|
d, nil,
|
|
},
|
|
srcOnly: fs.DirEntries{
|
|
a, b, c, d,
|
|
},
|
|
},
|
|
{
|
|
what: "typical sync #1",
|
|
input: fs.DirEntries{
|
|
a, nil,
|
|
b, b,
|
|
nil, c,
|
|
nil, d,
|
|
},
|
|
srcOnly: fs.DirEntries{
|
|
a,
|
|
},
|
|
dstOnly: fs.DirEntries{
|
|
c, d,
|
|
},
|
|
matches: []matchPair{
|
|
{b, b},
|
|
},
|
|
},
|
|
{
|
|
what: "typical sync #2",
|
|
input: fs.DirEntries{
|
|
a, a,
|
|
b, b,
|
|
nil, c,
|
|
d, d,
|
|
},
|
|
dstOnly: fs.DirEntries{
|
|
c,
|
|
},
|
|
matches: []matchPair{
|
|
{a, a},
|
|
{b, b},
|
|
{d, d},
|
|
},
|
|
},
|
|
{
|
|
what: "One duplicate",
|
|
input: fs.DirEntries{
|
|
A, A,
|
|
a, a,
|
|
a, nil,
|
|
b, b,
|
|
},
|
|
matches: []matchPair{
|
|
{A, A},
|
|
{a, a},
|
|
{b, b},
|
|
},
|
|
},
|
|
{
|
|
what: "Two duplicates",
|
|
input: fs.DirEntries{
|
|
a, a,
|
|
a, a,
|
|
a, nil,
|
|
},
|
|
matches: []matchPair{
|
|
{a, a},
|
|
},
|
|
},
|
|
{
|
|
what: "Case insensitive duplicate - no transform",
|
|
input: fs.DirEntries{
|
|
a, a,
|
|
A, A,
|
|
},
|
|
matches: []matchPair{
|
|
{A, A},
|
|
{a, a},
|
|
},
|
|
},
|
|
{
|
|
what: "Case insensitive duplicate - transform to lower case",
|
|
input: fs.DirEntries{
|
|
a, A,
|
|
A, a,
|
|
},
|
|
matches: []matchPair{
|
|
{a, A}, // the first duplicate will be returned with a stable sort
|
|
},
|
|
transforms: []matchTransformFn{strings.ToLower},
|
|
},
|
|
{
|
|
what: "Unicode near-duplicate that becomes duplicate with normalization",
|
|
input: fs.DirEntries{
|
|
uE1, uE1,
|
|
uE2, uE2,
|
|
},
|
|
matches: []matchPair{
|
|
{uE1, uE1},
|
|
},
|
|
transforms: []matchTransformFn{norm.NFC.String},
|
|
},
|
|
{
|
|
what: "Unicode near-duplicate with no normalization",
|
|
input: fs.DirEntries{
|
|
uE1, uE1,
|
|
uE2, uE2,
|
|
},
|
|
matches: []matchPair{
|
|
{uE1, uE1},
|
|
{uE2, uE2},
|
|
},
|
|
},
|
|
{
|
|
what: "File and directory are not duplicates - srcOnly",
|
|
input: fs.DirEntries{
|
|
dirA, nil,
|
|
A, nil,
|
|
},
|
|
srcOnly: fs.DirEntries{
|
|
dirA,
|
|
A,
|
|
},
|
|
},
|
|
{
|
|
what: "File and directory are not duplicates - matches",
|
|
input: fs.DirEntries{
|
|
dirA, dirA,
|
|
A, A,
|
|
},
|
|
matches: []matchPair{
|
|
{dirA, dirA},
|
|
{A, A},
|
|
},
|
|
},
|
|
{
|
|
what: "Sync with directory #1",
|
|
input: fs.DirEntries{
|
|
dirA, nil,
|
|
A, nil,
|
|
b, b,
|
|
nil, c,
|
|
nil, d,
|
|
},
|
|
srcOnly: fs.DirEntries{
|
|
dirA,
|
|
A,
|
|
},
|
|
dstOnly: fs.DirEntries{
|
|
c, d,
|
|
},
|
|
matches: []matchPair{
|
|
{b, b},
|
|
},
|
|
},
|
|
{
|
|
what: "Sync with 2 directories",
|
|
input: fs.DirEntries{
|
|
dirA, dirA,
|
|
A, nil,
|
|
nil, dirb,
|
|
nil, b,
|
|
},
|
|
srcOnly: fs.DirEntries{
|
|
A,
|
|
},
|
|
dstOnly: fs.DirEntries{
|
|
dirb,
|
|
b,
|
|
},
|
|
matches: []matchPair{
|
|
{dirA, dirA},
|
|
},
|
|
},
|
|
} {
|
|
t.Run(fmt.Sprintf("TestMatchListings-%s", test.what), func(t *testing.T) {
|
|
ctx := context.Background()
|
|
var wg sync.WaitGroup
|
|
|
|
// Skeleton March for testing
|
|
m := March{
|
|
Ctx: context.Background(),
|
|
transforms: test.transforms,
|
|
}
|
|
|
|
// Make a channel to send the source (0) or dest (1) using a list.Sorter
|
|
makeChan := func(offset int) <-chan fs.DirEntry {
|
|
out := make(chan fs.DirEntry)
|
|
ls, err := list.NewSorter(ctx, nil, list.SortToChan(out), m.key)
|
|
require.NoError(t, err)
|
|
defer ls.CleanUp()
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for i := 0; i < len(test.input); i += 2 {
|
|
entry := test.input[i+offset]
|
|
if entry != nil {
|
|
require.NoError(t, ls.Add(fs.DirEntries{entry}))
|
|
}
|
|
}
|
|
require.NoError(t, ls.Send())
|
|
close(out)
|
|
}()
|
|
return out
|
|
}
|
|
|
|
var srcOnly fs.DirEntries
|
|
srcOnlyFn := func(entry fs.DirEntry) {
|
|
srcOnly = append(srcOnly, entry)
|
|
}
|
|
var dstOnly fs.DirEntries
|
|
dstOnlyFn := func(entry fs.DirEntry) {
|
|
dstOnly = append(dstOnly, entry)
|
|
}
|
|
var matches []matchPair
|
|
matchFn := func(dst, src fs.DirEntry) {
|
|
matches = append(matches, matchPair{dst: dst, src: src})
|
|
}
|
|
|
|
err := m.matchListings(makeChan(0), makeChan(1), srcOnlyFn, dstOnlyFn, matchFn)
|
|
require.NoError(t, err)
|
|
wg.Wait()
|
|
assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ")
|
|
assert.Equal(t, test.dstOnly, dstOnly, test.what, "dstOnly differ")
|
|
assert.Equal(t, test.matches, matches, test.what, "matches differ")
|
|
|
|
// now swap src and dst
|
|
srcOnly, dstOnly, matches = nil, nil, nil
|
|
err = m.matchListings(makeChan(0), makeChan(1), srcOnlyFn, dstOnlyFn, matchFn)
|
|
require.NoError(t, err)
|
|
wg.Wait()
|
|
assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ")
|
|
assert.Equal(t, test.dstOnly, dstOnly, test.what, "dstOnly differ")
|
|
assert.Equal(t, test.matches, matches, test.what, "matches differ")
|
|
})
|
|
}
|
|
}
|