mirror of
https://github.com/rclone/rclone.git
synced 2025-08-15 00:02:35 +02:00
operations: add logger to log list of sync results -- fixes #7282
Logger instruments the Sync routine with a status report for each file pair, making it possible to output a list of the synced files, along with their attributes and sigil categorization (match/differ/missing/etc.) It is very customizable by passing in a custom LoggerFn, options, and io.Writers to be written to. Possible uses include: - allow sync to write path lists to a file, in the same format as rclone check - allow sync to output a --dest-after file using the same format flags as lsf - receive results as JSON when calling sync from an internal function - predict the post-sync state of the destination For usage examples, see bisync.WriteResults() or sync.SyncLoggerFn()
This commit is contained in:
nielash
@ -3,14 +3,20 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mutex "sync" // renamed as "sync" already in use
|
||||
|
||||
_ "github.com/rclone/rclone/backend/all" // import all backends
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
@ -45,7 +51,9 @@ func TestCopyWithDryRun(t *testing.T) {
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
|
||||
ci.DryRun = true
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := CopyDir(ctx, r.Fremote, r.Flocal, false)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // error expected here because dry-run
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
@ -59,8 +67,10 @@ func TestCopy(t *testing.T) {
|
||||
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := CopyDir(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
@ -76,8 +86,10 @@ func TestCopyMissingDirectory(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = CopyDir(ctx, r.Fremote, nonExistingFs, false)
|
||||
require.Error(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
}
|
||||
|
||||
// Now with --no-traverse
|
||||
@ -90,8 +102,10 @@ func TestCopyNoTraverse(t *testing.T) {
|
||||
|
||||
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := CopyDir(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
@ -107,8 +121,10 @@ func TestCopyCheckFirst(t *testing.T) {
|
||||
|
||||
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := CopyDir(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
@ -125,8 +141,10 @@ func TestSyncNoTraverse(t *testing.T) {
|
||||
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
@ -143,8 +161,10 @@ func TestCopyWithDepth(t *testing.T) {
|
||||
// Check the MaxDepth too
|
||||
ci.MaxDepth = 1
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := CopyDir(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file1, file2)
|
||||
r.CheckRemoteItems(t, file2)
|
||||
@ -169,8 +189,10 @@ func testCopyWithFilesFrom(t *testing.T, noTraverse bool) {
|
||||
|
||||
ci.NoTraverse = noTraverse
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = CopyDir(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file1, file2)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
@ -187,8 +209,10 @@ func TestCopyEmptyDirectories(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = CopyDir(ctx, r.Fremote, r.Flocal, true)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckRemoteListing(
|
||||
t,
|
||||
@ -211,8 +235,10 @@ func TestMoveEmptyDirectories(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = MoveDir(ctx, r.Fremote, r.Flocal, false, true)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckRemoteListing(
|
||||
t,
|
||||
@ -235,8 +261,10 @@ func TestSyncEmptyDirectories(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckRemoteListing(
|
||||
t,
|
||||
@ -262,8 +290,10 @@ func TestServerSideCopy(t *testing.T) {
|
||||
defer finaliseCopy()
|
||||
t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = CopyDir(ctx, FremoteCopy, r.Fremote, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
fstest.CheckItems(t, FremoteCopy, file1)
|
||||
}
|
||||
@ -280,8 +310,10 @@ func TestCopyAfterDelete(t *testing.T) {
|
||||
err := operations.Mkdir(ctx, r.Flocal, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = CopyDir(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
@ -294,8 +326,10 @@ func TestCopyRedownload(t *testing.T) {
|
||||
file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := CopyDir(ctx, r.Flocal, r.Fremote, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// Test with combined precision of local and remote as we copied it there and back
|
||||
r.CheckLocalListing(t, []fstest.Item{file1}, nil)
|
||||
@ -314,8 +348,10 @@ func TestSyncBasedOnCheckSum(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// We should have transferred exactly one file.
|
||||
assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())
|
||||
@ -326,8 +362,10 @@ func TestSyncBasedOnCheckSum(t *testing.T) {
|
||||
r.CheckLocalItems(t, file2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// We should have transferred no files
|
||||
assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())
|
||||
@ -348,8 +386,10 @@ func TestSyncSizeOnly(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// We should have transferred exactly one file.
|
||||
assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())
|
||||
@ -360,8 +400,10 @@ func TestSyncSizeOnly(t *testing.T) {
|
||||
r.CheckLocalItems(t, file2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// We should have transferred no files
|
||||
assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())
|
||||
@ -382,8 +424,10 @@ func TestSyncIgnoreSize(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// We should have transferred exactly one file.
|
||||
assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())
|
||||
@ -394,8 +438,10 @@ func TestSyncIgnoreSize(t *testing.T) {
|
||||
r.CheckLocalItems(t, file2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// We should have transferred no files
|
||||
assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())
|
||||
@ -411,8 +457,10 @@ func TestSyncIgnoreTimes(t *testing.T) {
|
||||
r.CheckRemoteItems(t, file1)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// We should have transferred exactly 0 files because the
|
||||
// files were identical.
|
||||
@ -421,8 +469,10 @@ func TestSyncIgnoreTimes(t *testing.T) {
|
||||
ci.IgnoreTimes = true
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// We should have transferred exactly one file even though the
|
||||
// files were identical.
|
||||
@ -441,18 +491,22 @@ func TestSyncIgnoreExisting(t *testing.T) {
|
||||
ci.IgnoreExisting = true
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// Change everything
|
||||
r.WriteFile("existing", "newpotatoes", t2)
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
// Items should not change
|
||||
r.CheckRemoteItems(t, file1)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
}
|
||||
|
||||
func TestSyncIgnoreErrors(t *testing.T) {
|
||||
@ -490,8 +544,10 @@ func TestSyncIgnoreErrors(t *testing.T) {
|
||||
)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
_ = fs.CountError(errors.New("boom"))
|
||||
assert.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalListing(
|
||||
t,
|
||||
@ -530,8 +586,10 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
|
||||
ci.DryRun = true
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file2)
|
||||
@ -539,8 +597,10 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
|
||||
ci.DryRun = false
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
@ -565,8 +625,10 @@ func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) {
|
||||
r.CheckRemoteItems(t, file2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file2)
|
||||
@ -586,8 +648,10 @@ func TestSyncDoesntUpdateModtime(t *testing.T) {
|
||||
r.CheckRemoteItems(t, file2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
@ -606,8 +670,10 @@ func TestSyncAfterAddingAFile(t *testing.T) {
|
||||
r.CheckRemoteItems(t, file1)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
r.CheckLocalItems(t, file1, file2)
|
||||
r.CheckRemoteItems(t, file1, file2)
|
||||
}
|
||||
@ -621,8 +687,10 @@ func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
|
||||
r.CheckLocalItems(t, file2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
r.CheckLocalItems(t, file2)
|
||||
r.CheckRemoteItems(t, file2)
|
||||
}
|
||||
@ -644,8 +712,10 @@ func TestSyncAfterChangingContentsOnly(t *testing.T) {
|
||||
r.CheckLocalItems(t, file2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
r.CheckLocalItems(t, file2)
|
||||
r.CheckRemoteItems(t, file2)
|
||||
}
|
||||
@ -661,9 +731,11 @@ func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
|
||||
|
||||
ci.DryRun = true
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
ci.DryRun = false
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalItems(t, file3, file1)
|
||||
r.CheckRemoteItems(t, file3, file2)
|
||||
@ -679,8 +751,10 @@ func testSyncAfterRemovingAFileAndAddingAFile(ctx context.Context, t *testing.T)
|
||||
r.CheckLocalItems(t, file1, file3)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
r.CheckLocalItems(t, file1, file3)
|
||||
r.CheckRemoteItems(t, file1, file3)
|
||||
}
|
||||
@ -724,8 +798,10 @@ func testSyncAfterRemovingAFileAndAddingAFileSubDir(ctx context.Context, t *test
|
||||
)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalListing(
|
||||
t,
|
||||
@ -788,10 +864,12 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) {
|
||||
},
|
||||
)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
_ = fs.CountError(errors.New("boom"))
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
assert.Equal(t, fs.ErrorNotDeleting, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalListing(
|
||||
t,
|
||||
@ -861,8 +939,10 @@ func TestCopyDeleteBefore(t *testing.T) {
|
||||
r.CheckLocalItems(t, file2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := CopyDir(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckRemoteItems(t, file1, file2)
|
||||
r.CheckLocalItems(t, file2)
|
||||
@ -884,15 +964,19 @@ func TestSyncWithExclude(t *testing.T) {
|
||||
ctx = filter.ReplaceConfig(ctx, fi)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
r.CheckRemoteItems(t, file2, file1)
|
||||
|
||||
// Now sync the other way round and check enormous doesn't get
|
||||
// deleted as it is excluded from the sync
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Flocal, r.Fremote, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
r.CheckLocalItems(t, file2, file1, file3)
|
||||
}
|
||||
|
||||
@ -913,15 +997,19 @@ func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) {
|
||||
ctx = filter.ReplaceConfig(ctx, fi)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
r.CheckRemoteItems(t, file2)
|
||||
|
||||
// Check sync the other way round to make sure enormous gets
|
||||
// deleted even though it is excluded
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Flocal, r.Fremote, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
r.CheckLocalItems(t, file2)
|
||||
}
|
||||
|
||||
@ -950,9 +1038,11 @@ func TestSyncWithUpdateOlder(t *testing.T) {
|
||||
ci.UpdateOlder = true
|
||||
ci.ModifyWindow = fs.ModTimeNotSupported
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
r.CheckRemoteItems(t, oneO, twoF, threeO, fourF, fiveF)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // no modtime
|
||||
|
||||
if r.Fremote.Hashes().Count() == 0 {
|
||||
t.Logf("Skip test with --checksum as no hashes supported")
|
||||
@ -994,8 +1084,10 @@ func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) {
|
||||
r.CheckRemoteItems(t)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx) // not currently supported (but tests do pass for CutoffModeSoft)
|
||||
startTime := time.Now()
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.True(t, errors.Is(err, ErrorMaxDurationReached))
|
||||
|
||||
if cutoffMode == fs.CutoffModeHard {
|
||||
@ -1042,7 +1134,9 @@ func TestSyncWithTrackRenames(t *testing.T) {
|
||||
f2 := r.WriteFile("yam", "Yam Content", t2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckRemoteItems(t, f1, f2)
|
||||
r.CheckLocalItems(t, f1, f2)
|
||||
@ -1051,7 +1145,9 @@ func TestSyncWithTrackRenames(t *testing.T) {
|
||||
f2 = r.RenameFile(f2, "yaml")
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckRemoteItems(t, f1, f2)
|
||||
|
||||
@ -1110,7 +1206,9 @@ func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) {
|
||||
f2 := r.WriteFile("yam", "Yam Content", t2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckRemoteItems(t, f1, f2)
|
||||
r.CheckLocalItems(t, f1, f2)
|
||||
@ -1119,7 +1217,9 @@ func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) {
|
||||
f2 = r.RenameFile(f2, "yaml")
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckRemoteItems(t, f1, f2)
|
||||
|
||||
@ -1145,7 +1245,9 @@ func TestSyncWithTrackRenamesStrategyLeaf(t *testing.T) {
|
||||
f2 := r.WriteFile("sub/yam", "Yam Content", t2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckRemoteItems(t, f1, f2)
|
||||
r.CheckLocalItems(t, f1, f2)
|
||||
@ -1154,7 +1256,9 @@ func TestSyncWithTrackRenamesStrategyLeaf(t *testing.T) {
|
||||
f2 = r.RenameFile(f2, "yam")
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckRemoteItems(t, f1, f2)
|
||||
|
||||
@ -1200,8 +1304,10 @@ func testServerSideMove(ctx context.Context, t *testing.T, r *fstest.Run, withFi
|
||||
|
||||
// Do server-side move
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx) // not currently supported -- doesn't list all contents of dir.
|
||||
err = MoveDir(ctx, FremoteMove, r.Fremote, testDeleteEmptyDirs, false)
|
||||
require.NoError(t, err)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
if withFilter {
|
||||
r.CheckRemoteItems(t, file2)
|
||||
@ -1227,8 +1333,10 @@ func testServerSideMove(ctx context.Context, t *testing.T, r *fstest.Run, withFi
|
||||
|
||||
// Move it back to a new empty remote, dst does not exist this time
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err = MoveDir(ctx, FremoteMove2, FremoteMove, testDeleteEmptyDirs, false)
|
||||
require.NoError(t, err)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
if withFilter {
|
||||
fstest.CheckItems(t, FremoteMove2, file1, file3u)
|
||||
@ -1252,8 +1360,10 @@ func TestMoveWithDeleteEmptySrcDirs(t *testing.T) {
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
|
||||
// run move with --delete-empty-src-dirs
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := MoveDir(ctx, r.Fremote, r.Flocal, true, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalListing(
|
||||
t,
|
||||
@ -1270,8 +1380,10 @@ func TestMoveWithoutDeleteEmptySrcDirs(t *testing.T) {
|
||||
file2 := r.WriteFile("nested/sub dir/file", "nested", t1)
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := MoveDir(ctx, r.Fremote, r.Flocal, false, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
r.CheckLocalListing(
|
||||
t,
|
||||
@ -1295,8 +1407,10 @@ func TestMoveWithIgnoreExisting(t *testing.T) {
|
||||
ci.IgnoreExisting = true
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err := MoveDir(ctx, r.Fremote, r.Flocal, false, false)
|
||||
require.NoError(t, err)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
r.CheckLocalListing(
|
||||
t,
|
||||
[]fstest.Item{},
|
||||
@ -1314,8 +1428,10 @@ func TestMoveWithIgnoreExisting(t *testing.T) {
|
||||
// Recreate first file with modified content
|
||||
file1b := r.WriteFile("existing", "newpotatoes", t2)
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = MoveDir(ctx, r.Fremote, r.Flocal, false, false)
|
||||
require.NoError(t, err)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
// Source items should still exist in modified state
|
||||
r.CheckLocalListing(
|
||||
t,
|
||||
@ -1379,8 +1495,10 @@ func TestServerSideMoveOverlap(t *testing.T) {
|
||||
r.CheckRemoteItems(t, file1)
|
||||
|
||||
// Subdir move with no filters should return ErrorCantMoveOverlapping
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)
|
||||
assert.EqualError(t, err, fs.ErrorOverlapping.Error())
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
// Now try with a filter which should also fail with ErrorCantMoveOverlapping
|
||||
fi, err := filter.NewFilter(nil)
|
||||
@ -1388,8 +1506,10 @@ func TestServerSideMoveOverlap(t *testing.T) {
|
||||
fi.Opt.MinSize = 40
|
||||
ctx = filter.ReplaceConfig(ctx, fi)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)
|
||||
assert.EqualError(t, err, fs.ErrorOverlapping.Error())
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
}
|
||||
|
||||
// Test a sync with overlap
|
||||
@ -1407,10 +1527,18 @@ func TestSyncOverlap(t *testing.T) {
|
||||
assert.Equal(t, fs.ErrorOverlapping.Error(), err.Error())
|
||||
}
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
checkErr(Sync(ctx, FremoteSync, r.Fremote, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
checkErr(Sync(ctx, r.Fremote, FremoteSync, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
checkErr(Sync(ctx, r.Fremote, r.Fremote, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
checkErr(Sync(ctx, FremoteSync, FremoteSync, false))
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
}
|
||||
|
||||
// Test a sync with filtered overlap
|
||||
@ -1453,29 +1581,63 @@ func TestSyncOverlapWithFilter(t *testing.T) {
|
||||
}
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkNoErr(Sync(filterCtx, FremoteSync, r.Fremote, false))
|
||||
checkErr(Sync(ctx, FremoteSync, r.Fremote, false))
|
||||
checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(ctx, r.Fremote, FremoteSync, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(filterCtx, r.Fremote, r.Fremote, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(ctx, r.Fremote, r.Fremote, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(filterCtx, FremoteSync, FremoteSync, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(ctx, FremoteSync, FremoteSync, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
|
||||
checkNoErr(Sync(filterCtx, FremoteSync2, r.Fremote, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(ctx, FremoteSync2, r.Fremote, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync2, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(ctx, r.Fremote, FremoteSync2, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(filterCtx, FremoteSync2, FremoteSync2, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(ctx, FremoteSync2, FremoteSync2, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
|
||||
checkNoErr(Sync(filterCtx, FremoteSync3, r.Fremote, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(ctx, FremoteSync3, r.Fremote, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
// Destination is excluded so this test makes no sense
|
||||
// checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync3, false))
|
||||
checkErr(Sync(ctx, r.Fremote, FremoteSync3, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(filterCtx, FremoteSync3, FremoteSync3, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
filterCtx = predictDstFromLogger(filterCtx)
|
||||
checkErr(Sync(ctx, FremoteSync3, FremoteSync3, false))
|
||||
testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
|
||||
}
|
||||
|
||||
// Test with CompareDest set
|
||||
@ -1494,7 +1656,9 @@ func TestSyncCompareDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx) // not currently supported due to duplicate equal() checks
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
file1dst := file1
|
||||
@ -1508,7 +1672,9 @@ func TestSyncCompareDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1b)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
file1bdst := file1b
|
||||
@ -1524,7 +1690,9 @@ func TestSyncCompareDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1c)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckRemoteItems(t, file2, file3)
|
||||
@ -1536,14 +1704,18 @@ func TestSyncCompareDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1c, file5)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckRemoteItems(t, file2, file3, file4)
|
||||
|
||||
// check new dest, new compare
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckRemoteItems(t, file2, file3, file4)
|
||||
@ -1570,7 +1742,9 @@ func TestSyncCompareDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1c, file5b)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckRemoteItems(t, file2, file3, file4)
|
||||
@ -1584,7 +1758,9 @@ func TestSyncCompareDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1c, file5c)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
file5cdst := file5c
|
||||
@ -1615,7 +1791,9 @@ func TestSyncMultipleCompareDest(t *testing.T) {
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dest")
|
||||
require.NoError(t, err)
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
require.NoError(t, Sync(ctx, fdst, r.Flocal, false))
|
||||
// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
|
||||
fdest3 := fsrc3
|
||||
fdest3.Path = "dest/3"
|
||||
@ -1644,7 +1822,9 @@ func TestSyncCopyDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
file1dst := file1
|
||||
@ -1658,7 +1838,9 @@ func TestSyncCopyDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1b)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
file1bdst := file1b
|
||||
@ -1677,7 +1859,9 @@ func TestSyncCopyDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1c)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
file2dst := file2
|
||||
@ -1694,6 +1878,8 @@ func TestSyncCopyDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1c, file5)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
@ -1704,7 +1890,9 @@ func TestSyncCopyDest(t *testing.T) {
|
||||
|
||||
// check new dest, new copy
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)
|
||||
@ -1716,7 +1904,9 @@ func TestSyncCopyDest(t *testing.T) {
|
||||
r.CheckLocalItems(t, file1c, file5, file7)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, fdst, r.Flocal, false)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
file7dst := file7
|
||||
@ -1928,7 +2118,9 @@ func TestSyncUTFNorm(t *testing.T) {
|
||||
r.CheckRemoteItems(t, file2)
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // can't test this on macOS
|
||||
require.NoError(t, err)
|
||||
|
||||
// We should have transferred exactly one file, but kept the
|
||||
@ -1954,7 +2146,9 @@ func TestSyncImmutable(t *testing.T) {
|
||||
|
||||
// Should succeed
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
@ -1966,7 +2160,9 @@ func TestSyncImmutable(t *testing.T) {
|
||||
|
||||
// Should fail with ErrorImmutableModified and not modify local or remote files
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
assert.EqualError(t, err, fs.ErrorImmutableModified.Error())
|
||||
r.CheckLocalItems(t, file2)
|
||||
r.CheckRemoteItems(t, file1)
|
||||
@ -1993,7 +2189,9 @@ func TestSyncIgnoreCase(t *testing.T) {
|
||||
|
||||
// Should not copy files that are differently-cased but otherwise identical
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // can't test this on macOS
|
||||
require.NoError(t, err)
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, file2)
|
||||
@ -2025,7 +2223,9 @@ func TestMaxTransfer(t *testing.T) {
|
||||
|
||||
accounting.GlobalStats().ResetCounters()
|
||||
|
||||
// ctx = predictDstFromLogger(ctx) // not currently supported
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
expectedErr := fserrors.FsError(accounting.ErrorMaxTransferLimitReachedFatal)
|
||||
if cutoff != fs.CutoffModeHard {
|
||||
expectedErr = accounting.ErrorMaxTransferLimitReachedGraceful
|
||||
@ -2075,7 +2275,9 @@ func testSyncConcurrent(t *testing.T, subtest string) {
|
||||
|
||||
r.CheckRemoteItems(t, itemsBefore...)
|
||||
stats.ResetErrors()
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
if errors.Is(err, fs.ErrorCantUploadEmptyFiles) {
|
||||
t.Skipf("Skip test because remote cannot upload empty files")
|
||||
}
|
||||
@ -2091,3 +2293,124 @@ func TestSyncConcurrentDelete(t *testing.T) {
|
||||
func TestSyncConcurrentTruncate(t *testing.T) {
|
||||
testSyncConcurrent(t, "truncate")
|
||||
}
|
||||
|
||||
// for testing logger:
|
||||
func predictDstFromLogger(ctx context.Context) context.Context {
|
||||
opt := operations.NewLoggerOpt()
|
||||
var lock mutex.Mutex
|
||||
|
||||
opt.LoggerFn = func(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
// ignore dirs for our purposes here
|
||||
if err == fs.ErrorIsDir {
|
||||
return
|
||||
}
|
||||
winner := operations.WinningSide(ctx, sigil, src, dst, err)
|
||||
if winner.Obj != nil {
|
||||
file := winner.Obj
|
||||
obj, ok := file.(fs.ObjectInfo)
|
||||
checksum := ""
|
||||
if ok {
|
||||
if obj.Fs().Hashes().GetOne() == hash.MD5 {
|
||||
// skip if no MD5
|
||||
checksum, _ = obj.Hash(ctx, obj.Fs().Hashes().GetOne())
|
||||
}
|
||||
}
|
||||
errMsg := ""
|
||||
if winner.Err != nil {
|
||||
errMsg = ";" + winner.Err.Error()
|
||||
}
|
||||
operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format("2006-01-02 15:04:05.000000000"), checksum, file.Size(), file.Remote(), errMsg)
|
||||
}
|
||||
}
|
||||
return operations.WithSyncLogger(ctx, opt)
|
||||
}
|
||||
|
||||
func DstLsf(ctx context.Context, Fremote fs.Fs) *bytes.Buffer {
|
||||
var opt = operations.ListJSONOpt{
|
||||
NoModTime: false,
|
||||
NoMimeType: true,
|
||||
DirsOnly: false,
|
||||
FilesOnly: true,
|
||||
Recurse: true,
|
||||
ShowHash: true,
|
||||
HashTypes: []string{"MD5"},
|
||||
}
|
||||
|
||||
var list operations.ListFormat
|
||||
|
||||
list.SetSeparator(";")
|
||||
timeFormat := operations.FormatForLSFPrecision(Fremote.Precision())
|
||||
list.AddModTime(timeFormat)
|
||||
list.AddHash(hash.MD5)
|
||||
list.AddSize()
|
||||
list.AddPath()
|
||||
|
||||
out := new(bytes.Buffer)
|
||||
|
||||
err := operations.ListJSON(ctx, Fremote, "", &opt, func(item *operations.ListJSONItem) error {
|
||||
_, _ = fmt.Fprintln(out, list.Format(item))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
fs.Errorf(Fremote, "ListJSON error: %v", err)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func LoggerMatchesLsf(logger, lsf *bytes.Buffer) error {
|
||||
loggerSplit := bytes.Split(logger.Bytes(), []byte("\n"))
|
||||
sort.SliceStable(loggerSplit, func(i int, j int) bool { return string(loggerSplit[i]) < string(loggerSplit[j]) })
|
||||
lsfSplit := bytes.Split(lsf.Bytes(), []byte("\n"))
|
||||
sort.SliceStable(lsfSplit, func(i int, j int) bool { return string(lsfSplit[i]) < string(lsfSplit[j]) })
|
||||
|
||||
loggerJoined := bytes.Join(loggerSplit, []byte("\n"))
|
||||
lsfJoined := bytes.Join(lsfSplit, []byte("\n"))
|
||||
|
||||
if bytes.Equal(loggerJoined, lsfJoined) {
|
||||
return nil
|
||||
}
|
||||
Diff(string(loggerJoined), string(lsfJoined))
|
||||
return fmt.Errorf("logger does not match lsf! \nlogger: \n%s \nlsf: \n%s", loggerJoined, lsfJoined)
|
||||
}
|
||||
|
||||
func Diff(rev1, rev2 string) {
|
||||
fmt.Printf("Diff of %q and %q\n", "logger", "lsf")
|
||||
cmd := exec.Command("bash", "-c", fmt.Sprintf(`diff <(echo "%s") <(echo "%s")`, rev1, rev2))
|
||||
out, _ := cmd.Output()
|
||||
_, _ = os.Stdout.Write(out)
|
||||
}
|
||||
|
||||
func testLoggerVsLsf(ctx context.Context, Fremote fs.Fs, logger *bytes.Buffer, t *testing.T) {
|
||||
var newlogger bytes.Buffer
|
||||
canTestModtime := fs.GetModifyWindow(ctx, Fremote) != fs.ModTimeNotSupported
|
||||
canTestHash := Fremote.Hashes().Contains(hash.MD5)
|
||||
if !canTestHash || !canTestModtime {
|
||||
loggerSplit := bytes.Split(logger.Bytes(), []byte("\n"))
|
||||
for i, line := range loggerSplit {
|
||||
elements := bytes.Split(line, []byte(";"))
|
||||
if len(elements) >= 2 {
|
||||
if !canTestModtime {
|
||||
elements[0] = []byte("")
|
||||
}
|
||||
if !canTestHash {
|
||||
elements[1] = []byte("")
|
||||
}
|
||||
}
|
||||
loggerSplit[i] = bytes.Join(elements, []byte(";"))
|
||||
}
|
||||
newlogger.Write(bytes.Join(loggerSplit, []byte("\n")))
|
||||
} else {
|
||||
newlogger.Write(logger.Bytes())
|
||||
}
|
||||
|
||||
lsf := DstLsf(ctx, Fremote)
|
||||
err := LoggerMatchesLsf(&newlogger, lsf)
|
||||
r := fstest.NewRun(t)
|
||||
if r.Flocal.Precision() == Fremote.Precision() && r.Flocal.Hashes().Contains(hash.MD5) && canTestHash {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user