rclone/fs/sync/sync_test.go

// Test sync/copy/move

package sync

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"runtime"
	"sort"
	"strings"
	"testing"
	"time"

	mutex "sync" // renamed as "sync" already in use

	_ "github.com/rclone/rclone/backend/all" // import all backends
	"github.com/rclone/rclone/cmd/bisync/bilib"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/accounting"
	"github.com/rclone/rclone/fs/filter"
	"github.com/rclone/rclone/fs/fserrors"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/fs/operations"
	"github.com/rclone/rclone/fstest"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/text/unicode/norm"
)

// Some times used in the tests
var (
	t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
	t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")
	t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")
)

// TestMain drives the tests
func TestMain(m *testing.M) {
	fstest.TestMain(m)
}

// Check dry run is working
func TestCopyWithDryRun(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
	r.Mkdir(ctx, r.Fremote)

	ci.DryRun = true
	ctx = predictDstFromLogger(ctx)
	err := CopyDir(ctx, r.Fremote, r.Flocal, false)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // error expected here because dry-run
	require.NoError(t, err)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t)
}

// Now without dry run
func TestCopy(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
	_, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2)
	if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
		require.NoError(t, err)
	}
	r.Mkdir(ctx, r.Fremote)

	ctx = predictDstFromLogger(ctx)
	err = CopyDir(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file1)

	// Check that the modtimes of the directories are as expected
	r.CheckDirectoryModTimes(t, "sub dir")
}

func TestCopyMetadata(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	ci.Metadata = true
	r := fstest.NewRun(t)
	features := r.Fremote.Features()

	if !features.ReadMetadata && !features.WriteMetadata && !features.UserMetadata &&
		!features.ReadDirMetadata && !features.WriteDirMetadata && !features.UserDirMetadata {
		t.Skip("Skipping as metadata not supported")
	}

	const content = "hello metadata world!"
	const dirPath = "metadata sub dir"
	const filePath = dirPath + "/hello metadata world"

	fileMetadata := fs.Metadata{
		// System metadata supported by all backends
		"mtime": t1.Format(time.RFC3339Nano),
		// User metadata
		"potato": "jersey",
	}

	dirMetadata := fs.Metadata{
		// System metadata supported by all backends
		"mtime": t2.Format(time.RFC3339Nano),
		// User metadata
		"potato": "king edward",
	}

	// Make the directory with metadata - may fall back to Mkdir
	_, err := operations.MkdirMetadata(ctx, r.Flocal, dirPath, dirMetadata)
	require.NoError(t, err)

	// Upload the file with metadata
	in := io.NopCloser(bytes.NewBufferString(content))
	_, err = operations.Rcat(ctx, r.Flocal, filePath, in, t1, fileMetadata)
	require.NoError(t, err)
	file1 := fstest.NewItem(filePath, content, t1)

	// Reset the time of the directory
	_, err = operations.SetDirModTime(ctx, r.Flocal, nil, dirPath, t2)
	if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
		require.NoError(t, err)
	}

	ctx = predictDstFromLogger(ctx)
	err = CopyDir(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file1)

	// Check that the modtimes of the directories are as expected
	r.CheckDirectoryModTimes(t, dirPath)

	// Check that the metadata on the directory and file is correct
	if features.ReadMetadata {
		fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewObject(ctx, t, r.Fremote, filePath), fileMetadata)
	}
	if features.ReadDirMetadata {
		fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, dirPath), dirMetadata)
	}
}

func TestCopyMissingDirectory(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	r.Mkdir(ctx, r.Fremote)

	nonExistingFs, err := fs.NewFs(ctx, "/non-existing")
	if err != nil {
		t.Fatal(err)
	}

	ctx = predictDstFromLogger(ctx)
	err = CopyDir(ctx, r.Fremote, nonExistingFs, false)
	require.Error(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
}

// Now with --no-traverse
func TestCopyNoTraverse(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	ci.NoTraverse = true

	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)

	ctx = predictDstFromLogger(ctx)
	err := CopyDir(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file1)
}

// Now with --check-first
func TestCopyCheckFirst(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	ci.CheckFirst = true

	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)

	ctx = predictDstFromLogger(ctx)
	err := CopyDir(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file1)
}

// Now with --no-traverse
func TestSyncNoTraverse(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	ci.NoTraverse = true

	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file1)
}

// Test copy with depth
func TestCopyWithDepth(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
	file2 := r.WriteFile("hello world2", "hello world2", t2)

	// Check the MaxDepth too
	ci.MaxDepth = 1

	ctx = predictDstFromLogger(ctx)
	err := CopyDir(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1, file2)
	r.CheckRemoteItems(t, file2)
}

// Test copy with files from
func testCopyWithFilesFrom(t *testing.T, noTraverse bool) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	file1 := r.WriteFile("potato2", "hello world", t1)
	file2 := r.WriteFile("hello world2", "hello world2", t2)

	// Set the --files-from equivalent
	f, err := filter.NewFilter(nil)
	require.NoError(t, err)
	require.NoError(t, f.AddFile("potato2"))
	require.NoError(t, f.AddFile("notfound"))

	// Change the active filter
	ctx = filter.ReplaceConfig(ctx, f)

	ci.NoTraverse = noTraverse

	ctx = predictDstFromLogger(ctx)
	err = CopyDir(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1, file2)
	r.CheckRemoteItems(t, file1)
}
func TestCopyWithFilesFrom(t *testing.T)              { testCopyWithFilesFrom(t, false) }
func TestCopyWithFilesFromAndNoTraverse(t *testing.T) { testCopyWithFilesFrom(t, true) }

// Test copy empty directories
func TestCopyEmptyDirectories(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
	_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
	require.NoError(t, err)
	r.Mkdir(ctx, r.Fremote)

	// Set the modtime on "sub dir" to something specific
	// Without this it fails on the CI and in VirtualBox with variances of up to 10mS
	_, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)
	require.NoError(t, err)

	ctx = predictDstFromLogger(ctx)
	err = CopyDir(ctx, r.Fremote, r.Flocal, true)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file1,
		},
		[]string{
			"sub dir",
			"sub dir2",
		},
	)

	// Check that the modtimes of the directories are as expected
	r.CheckDirectoryModTimes(t, "sub dir", "sub dir2")
}

// Test move empty directories
func TestMoveEmptyDirectories(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
	_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
	require.NoError(t, err)
	subDir := fstest.NewDirectory(ctx, t, r.Flocal, "sub dir")
	subDirT := subDir.ModTime(ctx)
	r.Mkdir(ctx, r.Fremote)

	ctx = predictDstFromLogger(ctx)
	err = MoveDir(ctx, r.Fremote, r.Flocal, false, true)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file1,
		},
		[]string{
			"sub dir",
			"sub dir2",
		},
	)

	// Check that the modtimes of the directories are as expected
	r.CheckDirectoryModTimes(t, "sub dir2")
	// Note that "sub dir" mod time is updated when file1 is deleted from it
	// So check it more manually
	got := fstest.NewDirectory(ctx, t, r.Fremote, "sub dir")
	fstest.CheckDirModTime(ctx, t, r.Fremote, got, subDirT)
}

// Test that --no-update-dir-modtime is working
func TestSyncNoUpdateDirModtime(t *testing.T) {
	r := fstest.NewRun(t)
	if r.Fremote.Features().DirSetModTime == nil {
		t.Skip("Skipping test as backend does not support DirSetModTime")
	}

	ctx, ci := fs.AddConfig(context.Background())
	ci.NoUpdateDirModTime = true
	const name = "sub dir no update dir modtime"

	// Set the modtime on name to something specific
	_, err := operations.MkdirModTime(ctx, r.Flocal, name, t1)
	require.NoError(t, err)

	// Create the remote directory with the current time
	require.NoError(t, r.Fremote.Mkdir(ctx, name))

	// Read its modification time
	wantT := fstest.NewDirectory(ctx, t, r.Fremote, name).ModTime(ctx)

	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, true)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteListing(
		t,
		[]fstest.Item{},
		[]string{
			name,
		},
	)

	// Read the new directory modification time - it should not have changed
	gotT := fstest.NewDirectory(ctx, t, r.Fremote, name).ModTime(ctx)
	fstest.AssertTimeEqualWithPrecision(t, name, wantT, gotT, r.Fremote.Precision())
}

// Test sync empty directories
func TestSyncEmptyDirectories(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
	_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
	require.NoError(t, err)

	// Set the modtime on "sub dir" to something specific
	// Without this it fails on the CI and in VirtualBox with variances of up to 10mS
	_, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)
	require.NoError(t, err)

	r.Mkdir(ctx, r.Fremote)

	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, true)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file1,
		},
		[]string{
			"sub dir",
			"sub dir2",
		},
	)

	// Check that the modtimes of the directories are as expected
	r.CheckDirectoryModTimes(t, "sub dir", "sub dir2")
}

// Test delayed mod time setting
func TestSyncSetDelayedModTimes(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)

	if !r.Fremote.Features().DirModTimeUpdatesOnWrite {
		t.Skip("Backend doesn't have DirModTimeUpdatesOnWrite set")
	}

	// Create directories without timestamps
	require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d1/e1/f1"))
	require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b2/c1/d1/e1/f1"))
	require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f1"))
	require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f2"))

	dirs := []string{
		"a1",
		"a1/b1",
		"a1/b1/c1",
		"a1/b1/c1/d1",
		"a1/b1/c1/d1/e1",
		"a1/b1/c1/d1/e1/f1",
		"a1/b1/c1/d2",
		"a1/b1/c1/d2/e1",
		"a1/b1/c1/d2/e1/f1",
		"a1/b1/c1/d2/e1/f2",
		"a1/b2",
		"a1/b2/c1",
		"a1/b2/c1/d1",
		"a1/b2/c1/d1/e1",
		"a1/b2/c1/d1/e1/f1",
	}
	r.CheckLocalListing(t, []fstest.Item{}, dirs)

	// Timestamp the directories in reverse order
	ts := t1
	for i := len(dirs) - 1; i >= 0; i-- {
		dir := dirs[i]
		_, err := operations.SetDirModTime(ctx, r.Flocal, nil, dir, ts)
		require.NoError(t, err)
		ts = ts.Add(time.Minute)
	}

	r.Mkdir(ctx, r.Fremote)

	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, true)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteListing(t, []fstest.Item{}, dirs)

	// Check that the modtimes of the directories are as expected
	r.CheckDirectoryModTimes(t, dirs...)
}

// Test a server-side copy if possible, or the backup path if not
func TestServerSideCopy(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1)
	r.CheckRemoteItems(t, file1)

	FremoteCopy, _, finaliseCopy, err := fstest.RandomRemote()
	require.NoError(t, err)
	defer finaliseCopy()
	t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy)

	ctx = predictDstFromLogger(ctx)
	err = CopyDir(ctx, FremoteCopy, r.Fremote, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	fstest.CheckItems(t, FremoteCopy, file1)
}

// Check that if the local file doesn't exist when we copy it up,
// nothing happens to the remote file
func TestCopyAfterDelete(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1)
	r.CheckLocalItems(t)
	r.CheckRemoteItems(t, file1)

	err := operations.Mkdir(ctx, r.Flocal, "")
	require.NoError(t, err)

	ctx = predictDstFromLogger(ctx)
	err = CopyDir(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t)
	r.CheckRemoteItems(t, file1)
}

// Check the copy downloading a file
func TestCopyRedownload(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1)
	r.CheckRemoteItems(t, file1)

	ctx = predictDstFromLogger(ctx)
	err := CopyDir(ctx, r.Flocal, r.Fremote, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// Test with combined precision of local and remote as we copied it there and back
	r.CheckLocalListing(t, []fstest.Item{file1}, nil)
}

// Create a file and sync it. Change the last modified date and resync.
// If we're only doing sync by size and checksum, we expect nothing to
// to be transferred on the second sync.
func TestSyncBasedOnCheckSum(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	ci.CheckSum = true

	file1 := r.WriteFile("check sum", "-", t1)
	r.CheckLocalItems(t, file1)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// We should have transferred exactly one file.
	assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())
	r.CheckRemoteItems(t, file1)

	// Change last modified date only
	file2 := r.WriteFile("check sum", "-", t2)
	r.CheckLocalItems(t, file2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// We should have transferred no files
	assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())
	r.CheckLocalItems(t, file2)
	r.CheckRemoteItems(t, file1)
}

// Create a file and sync it. Change the last modified date and the
// file contents but not the size.  If we're only doing sync by size
// only, we expect nothing to to be transferred on the second sync.
func TestSyncSizeOnly(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	ci.SizeOnly = true

	file1 := r.WriteFile("sizeonly", "potato", t1)
	r.CheckLocalItems(t, file1)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// We should have transferred exactly one file.
	assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())
	r.CheckRemoteItems(t, file1)

	// Update mtime, md5sum but not length of file
	file2 := r.WriteFile("sizeonly", "POTATO", t2)
	r.CheckLocalItems(t, file2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// We should have transferred no files
	assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())
	r.CheckLocalItems(t, file2)
	r.CheckRemoteItems(t, file1)
}

// Create a file and sync it. Keep the last modified date but change
// the size.  With --ignore-size we expect nothing to to be
// transferred on the second sync.
func TestSyncIgnoreSize(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	ci.IgnoreSize = true

	file1 := r.WriteFile("ignore-size", "contents", t1)
	r.CheckLocalItems(t, file1)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// We should have transferred exactly one file.
	assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())
	r.CheckRemoteItems(t, file1)

	// Update size but not date of file
	file2 := r.WriteFile("ignore-size", "longer contents but same date", t1)
	r.CheckLocalItems(t, file2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// We should have transferred no files
	assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())
	r.CheckLocalItems(t, file2)
	r.CheckRemoteItems(t, file1)
}

func TestSyncIgnoreTimes(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	file1 := r.WriteBoth(ctx, "existing", "potato", t1)
	r.CheckRemoteItems(t, file1)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// We should have transferred exactly 0 files because the
	// files were identical.
	assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())

	ci.IgnoreTimes = true

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// We should have transferred exactly one file even though the
	// files were identical.
	assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file1)
}

func TestSyncIgnoreExisting(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	file1 := r.WriteFile("existing", "potato", t1)

	ci.IgnoreExisting = true

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file1)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// Change everything
	r.WriteFile("existing", "newpotatoes", t2)
	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	// Items should not change
	r.CheckRemoteItems(t, file1)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
}

func TestSyncIgnoreErrors(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	ci.IgnoreErrors = true
	file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
	file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2)
	file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2)
	require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d"))

	r.CheckLocalListing(
		t,
		[]fstest.Item{
			file1,
			file3,
		},
		[]string{
			"a",
			"c",
		},
	)
	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file2,
			file3,
		},
		[]string{
			"b",
			"c",
			"d",
		},
	)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	_ = fs.CountError(errors.New("boom"))
	assert.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalListing(
		t,
		[]fstest.Item{
			file1,
			file3,
		},
		[]string{
			"a",
			"c",
		},
	)
	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file1,
			file3,
		},
		[]string{
			"a",
			"c",
		},
	)
}

func TestSyncAfterChangingModtimeOnly(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	file1 := r.WriteFile("empty space", "-", t2)
	file2 := r.WriteObject(ctx, "empty space", "-", t1)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file2)

	ci.DryRun = true

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file2)

	ci.DryRun = false

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file1)
}

func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	if r.Fremote.Hashes().Count() == 0 {
		t.Logf("Can't check this if no hashes supported")
		return
	}

	ci.NoUpdateModTime = true

	file1 := r.WriteFile("empty space", "-", t2)
	file2 := r.WriteObject(ctx, "empty space", "-", t1)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file2)
}

func TestSyncDoesntUpdateModtime(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	if fs.GetModifyWindow(ctx, r.Fremote) == fs.ModTimeNotSupported {
		t.Skip("Can't run this test on fs which doesn't support mod time")
	}

	file1 := r.WriteFile("foo", "foo", t2)
	file2 := r.WriteObject(ctx, "foo", "bar", t1)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file1)

	// We should have transferred exactly one file, not set the mod time
	assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())
}

func TestSyncAfterAddingAFile(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteBoth(ctx, "empty space", "-", t2)
	file2 := r.WriteFile("potato", "------------------------------------------------------------", t3)

	r.CheckLocalItems(t, file1, file2)
	r.CheckRemoteItems(t, file1)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckLocalItems(t, file1, file2)
	r.CheckRemoteItems(t, file1, file2)
}

func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteObject(ctx, "potato", "------------------------------------------------------------", t3)
	file2 := r.WriteFile("potato", "smaller but same date", t3)
	r.CheckRemoteItems(t, file1)
	r.CheckLocalItems(t, file2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckLocalItems(t, file2)
	r.CheckRemoteItems(t, file2)
}

// Sync after changing a file's contents, changing modtime but length
// remaining the same
func TestSyncAfterChangingContentsOnly(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	var file1 fstest.Item
	if r.Fremote.Precision() == fs.ModTimeNotSupported {
		t.Logf("ModTimeNotSupported so forcing file to be a different size")
		file1 = r.WriteObject(ctx, "potato", "different size to make sure it syncs", t3)
	} else {
		file1 = r.WriteObject(ctx, "potato", "smaller but same date", t3)
	}
	file2 := r.WriteFile("potato", "SMALLER BUT SAME DATE", t2)
	r.CheckRemoteItems(t, file1)
	r.CheckLocalItems(t, file2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckLocalItems(t, file2)
	r.CheckRemoteItems(t, file2)
}

// Sync after removing a file and adding a file --dry-run
func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
	file2 := r.WriteObject(ctx, "potato", "SMALLER BUT SAME DATE", t2)
	file3 := r.WriteBoth(ctx, "empty space", "-", t2)

	ci.DryRun = true
	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	ci.DryRun = false
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalItems(t, file3, file1)
	r.CheckRemoteItems(t, file3, file2)
}

// Sync after removing a file and adding a file
func testSyncAfterRemovingAFileAndAddingAFile(ctx context.Context, t *testing.T) {
	r := fstest.NewRun(t)
	file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
	file2 := r.WriteObject(ctx, "potato", "SMALLER BUT SAME DATE", t2)
	file3 := r.WriteBoth(ctx, "empty space", "-", t2)
	r.CheckRemoteItems(t, file2, file3)
	r.CheckLocalItems(t, file1, file3)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckLocalItems(t, file1, file3)
	r.CheckRemoteItems(t, file1, file3)
}

func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
	testSyncAfterRemovingAFileAndAddingAFile(context.Background(), t)
}

// Sync after removing a file and adding a file
func testSyncAfterRemovingAFileAndAddingAFileSubDir(ctx context.Context, t *testing.T) {
	r := fstest.NewRun(t)
	file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
	file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2)
	file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2)
	require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d"))
	require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d/e"))

	r.CheckLocalListing(
		t,
		[]fstest.Item{
			file1,
			file3,
		},
		[]string{
			"a",
			"c",
		},
	)
	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file2,
			file3,
		},
		[]string{
			"b",
			"c",
			"d",
			"d/e",
		},
	)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalListing(
		t,
		[]fstest.Item{
			file1,
			file3,
		},
		[]string{
			"a",
			"c",
		},
	)
	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file1,
			file3,
		},
		[]string{
			"a",
			"c",
		},
	)
}

func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) {
	testSyncAfterRemovingAFileAndAddingAFileSubDir(context.Background(), t)
}

// Sync after removing a file and adding a file with IO Errors
func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
	file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2)
	file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2)
	require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d"))

	r.CheckLocalListing(
		t,
		[]fstest.Item{
			file1,
			file3,
		},
		[]string{
			"a",
			"c",
		},
	)
	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file2,
			file3,
		},
		[]string{
			"b",
			"c",
			"d",
		},
	)

	ctx = predictDstFromLogger(ctx)
	accounting.GlobalStats().ResetCounters()
	_ = fs.CountError(errors.New("boom"))
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	assert.Equal(t, fs.ErrorNotDeleting, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalListing(
		t,
		[]fstest.Item{
			file1,
			file3,
		},
		[]string{
			"a",
			"c",
		},
	)
	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file1,
			file2,
			file3,
		},
		[]string{
			"a",
			"b",
			"c",
			"d",
		},
	)
}

// Sync test delete after
func TestSyncDeleteAfter(t *testing.T) {
	ctx := context.Background()
	ci := fs.GetConfig(ctx)
	// This is the default so we've checked this already
	// check it is the default
	require.Equal(t, ci.DeleteMode, fs.DeleteModeAfter, "Didn't default to --delete-after")
}

// Sync test delete during
func TestSyncDeleteDuring(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	ci.DeleteMode = fs.DeleteModeDuring

	testSyncAfterRemovingAFileAndAddingAFile(ctx, t)
}

// Sync test delete before
func TestSyncDeleteBefore(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	ci.DeleteMode = fs.DeleteModeBefore

	testSyncAfterRemovingAFileAndAddingAFile(ctx, t)
}

// Copy test delete before - shouldn't delete anything
func TestCopyDeleteBefore(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	ci.DeleteMode = fs.DeleteModeBefore

	file1 := r.WriteObject(ctx, "potato", "hopefully not deleted", t1)
	file2 := r.WriteFile("potato2", "hopefully copied in", t1)
	r.CheckRemoteItems(t, file1)
	r.CheckLocalItems(t, file2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := CopyDir(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteItems(t, file1, file2)
	r.CheckLocalItems(t, file2)
}

// Test with exclude
func TestSyncWithExclude(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
	file2 := r.WriteBoth(ctx, "empty space", "-", t2)
	file3 := r.WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
	r.CheckRemoteItems(t, file1, file2)
	r.CheckLocalItems(t, file1, file2, file3)

	fi, err := filter.NewFilter(nil)
	require.NoError(t, err)
	fi.Opt.MaxSize = 40
	ctx = filter.ReplaceConfig(ctx, fi)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckRemoteItems(t, file2, file1)

	// Now sync the other way round and check enormous doesn't get
	// deleted as it is excluded from the sync
	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Flocal, r.Fremote, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckLocalItems(t, file2, file1, file3)
}

// Test with exclude and delete excluded
func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) // 60 bytes
	file2 := r.WriteBoth(ctx, "empty space", "-", t2)
	file3 := r.WriteBoth(ctx, "enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
	r.CheckRemoteItems(t, file1, file2, file3)
	r.CheckLocalItems(t, file1, file2, file3)

	fi, err := filter.NewFilter(nil)
	require.NoError(t, err)
	fi.Opt.MaxSize = 40
	fi.Opt.DeleteExcluded = true
	ctx = filter.ReplaceConfig(ctx, fi)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckRemoteItems(t, file2)

	// Check sync the other way round to make sure enormous gets
	// deleted even though it is excluded
	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Flocal, r.Fremote, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckLocalItems(t, file2)
}

// Test with UpdateOlder set
func TestSyncWithUpdateOlder(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	if fs.GetModifyWindow(ctx, r.Fremote) == fs.ModTimeNotSupported {
		t.Skip("Can't run this test on fs which doesn't support mod time")
	}
	t2plus := t2.Add(time.Second / 2)
	t2minus := t2.Add(time.Second / 2)
	oneF := r.WriteFile("one", "one", t1)
	twoF := r.WriteFile("two", "two", t3)
	threeF := r.WriteFile("three", "three", t2)
	fourF := r.WriteFile("four", "four", t2)
	fiveF := r.WriteFile("five", "five", t2)
	r.CheckLocalItems(t, oneF, twoF, threeF, fourF, fiveF)
	oneO := r.WriteObject(ctx, "one", "ONE", t2)
	twoO := r.WriteObject(ctx, "two", "TWO", t2)
	threeO := r.WriteObject(ctx, "three", "THREE", t2plus)
	fourO := r.WriteObject(ctx, "four", "FOURFOUR", t2minus)
	r.CheckRemoteItems(t, oneO, twoO, threeO, fourO)

	ci.UpdateOlder = true
	ci.ModifyWindow = fs.ModTimeNotSupported

	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	r.CheckRemoteItems(t, oneO, twoF, threeO, fourF, fiveF)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // no modtime

	if r.Fremote.Hashes().Count() == 0 {
		t.Logf("Skip test with --checksum as no hashes supported")
		return
	}

	// now enable checksum
	ci.CheckSum = true

	err = Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	r.CheckRemoteItems(t, oneO, twoF, threeF, fourF, fiveF)
}

// Test with a max transfer duration
func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	if *fstest.RemoteName != "" {
		t.Skip("Skipping test on non local remote")
	}
	r := fstest.NewRun(t)

	maxDuration := 250 * time.Millisecond
	ci.MaxDuration = maxDuration
	ci.CutoffMode = cutoffMode
	ci.CheckFirst = true
	ci.OrderBy = "size"
	ci.Transfers = 1
	ci.Checkers = 1
	bytesPerSecond := 10 * 1024
	accounting.TokenBucket.SetBwLimit(fs.BwPair{Tx: fs.SizeSuffix(bytesPerSecond), Rx: fs.SizeSuffix(bytesPerSecond)})
	defer accounting.TokenBucket.SetBwLimit(fs.BwPair{Tx: -1, Rx: -1})

	// write one small file which we expect to transfer and one big one which we don't
	file1 := r.WriteFile("file1", string(make([]byte, 16)), t1)
	file2 := r.WriteFile("file2", string(make([]byte, 50*1024)), t1)
	r.CheckLocalItems(t, file1, file2)
	r.CheckRemoteItems(t)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx) // not currently supported (but tests do pass for CutoffModeSoft)
	startTime := time.Now()
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	require.True(t, errors.Is(err, ErrorMaxDurationReached))

	if cutoffMode == fs.CutoffModeHard {
		r.CheckRemoteItems(t, file1)
		assert.Equal(t, int64(1), accounting.GlobalStats().GetTransfers())
	} else {
		r.CheckRemoteItems(t, file1, file2)
		assert.Equal(t, int64(2), accounting.GlobalStats().GetTransfers())
	}

	elapsed := time.Since(startTime)
	const maxTransferTime = 20 * time.Second

	what := fmt.Sprintf("expecting elapsed time %v between %v and %v", elapsed, maxDuration, maxTransferTime)
	assert.True(t, elapsed >= maxDuration, what)
	assert.True(t, elapsed < maxTransferTime, what)
}

func TestSyncWithMaxDuration(t *testing.T) {
	t.Run("Hard", func(t *testing.T) {
		testSyncWithMaxDuration(t, fs.CutoffModeHard)
	})
	t.Run("Soft", func(t *testing.T) {
		testSyncWithMaxDuration(t, fs.CutoffModeSoft)
	})
}

// Test with TrackRenames set
func TestSyncWithTrackRenames(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	ci.TrackRenames = true
	defer func() {
		ci.TrackRenames = false
	}()

	haveHash := r.Fremote.Hashes().Overlap(r.Flocal.Hashes()).GetOne() != hash.None
	canTrackRenames := haveHash && operations.CanServerSideMove(r.Fremote)
	t.Logf("Can track renames: %v", canTrackRenames)

	f1 := r.WriteFile("potato", "Potato Content", t1)
	f2 := r.WriteFile("yam", "Yam Content", t2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteItems(t, f1, f2)
	r.CheckLocalItems(t, f1, f2)

	// Now rename locally.
	f2 = r.RenameFile(f2, "yaml")

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteItems(t, f1, f2)

	// Check we renamed something if we should have
	if canTrackRenames {
		renames := accounting.GlobalStats().Renames(0)
		assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames))
	}
}

func TestParseRenamesStrategyModtime(t *testing.T) {
	for _, test := range []struct {
		in      string
		want    trackRenamesStrategy
		wantErr bool
	}{
		{"", 0, false},
		{"modtime", trackRenamesStrategyModtime, false},
		{"hash", trackRenamesStrategyHash, false},
		{"size", 0, false},
		{"modtime,hash", trackRenamesStrategyModtime | trackRenamesStrategyHash, false},
		{"hash,modtime,size", trackRenamesStrategyModtime | trackRenamesStrategyHash, false},
		{"size,boom", 0, true},
	} {
		got, err := parseTrackRenamesStrategy(test.in)
		assert.Equal(t, test.want, got, test.in)
		assert.Equal(t, test.wantErr, err != nil, test.in)
	}
}

func TestRenamesStrategyModtime(t *testing.T) {
	both := trackRenamesStrategyHash | trackRenamesStrategyModtime
	hash := trackRenamesStrategyHash
	modTime := trackRenamesStrategyModtime

	assert.True(t, both.hash())
	assert.True(t, both.modTime())
	assert.True(t, hash.hash())
	assert.False(t, hash.modTime())
	assert.False(t, modTime.hash())
	assert.True(t, modTime.modTime())
}

func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	ci.TrackRenames = true
	ci.TrackRenamesStrategy = "modtime"

	canTrackRenames := operations.CanServerSideMove(r.Fremote) && r.Fremote.Precision() != fs.ModTimeNotSupported
	t.Logf("Can track renames: %v", canTrackRenames)

	f1 := r.WriteFile("potato", "Potato Content", t1)
	f2 := r.WriteFile("yam", "Yam Content", t2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteItems(t, f1, f2)
	r.CheckLocalItems(t, f1, f2)

	// Now rename locally.
	f2 = r.RenameFile(f2, "yaml")

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteItems(t, f1, f2)

	// Check we renamed something if we should have
	if canTrackRenames {
		renames := accounting.GlobalStats().Renames(0)
		assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames))
	}
}

func TestSyncWithTrackRenamesStrategyLeaf(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	ci.TrackRenames = true
	ci.TrackRenamesStrategy = "leaf"

	canTrackRenames := operations.CanServerSideMove(r.Fremote) && r.Fremote.Precision() != fs.ModTimeNotSupported
	t.Logf("Can track renames: %v", canTrackRenames)

	f1 := r.WriteFile("potato", "Potato Content", t1)
	f2 := r.WriteFile("sub/yam", "Yam Content", t2)

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteItems(t, f1, f2)
	r.CheckLocalItems(t, f1, f2)

	// Now rename locally.
	f2 = r.RenameFile(f2, "yam")

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckRemoteItems(t, f1, f2)

	// Check we renamed something if we should have
	if canTrackRenames {
		renames := accounting.GlobalStats().Renames(0)
		assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames))
	}
}

func toyFileTransfers(r *fstest.Run) int64 {
	remote := r.Fremote.Name()
	transfers := 1
	if strings.HasPrefix(remote, "TestChunker") && strings.HasSuffix(remote, "S3") {
		transfers++ // Extra Copy because S3 emulates Move as Copy+Delete.
	}
	return int64(transfers)
}

// Test a server-side move if possible, or the backup path if not
func testServerSideMove(ctx context.Context, t *testing.T, r *fstest.Run, withFilter, testDeleteEmptyDirs bool) {
	FremoteMove, _, finaliseMove, err := fstest.RandomRemote()
	require.NoError(t, err)
	defer finaliseMove()

	file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
	file2 := r.WriteBoth(ctx, "empty space", "-", t2)
	file3u := r.WriteBoth(ctx, "potato3", "------------------------------------------------------------ UPDATED", t2)

	if testDeleteEmptyDirs {
		err := operations.Mkdir(ctx, r.Fremote, "tomatoDir")
		require.NoError(t, err)
	}

	r.CheckRemoteItems(t, file2, file1, file3u)

	t.Logf("Server side move (if possible) %v -> %v", r.Fremote, FremoteMove)

	// Write just one file in the new remote
	r.WriteObjectTo(ctx, FremoteMove, "empty space", "-", t2, false)
	file3 := r.WriteObjectTo(ctx, FremoteMove, "potato3", "------------------------------------------------------------", t1, false)
	fstest.CheckItems(t, FremoteMove, file2, file3)

	// Do server-side move
	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx) // not currently supported -- doesn't list all contents of dir.
	err = MoveDir(ctx, FremoteMove, r.Fremote, testDeleteEmptyDirs, false)
	require.NoError(t, err)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	if withFilter {
		r.CheckRemoteItems(t, file2)
	} else {
		r.CheckRemoteItems(t)
	}

	if testDeleteEmptyDirs {
		r.CheckRemoteListing(t, nil, []string{})
	}

	fstest.CheckItems(t, FremoteMove, file2, file1, file3u)

	// Create a new empty remote for stuff to be moved into
	FremoteMove2, _, finaliseMove2, err := fstest.RandomRemote()
	require.NoError(t, err)
	defer finaliseMove2()

	if testDeleteEmptyDirs {
		err := operations.Mkdir(ctx, FremoteMove, "tomatoDir")
		require.NoError(t, err)
	}

	// Move it back to a new empty remote, dst does not exist this time
	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = MoveDir(ctx, FremoteMove2, FremoteMove, testDeleteEmptyDirs, false)
	require.NoError(t, err)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	if withFilter {
		fstest.CheckItems(t, FremoteMove2, file1, file3u)
		fstest.CheckItems(t, FremoteMove, file2)
	} else {
		fstest.CheckItems(t, FremoteMove2, file2, file1, file3u)
		fstest.CheckItems(t, FremoteMove)
	}

	if testDeleteEmptyDirs {
		fstest.CheckListingWithPrecision(t, FremoteMove, nil, []string{}, fs.GetModifyWindow(ctx, r.Fremote))
	}
}

// Test MoveDir on Local
func TestServerSideMoveLocal(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	f1 := r.WriteFile("dir1/file1.txt", "hello", t1)
	f2 := r.WriteFile("dir2/file2.txt", "hello again", t2)
	r.CheckLocalItems(t, f1, f2)

	dir1, err := fs.NewFs(ctx, r.Flocal.Root()+"/dir1")
	require.NoError(t, err)
	dir2, err := fs.NewFs(ctx, r.Flocal.Root()+"/dir2")
	require.NoError(t, err)
	err = MoveDir(ctx, dir2, dir1, false, false)
	require.NoError(t, err)
}

// Test move
func TestMoveWithDeleteEmptySrcDirs(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
	file2 := r.WriteFile("nested/sub dir/file", "nested", t1)
	r.Mkdir(ctx, r.Fremote)

	// run move with --delete-empty-src-dirs
	ctx = predictDstFromLogger(ctx)
	err := MoveDir(ctx, r.Fremote, r.Flocal, true, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalListing(
		t,
		nil,
		[]string{},
	)
	r.CheckRemoteItems(t, file1, file2)
}

func TestMoveWithoutDeleteEmptySrcDirs(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
	file2 := r.WriteFile("nested/sub dir/file", "nested", t1)
	r.Mkdir(ctx, r.Fremote)

	ctx = predictDstFromLogger(ctx)
	err := MoveDir(ctx, r.Fremote, r.Flocal, false, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	r.CheckLocalListing(
		t,
		nil,
		[]string{
			"sub dir",
			"nested",
			"nested/sub dir",
		},
	)
	r.CheckRemoteItems(t, file1, file2)
}

func TestMoveWithIgnoreExisting(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	file1 := r.WriteFile("existing", "potato", t1)
	file2 := r.WriteFile("existing-b", "tomato", t1)

	ci.IgnoreExisting = true

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err := MoveDir(ctx, r.Fremote, r.Flocal, false, false)
	require.NoError(t, err)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckLocalListing(
		t,
		[]fstest.Item{},
		[]string{},
	)
	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file1,
			file2,
		},
		[]string{},
	)

	// Recreate first file with modified content
	file1b := r.WriteFile("existing", "newpotatoes", t2)
	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = MoveDir(ctx, r.Fremote, r.Flocal, false, false)
	require.NoError(t, err)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	// Source items should still exist in modified state
	r.CheckLocalListing(
		t,
		[]fstest.Item{
			file1b,
		},
		[]string{},
	)
	// Dest items should not have changed
	r.CheckRemoteListing(
		t,
		[]fstest.Item{
			file1,
			file2,
		},
		[]string{},
	)
}

// Test a server-side move if possible, or the backup path if not
func TestServerSideMove(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	testServerSideMove(ctx, t, r, false, false)
}

// Test a server-side move if possible, or the backup path if not
func TestServerSideMoveWithFilter(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)

	fi, err := filter.NewFilter(nil)
	require.NoError(t, err)
	fi.Opt.MinSize = 40
	ctx = filter.ReplaceConfig(ctx, fi)

	testServerSideMove(ctx, t, r, true, false)
}

// Test a server-side move if possible
func TestServerSideMoveDeleteEmptySourceDirs(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)
	testServerSideMove(ctx, t, r, false, true)
}

// Test a server-side move with overlap
func TestServerSideMoveOverlap(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)

	if r.Fremote.Features().DirMove != nil {
		t.Skip("Skipping test as remote supports DirMove")
	}

	subRemoteName := r.FremoteName + "/rclone-move-test"
	FremoteMove, err := fs.NewFs(ctx, subRemoteName)
	require.NoError(t, err)

	file1 := r.WriteObject(ctx, "potato2", "------------------------------------------------------------", t1)
	r.CheckRemoteItems(t, file1)

	// Subdir move with no filters should return ErrorCantMoveOverlapping
	// ctx = predictDstFromLogger(ctx)
	err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)
	assert.EqualError(t, err, fs.ErrorOverlapping.Error())
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)

	// Now try with a filter which should also fail with ErrorCantMoveOverlapping
	fi, err := filter.NewFilter(nil)
	require.NoError(t, err)
	fi.Opt.MinSize = 40
	ctx = filter.ReplaceConfig(ctx, fi)

	// ctx = predictDstFromLogger(ctx)
	err = MoveDir(ctx, FremoteMove, r.Fremote, false, false)
	assert.EqualError(t, err, fs.ErrorOverlapping.Error())
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
}

// Test a sync with overlap
func TestSyncOverlap(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)

	subRemoteName := r.FremoteName + "/rclone-sync-test"
	FremoteSync, err := fs.NewFs(ctx, subRemoteName)
	require.NoError(t, err)

	checkErr := func(err error) {
		require.Error(t, err)
		assert.True(t, fserrors.IsFatalError(err))
		assert.Equal(t, fs.ErrorOverlapping.Error(), err.Error())
	}

	ctx = predictDstFromLogger(ctx)
	checkErr(Sync(ctx, FremoteSync, r.Fremote, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	ctx = predictDstFromLogger(ctx)
	checkErr(Sync(ctx, r.Fremote, FremoteSync, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	ctx = predictDstFromLogger(ctx)
	checkErr(Sync(ctx, r.Fremote, r.Fremote, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	ctx = predictDstFromLogger(ctx)
	checkErr(Sync(ctx, FremoteSync, FremoteSync, false))
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
}

// Test a sync with filtered overlap
func TestSyncOverlapWithFilter(t *testing.T) {
	ctx := context.Background()
	r := fstest.NewRun(t)

	fi, err := filter.NewFilter(nil)
	require.NoError(t, err)
	require.NoError(t, fi.Add(false, "/rclone-sync-test/"))
	require.NoError(t, fi.Add(false, "*/layer2/"))
	fi.Opt.ExcludeFile = []string{".ignore"}
	filterCtx := filter.ReplaceConfig(ctx, fi)

	subRemoteName := r.FremoteName + "/rclone-sync-test"
	FremoteSync, err := fs.NewFs(ctx, subRemoteName)
	require.NoError(t, FremoteSync.Mkdir(ctx, ""))
	require.NoError(t, err)

	subRemoteName2 := r.FremoteName + "/rclone-sync-test-include/layer2"
	FremoteSync2, err := fs.NewFs(ctx, subRemoteName2)
	require.NoError(t, FremoteSync2.Mkdir(ctx, ""))
	require.NoError(t, err)

	subRemoteName3 := r.FremoteName + "/rclone-sync-test-ignore-file"
	FremoteSync3, err := fs.NewFs(ctx, subRemoteName3)
	require.NoError(t, FremoteSync3.Mkdir(ctx, ""))
	require.NoError(t, err)
	r.WriteObject(context.Background(), "rclone-sync-test-ignore-file/.ignore", "-", t1)

	checkErr := func(err error) {
		require.Error(t, err)
		assert.True(t, fserrors.IsFatalError(err))
		assert.Equal(t, fs.ErrorOverlapping.Error(), err.Error())
		accounting.GlobalStats().ResetCounters()
	}

	checkNoErr := func(err error) {
		require.NoError(t, err)
	}

	accounting.GlobalStats().ResetCounters()
	filterCtx = predictDstFromLogger(filterCtx)
	checkNoErr(Sync(filterCtx, FremoteSync, r.Fremote, false))
	checkErr(Sync(ctx, FremoteSync, r.Fremote, false))
	checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(ctx, r.Fremote, FremoteSync, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(filterCtx, r.Fremote, r.Fremote, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(ctx, r.Fremote, r.Fremote, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(filterCtx, FremoteSync, FremoteSync, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(ctx, FremoteSync, FremoteSync, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)

	checkNoErr(Sync(filterCtx, FremoteSync2, r.Fremote, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(ctx, FremoteSync2, r.Fremote, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync2, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(ctx, r.Fremote, FremoteSync2, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(filterCtx, FremoteSync2, FremoteSync2, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(ctx, FremoteSync2, FremoteSync2, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)

	checkNoErr(Sync(filterCtx, FremoteSync3, r.Fremote, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(ctx, FremoteSync3, r.Fremote, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	// Destination is excluded so this test makes no sense
	// checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync3, false))
	checkErr(Sync(ctx, r.Fremote, FremoteSync3, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(filterCtx, FremoteSync3, FremoteSync3, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
	filterCtx = predictDstFromLogger(filterCtx)
	checkErr(Sync(ctx, FremoteSync3, FremoteSync3, false))
	testLoggerVsLsf(filterCtx, r.Fremote, operations.GetLoggerOpt(filterCtx).JSON, t)
}

// Test with CompareDest set
func TestSyncCompareDest(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	ci.CompareDest = []string{r.FremoteName + "/CompareDest"}

	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
	require.NoError(t, err)

	// check empty dest, empty compare
	file1 := r.WriteFile("one", "one", t1)
	r.CheckLocalItems(t, file1)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx) // not currently supported due to duplicate equal() checks
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)

	file1dst := file1
	file1dst.Path = "dst/one"

	r.CheckRemoteItems(t, file1dst)

	// check old dest, empty compare
	file1b := r.WriteFile("one", "onet2", t2)
	r.CheckRemoteItems(t, file1dst)
	r.CheckLocalItems(t, file1b)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)

	file1bdst := file1b
	file1bdst.Path = "dst/one"

	r.CheckRemoteItems(t, file1bdst)

	// check old dest, new compare
	file3 := r.WriteObject(ctx, "dst/one", "one", t1)
	file2 := r.WriteObject(ctx, "CompareDest/one", "onet2", t2)
	file1c := r.WriteFile("one", "onet2", t2)
	r.CheckRemoteItems(t, file2, file3)
	r.CheckLocalItems(t, file1c)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)

	r.CheckRemoteItems(t, file2, file3)

	// check empty dest, new compare
	file4 := r.WriteObject(ctx, "CompareDest/two", "two", t2)
	file5 := r.WriteFile("two", "two", t2)
	r.CheckRemoteItems(t, file2, file3, file4)
	r.CheckLocalItems(t, file1c, file5)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)

	r.CheckRemoteItems(t, file2, file3, file4)

	// check new dest, new compare
	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)

	r.CheckRemoteItems(t, file2, file3, file4)

	// Work out if we actually have hashes for uploaded files
	haveHash := false
	if ht := fdst.Hashes().GetOne(); ht != hash.None {
		file2obj, err := fdst.NewObject(ctx, "one")
		if err == nil {
			file2objHash, err := file2obj.Hash(ctx, ht)
			if err == nil {
				haveHash = file2objHash != ""
			}
		}
	}

	// check new dest, new compare, src timestamp differs
	//
	// we only check this if we the file we uploaded previously
	// actually has a hash otherwise the differing timestamp is
	// always copied.
	if haveHash {
		file5b := r.WriteFile("two", "two", t3)
		r.CheckLocalItems(t, file1c, file5b)

		accounting.GlobalStats().ResetCounters()
		// ctx = predictDstFromLogger(ctx)
		err = Sync(ctx, fdst, r.Flocal, false)
		// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
		require.NoError(t, err)

		r.CheckRemoteItems(t, file2, file3, file4)
	} else {
		t.Log("No hash on uploaded file so skipping compare timestamp test")
	}

	// check empty dest, old compare
	file5c := r.WriteFile("two", "twot3", t3)
	r.CheckRemoteItems(t, file2, file3, file4)
	r.CheckLocalItems(t, file1c, file5c)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)

	file5cdst := file5c
	file5cdst.Path = "dst/two"

	r.CheckRemoteItems(t, file2, file3, file4, file5cdst)
}

// Test with multiple CompareDest
func TestSyncMultipleCompareDest(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)
	precision := fs.GetModifyWindow(ctx, r.Fremote, r.Flocal)

	ci.CompareDest = []string{r.FremoteName + "/pre-dest1", r.FremoteName + "/pre-dest2"}

	// check empty dest, new compare
	fsrc1 := r.WriteFile("1", "1", t1)
	fsrc2 := r.WriteFile("2", "2", t1)
	fsrc3 := r.WriteFile("3", "3", t1)
	r.CheckLocalItems(t, fsrc1, fsrc2, fsrc3)

	fdest1 := r.WriteObject(ctx, "pre-dest1/1", "1", t1)
	fdest2 := r.WriteObject(ctx, "pre-dest2/2", "2", t1)
	r.CheckRemoteItems(t, fdest1, fdest2)

	accounting.GlobalStats().ResetCounters()
	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dest")
	require.NoError(t, err)
	// ctx = predictDstFromLogger(ctx)
	require.NoError(t, Sync(ctx, fdst, r.Flocal, false))
	// testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t)

	fdest3 := fsrc3
	fdest3.Path = "dest/3"

	fstest.CheckItemsWithPrecision(t, fdst, precision, fsrc3)
	r.CheckRemoteItems(t, fdest1, fdest2, fdest3)
}

// Test with CopyDest set
func TestSyncCopyDest(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	if r.Fremote.Features().Copy == nil {
		t.Skip("Skipping test as remote does not support server-side copy")
	}

	ci.CopyDest = []string{r.FremoteName + "/CopyDest"}

	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
	require.NoError(t, err)

	// check empty dest, empty copy
	file1 := r.WriteFile("one", "one", t1)
	r.CheckLocalItems(t, file1)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // not currently supported
	require.NoError(t, err)

	file1dst := file1
	file1dst.Path = "dst/one"

	r.CheckRemoteItems(t, file1dst)

	// check old dest, empty copy
	file1b := r.WriteFile("one", "onet2", t2)
	r.CheckRemoteItems(t, file1dst)
	r.CheckLocalItems(t, file1b)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)

	file1bdst := file1b
	file1bdst.Path = "dst/one"

	r.CheckRemoteItems(t, file1bdst)

	// check old dest, new copy, backup-dir

	ci.BackupDir = r.FremoteName + "/BackupDir"

	file3 := r.WriteObject(ctx, "dst/one", "one", t1)
	file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2)
	file1c := r.WriteFile("one", "onet2", t2)
	r.CheckRemoteItems(t, file2, file3)
	r.CheckLocalItems(t, file1c)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)

	file2dst := file2
	file2dst.Path = "dst/one"
	file3.Path = "BackupDir/one"

	r.CheckRemoteItems(t, file2, file2dst, file3)
	ci.BackupDir = ""

	// check empty dest, new copy
	file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2)
	file5 := r.WriteFile("two", "two", t2)
	r.CheckRemoteItems(t, file2, file2dst, file3, file4)
	r.CheckLocalItems(t, file1c, file5)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	err = Sync(ctx, fdst, r.Flocal, false)
	require.NoError(t, err)

	file4dst := file4
	file4dst.Path = "dst/two"

	r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)

	// check new dest, new copy
	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)

	r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)

	// check empty dest, old copy
	file6 := r.WriteObject(ctx, "CopyDest/three", "three", t2)
	file7 := r.WriteFile("three", "threet3", t3)
	r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6)
	r.CheckLocalItems(t, file1c, file5, file7)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, fdst, r.Flocal, false)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)

	file7dst := file7
	file7dst.Path = "dst/three"

	r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6, file7dst)
}

// Test with BackupDir set
func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeepExtension bool) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	if !operations.CanServerSideMove(r.Fremote) {
		t.Skip("Skipping test as remote does not support server-side move")
	}
	r.Mkdir(ctx, r.Fremote)

	if backupDir != "" {
		ci.BackupDir = r.FremoteName + "/" + backupDir
		backupDir += "/"
	} else {
		ci.BackupDir = ""
		backupDir = "dst/"
		// Exclude the suffix from the sync otherwise the sync
		// deletes the old backup files
		flt, err := filter.NewFilter(nil)
		require.NoError(t, err)
		require.NoError(t, flt.AddRule("- *"+suffix))
		// Change the active filter
		ctx = filter.ReplaceConfig(ctx, flt)
	}
	ci.Suffix = suffix
	ci.SuffixKeepExtension = suffixKeepExtension

	// Make the setup so we have one, two, three in the dest
	// and one (different), two (same) in the source
	file1 := r.WriteObject(ctx, "dst/one", "one", t1)
	file2 := r.WriteObject(ctx, "dst/two", "two", t1)
	file3 := r.WriteObject(ctx, "dst/three.txt", "three", t1)
	file2a := r.WriteFile("two", "two", t1)
	file1a := r.WriteFile("one", "oneA", t2)

	r.CheckRemoteItems(t, file1, file2, file3)
	r.CheckLocalItems(t, file1a, file2a)

	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
	require.NoError(t, err)

	accounting.GlobalStats().ResetCounters()
	err = Sync(ctx, fdst, r.Flocal, false)
	require.NoError(t, err)

	// one should be moved to the backup dir and the new one installed
	file1.Path = backupDir + "one" + suffix
	file1a.Path = "dst/one"
	// two should be unchanged
	// three should be moved to the backup dir
	if suffixKeepExtension {
		file3.Path = backupDir + "three" + suffix + ".txt"
	} else {
		file3.Path = backupDir + "three.txt" + suffix
	}

	r.CheckRemoteItems(t, file1, file2, file3, file1a)

	// Now check what happens if we do it again
	// Restore a different three and update one in the source
	file3a := r.WriteObject(ctx, "dst/three.txt", "threeA", t2)
	file1b := r.WriteFile("one", "oneBB", t3)
	r.CheckRemoteItems(t, file1, file2, file3, file1a, file3a)

	// This should delete three and overwrite one again, checking
	// the files got overwritten correctly in backup-dir
	accounting.GlobalStats().ResetCounters()
	err = Sync(ctx, fdst, r.Flocal, false)
	require.NoError(t, err)

	// one should be moved to the backup dir and the new one installed
	file1a.Path = backupDir + "one" + suffix
	file1b.Path = "dst/one"
	// two should be unchanged
	// three should be moved to the backup dir
	if suffixKeepExtension {
		file3a.Path = backupDir + "three" + suffix + ".txt"
	} else {
		file3a.Path = backupDir + "three.txt" + suffix
	}

	r.CheckRemoteItems(t, file1b, file2, file3a, file1a)
}
func TestSyncBackupDir(t *testing.T) {
	testSyncBackupDir(t, "backup", "", false)
}
func TestSyncBackupDirWithSuffix(t *testing.T) {
	testSyncBackupDir(t, "backup", ".bak", false)
}
func TestSyncBackupDirWithSuffixKeepExtension(t *testing.T) {
	testSyncBackupDir(t, "backup", "-2019-01-01", true)
}
func TestSyncBackupDirSuffixOnly(t *testing.T) {
	testSyncBackupDir(t, "", ".bak", false)
}

// Test with Suffix set
func testSyncSuffix(t *testing.T, suffix string, suffixKeepExtension bool) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	if !operations.CanServerSideMove(r.Fremote) {
		t.Skip("Skipping test as remote does not support server-side move")
	}
	r.Mkdir(ctx, r.Fremote)

	ci.Suffix = suffix
	ci.SuffixKeepExtension = suffixKeepExtension

	// Make the setup so we have one, two, three in the dest
	// and one (different), two (same) in the source
	file1 := r.WriteObject(ctx, "dst/one", "one", t1)
	file2 := r.WriteObject(ctx, "dst/two", "two", t1)
	file3 := r.WriteObject(ctx, "dst/three.txt", "three", t1)
	file2a := r.WriteFile("two", "two", t1)
	file1a := r.WriteFile("one", "oneA", t2)
	file3a := r.WriteFile("three.txt", "threeA", t1)

	r.CheckRemoteItems(t, file1, file2, file3)
	r.CheckLocalItems(t, file1a, file2a, file3a)

	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
	require.NoError(t, err)

	accounting.GlobalStats().ResetCounters()
	err = operations.CopyFile(ctx, fdst, r.Flocal, "one", "one")
	require.NoError(t, err)
	err = operations.CopyFile(ctx, fdst, r.Flocal, "two", "two")
	require.NoError(t, err)
	err = operations.CopyFile(ctx, fdst, r.Flocal, "three.txt", "three.txt")
	require.NoError(t, err)

	// one should be moved to the backup dir and the new one installed
	file1.Path = "dst/one" + suffix
	file1a.Path = "dst/one"
	// two should be unchanged
	// three should be moved to the backup dir
	if suffixKeepExtension {
		file3.Path = "dst/three" + suffix + ".txt"
	} else {
		file3.Path = "dst/three.txt" + suffix
	}
	file3a.Path = "dst/three.txt"

	r.CheckRemoteItems(t, file1, file2, file3, file1a, file3a)

	// Now check what happens if we do it again
	// Restore a different three and update one in the source
	file3b := r.WriteFile("three.txt", "threeBDifferentSize", t3)
	file1b := r.WriteFile("one", "oneBB", t3)
	r.CheckRemoteItems(t, file1, file2, file3, file1a, file3a)

	// This should delete three and overwrite one again, checking
	// the files got overwritten correctly in backup-dir
	accounting.GlobalStats().ResetCounters()
	err = operations.CopyFile(ctx, fdst, r.Flocal, "one", "one")
	require.NoError(t, err)
	err = operations.CopyFile(ctx, fdst, r.Flocal, "two", "two")
	require.NoError(t, err)
	err = operations.CopyFile(ctx, fdst, r.Flocal, "three.txt", "three.txt")
	require.NoError(t, err)

	// one should be moved to the backup dir and the new one installed
	file1a.Path = "dst/one" + suffix
	file1b.Path = "dst/one"
	// two should be unchanged
	// three should be moved to the backup dir
	if suffixKeepExtension {
		file3a.Path = "dst/three" + suffix + ".txt"
	} else {
		file3a.Path = "dst/three.txt" + suffix
	}
	file3b.Path = "dst/three.txt"

	r.CheckRemoteItems(t, file1b, file3b, file2, file3a, file1a)
}
func TestSyncSuffix(t *testing.T)              { testSyncSuffix(t, ".bak", false) }
func TestSyncSuffixKeepExtension(t *testing.T) { testSyncSuffix(t, "-2019-01-01", true) }

// Check we can sync two files with differing UTF-8 representations
func TestSyncUTFNorm(t *testing.T) {
	ctx := context.Background()
	if runtime.GOOS == "darwin" {
		t.Skip("Can't test UTF normalization on OS X")
	}

	r := fstest.NewRun(t)

	// Two strings with different unicode normalization (from OS X)
	Encoding1 := "Testêé"
	Encoding2 := "Testêé"
	assert.NotEqual(t, Encoding1, Encoding2)
	assert.Equal(t, norm.NFC.String(Encoding1), norm.NFC.String(Encoding2))

	file1 := r.WriteFile(Encoding1, "This is a test", t1)
	r.CheckLocalItems(t, file1)

	file2 := r.WriteObject(ctx, Encoding2, "This is a old test", t2)
	r.CheckRemoteItems(t, file2)

	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // can't test this on macOS
	require.NoError(t, err)

	// We should have transferred exactly one file, but kept the
	// normalized state of the file.
	assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())
	r.CheckLocalItems(t, file1)
	file1.Path = file2.Path
	r.CheckRemoteItems(t, file1)
}

// Test --immutable
func TestSyncImmutable(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	ci.Immutable = true

	// Create file on source
	file1 := r.WriteFile("existing", "potato", t1)
	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t)

	// Should succeed
	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	require.NoError(t, err)
	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file1)

	// Modify file data and timestamp on source
	file2 := r.WriteFile("existing", "tomatoes", t2)
	r.CheckLocalItems(t, file2)
	r.CheckRemoteItems(t, file1)

	// Should fail with ErrorImmutableModified and not modify local or remote files
	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	err = Sync(ctx, r.Fremote, r.Flocal, false)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	assert.EqualError(t, err, fs.ErrorImmutableModified.Error())
	r.CheckLocalItems(t, file2)
	r.CheckRemoteItems(t, file1)
}

// Test --ignore-case-sync
func TestSyncIgnoreCase(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	// Only test if filesystems are case sensitive
	if r.Fremote.Features().CaseInsensitive || r.Flocal.Features().CaseInsensitive {
		t.Skip("Skipping test as local or remote are case-insensitive")
	}

	ci.IgnoreCaseSync = true

	// Create files with different filename casing
	file1 := r.WriteFile("existing", "potato", t1)
	r.CheckLocalItems(t, file1)
	file2 := r.WriteObject(ctx, "EXISTING", "potato", t1)
	r.CheckRemoteItems(t, file2)

	// Should not copy files that are differently-cased but otherwise identical
	accounting.GlobalStats().ResetCounters()
	// ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) // can't test this on macOS
	require.NoError(t, err)
	r.CheckLocalItems(t, file1)
	r.CheckRemoteItems(t, file2)
}

// Test --fix-case
func TestFixCase(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	r := fstest.NewRun(t)

	// Only test if remote is case insensitive
	if !r.Fremote.Features().CaseInsensitive {
		t.Skip("Skipping test as local or remote are case-sensitive")
	}

	ci.FixCase = true

	// Create files with different filename casing
	file1a := r.WriteFile("existing", "potato", t1)
	file1b := r.WriteFile("existingbutdifferent", "donut", t1)
	file1c := r.WriteFile("subdira/subdirb/subdirc/hello", "donut", t1)
	file1d := r.WriteFile("subdira/subdirb/subdirc/subdird/filewithoutcasedifferences", "donut", t1)
	r.CheckLocalItems(t, file1a, file1b, file1c, file1d)
	file2a := r.WriteObject(ctx, "EXISTING", "potato", t1)
	file2b := r.WriteObject(ctx, "EXISTINGBUTDIFFERENT", "lemonade", t1)
	file2c := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/HELLO", "lemonade", t1)
	file2d := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/subdird/filewithoutcasedifferences", "lemonade", t1)
	r.CheckRemoteItems(t, file2a, file2b, file2c, file2d)

	// Should force rename of dest file that is differently-cased
	accounting.GlobalStats().ResetCounters()
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	require.NoError(t, err)
	r.CheckLocalItems(t, file1a, file1b, file1c, file1d)
	r.CheckRemoteItems(t, file1a, file1b, file1c, file1d)
}

// Test that aborting on --max-transfer works
func TestMaxTransfer(t *testing.T) {
	ctx := context.Background()
	ctx, ci := fs.AddConfig(ctx)
	ci.MaxTransfer = 3 * 1024
	ci.Transfers = 1
	ci.Checkers = 1
	ci.CutoffMode = fs.CutoffModeHard

	test := func(t *testing.T, cutoff fs.CutoffMode) {
		r := fstest.NewRun(t)
		ci.CutoffMode = cutoff

		if r.Fremote.Name() != "local" {
			t.Skip("This test only runs on local")
		}

		// Create file on source
		file1 := r.WriteFile("file1", string(make([]byte, 5*1024)), t1)
		file2 := r.WriteFile("file2", string(make([]byte, 2*1024)), t1)
		file3 := r.WriteFile("file3", string(make([]byte, 3*1024)), t1)
		r.CheckLocalItems(t, file1, file2, file3)
		r.CheckRemoteItems(t)

		accounting.GlobalStats().ResetCounters()

		// ctx = predictDstFromLogger(ctx) // not currently supported
		err := Sync(ctx, r.Fremote, r.Flocal, false)
		// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
		expectedErr := fserrors.FsError(accounting.ErrorMaxTransferLimitReachedFatal)
		if cutoff != fs.CutoffModeHard {
			expectedErr = accounting.ErrorMaxTransferLimitReachedGraceful
		}
		fserrors.Count(expectedErr)
		assert.Equal(t, expectedErr, err)
	}

	t.Run("Hard", func(t *testing.T) { test(t, fs.CutoffModeHard) })
	t.Run("Soft", func(t *testing.T) { test(t, fs.CutoffModeSoft) })
	t.Run("Cautious", func(t *testing.T) { test(t, fs.CutoffModeCautious) })
}

func testSyncConcurrent(t *testing.T, subtest string) {
	const (
		NFILES     = 20
		NCHECKERS  = 4
		NTRANSFERS = 4
	)

	ctx, ci := fs.AddConfig(context.Background())
	ci.Checkers = NCHECKERS
	ci.Transfers = NTRANSFERS

	r := fstest.NewRun(t)
	stats := accounting.GlobalStats()

	itemsBefore := []fstest.Item{}
	itemsAfter := []fstest.Item{}
	for i := 0; i < NFILES; i++ {
		nameBoth := fmt.Sprintf("both%d", i)
		nameOnly := fmt.Sprintf("only%d", i)
		switch subtest {
		case "delete":
			fileBoth := r.WriteBoth(ctx, nameBoth, "potato", t1)
			fileOnly := r.WriteObject(ctx, nameOnly, "potato", t1)
			itemsBefore = append(itemsBefore, fileBoth, fileOnly)
			itemsAfter = append(itemsAfter, fileBoth)
		case "truncate":
			fileBoth := r.WriteBoth(ctx, nameBoth, "potato", t1)
			fileFull := r.WriteObject(ctx, nameOnly, "potato", t1)
			fileEmpty := r.WriteFile(nameOnly, "", t1)
			itemsBefore = append(itemsBefore, fileBoth, fileFull)
			itemsAfter = append(itemsAfter, fileBoth, fileEmpty)
		}
	}

	r.CheckRemoteItems(t, itemsBefore...)
	stats.ResetErrors()
	ctx = predictDstFromLogger(ctx)
	err := Sync(ctx, r.Fremote, r.Flocal, false)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	if errors.Is(err, fs.ErrorCantUploadEmptyFiles) {
		t.Skipf("Skip test because remote cannot upload empty files")
	}
	assert.NoError(t, err, "Sync must not return a error")
	assert.False(t, stats.Errored(), "Low level errors must not have happened")
	r.CheckRemoteItems(t, itemsAfter...)
}

func TestSyncConcurrentDelete(t *testing.T) {
	testSyncConcurrent(t, "delete")
}

func TestSyncConcurrentTruncate(t *testing.T) {
	testSyncConcurrent(t, "truncate")
}

// Tests that nothing is transferred when src and dst already match
// Run the same sync twice, ensure no action is taken the second time
func TestNothingToTransfer(t *testing.T) {
	accounting.GlobalStats().ResetCounters()
	ctx, _ := fs.AddConfig(context.Background())
	r := fstest.NewRun(t)
	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
	file2 := r.WriteFile("sub dir2/very/very/very/very/very/nested/subdir/hello world", "hello world", t1)
	r.CheckLocalItems(t, file1, file2)
	_, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2)
	if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
		require.NoError(t, err)
	}
	r.Mkdir(ctx, r.Fremote)
	_, err = operations.MkdirModTime(ctx, r.Fremote, "sub dir", t3)
	require.NoError(t, err)

	// set logging
	// (this checks log output as DirModtime operations do not yet have stats, and r.CheckDirectoryModTimes also does not tell us what actions were taken)
	oldLogLevel := fs.GetConfig(context.Background()).LogLevel
	defer func() { fs.GetConfig(context.Background()).LogLevel = oldLogLevel }() // reset to old val after test
	// need to do this as fs.Infof only respects the globalConfig
	fs.GetConfig(context.Background()).LogLevel = fs.LogLevelInfo

	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	output := bilib.CaptureOutput(func() {
		err = CopyDir(ctx, r.Fremote, r.Flocal, true)
		require.NoError(t, err)
	})
	require.NotNil(t, output)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckLocalItems(t, file1, file2)
	r.CheckRemoteItems(t, file1, file2)
	// Check that the modtimes of the directories are as expected
	r.CheckDirectoryModTimes(t, "sub dir")

	// check that actions were taken
	assert.True(t, strings.Contains(string(output), "Copied"), `expected to find at least one "Copied" log: `+string(output))
	if r.Fremote.Features().DirSetModTime != nil || r.Fremote.Features().MkdirMetadata != nil {
		assert.True(t, strings.Contains(string(output), "Set directory modification time"), `expected to find at least one "Set directory modification time" log: `+string(output))
	}
	assert.False(t, strings.Contains(string(output), "There was nothing to transfer"), `expected to find no "There was nothing to transfer" logs, but found one: `+string(output))
	assert.Equal(t, int64(2), accounting.GlobalStats().GetTransfers())

	// run it again and make sure no actions were taken
	accounting.GlobalStats().ResetCounters()
	ctx = predictDstFromLogger(ctx)
	output = bilib.CaptureOutput(func() {
		err = CopyDir(ctx, r.Fremote, r.Flocal, true)
		require.NoError(t, err)
	})
	require.NotNil(t, output)
	testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
	r.CheckLocalItems(t, file1, file2)
	r.CheckRemoteItems(t, file1, file2)
	// Check that the modtimes of the directories are as expected
	r.CheckDirectoryModTimes(t, "sub dir")

	// check that actions were NOT taken
	assert.False(t, strings.Contains(string(output), "Copied"), `expected to find no "Copied" logs, but found one: `+string(output))
	if r.Fremote.Features().DirSetModTime != nil || r.Fremote.Features().MkdirMetadata != nil {
		assert.False(t, strings.Contains(string(output), "Set directory modification time"), `expected to find no "Set directory modification time" logs, but found one: `+string(output))
	}
	assert.True(t, strings.Contains(string(output), "There was nothing to transfer"), `expected to find a "There was nothing to transfer" log: `+string(output))
	assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())
}

// for testing logger:
func predictDstFromLogger(ctx context.Context) context.Context {
	opt := operations.NewLoggerOpt()
	var lock mutex.Mutex

	opt.LoggerFn = func(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
		lock.Lock()
		defer lock.Unlock()

		// ignore dirs for our purposes here
		if err == fs.ErrorIsDir {
			return
		}
		winner := operations.WinningSide(ctx, sigil, src, dst, err)
		if winner.Obj != nil {
			file := winner.Obj
			obj, ok := file.(fs.ObjectInfo)
			checksum := ""
			timeFormat := "2006-01-02 15:04:05"
			if ok {
				if obj.Fs().Hashes().GetOne() == hash.MD5 {
					// skip if no MD5
					checksum, _ = obj.Hash(ctx, hash.MD5)
				}
				timeFormat = operations.FormatForLSFPrecision(obj.Fs().Precision())
			}
			errMsg := ""
			if winner.Err != nil {
				errMsg = ";" + winner.Err.Error()
			}
			operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), file.Remote(), errMsg)
		}
	}
	return operations.WithSyncLogger(ctx, opt)
}

func DstLsf(ctx context.Context, Fremote fs.Fs) *bytes.Buffer {
	var opt = operations.ListJSONOpt{
		NoModTime:  false,
		NoMimeType: true,
		DirsOnly:   false,
		FilesOnly:  true,
		Recurse:    true,
		ShowHash:   true,
		HashTypes:  []string{"MD5"},
	}

	var list operations.ListFormat

	list.SetSeparator(";")
	timeFormat := operations.FormatForLSFPrecision(Fremote.Precision())
	list.AddModTime(timeFormat)
	list.AddHash(hash.MD5)
	list.AddSize()
	list.AddPath()

	out := new(bytes.Buffer)

	err := operations.ListJSON(ctx, Fremote, "", &opt, func(item *operations.ListJSONItem) error {
		_, _ = fmt.Fprintln(out, list.Format(item))
		return nil
	})
	if err != nil {
		fs.Errorf(Fremote, "ListJSON error: %v", err)
	}

	return out
}

func LoggerMatchesLsf(logger, lsf *bytes.Buffer) error {
	loggerSplit := bytes.Split(logger.Bytes(), []byte("\n"))
	sort.SliceStable(loggerSplit, func(i int, j int) bool { return string(loggerSplit[i]) < string(loggerSplit[j]) })
	lsfSplit := bytes.Split(lsf.Bytes(), []byte("\n"))
	sort.SliceStable(lsfSplit, func(i int, j int) bool { return string(lsfSplit[i]) < string(lsfSplit[j]) })

	loggerJoined := bytes.Join(loggerSplit, []byte("\n"))
	lsfJoined := bytes.Join(lsfSplit, []byte("\n"))

	if bytes.Equal(loggerJoined, lsfJoined) {
		return nil
	}
	Diff(string(loggerJoined), string(lsfJoined))
	return fmt.Errorf("logger does not match lsf! \nlogger: \n%s \nlsf: \n%s", loggerJoined, lsfJoined)
}

func Diff(rev1, rev2 string) {
	fmt.Printf("Diff of %q and %q\n", "logger", "lsf")
	cmd := exec.Command("bash", "-c", fmt.Sprintf(`diff <(echo "%s")  <(echo "%s")`, rev1, rev2))
	out, _ := cmd.Output()
	_, _ = os.Stdout.Write(out)
}

func testLoggerVsLsf(ctx context.Context, Fremote fs.Fs, logger *bytes.Buffer, t *testing.T) {
	var newlogger bytes.Buffer
	canTestModtime := fs.GetModifyWindow(ctx, Fremote) != fs.ModTimeNotSupported
	canTestHash := Fremote.Hashes().Contains(hash.MD5)
	if !canTestHash || !canTestModtime {
		loggerSplit := bytes.Split(logger.Bytes(), []byte("\n"))
		for i, line := range loggerSplit {
			elements := bytes.Split(line, []byte(";"))
			if len(elements) >= 2 {
				if !canTestModtime {
					elements[0] = []byte("")
				}
				if !canTestHash {
					elements[1] = []byte("")
				}
			}
			loggerSplit[i] = bytes.Join(elements, []byte(";"))
		}
		newlogger.Write(bytes.Join(loggerSplit, []byte("\n")))
	} else {
		newlogger.Write(logger.Bytes())
	}

	r := fstest.NewRun(t)
	if r.Flocal.Precision() == Fremote.Precision() && r.Flocal.Hashes().Contains(hash.MD5) && canTestHash {
		lsf := DstLsf(ctx, Fremote)
		err := LoggerMatchesLsf(&newlogger, lsf)
		require.NoError(t, err)
	}
}