extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-26 10:25:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

sync: make --track-renames tests less fragile by using rename stat

Browse Source 
Before this change these tests attempted to measure transfers and
checks in lieu of having a rename statistic with a very complicated
heuristic.

The change switches over to using the rename statistic which should be
100% reliable.
This commit is contained in:
Nick Craig-Wood 2020-03-30 18:25:10 +01:00
parent b5f1bebc52
commit 5f9be3dd05
1 changed files with 6 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
fs/sync/sync_test.go
View File

@ -1067,64 +1067,9 @@ func TestSyncWithTrackRenames(t *testing.T) {
fstest.CheckItems(t, r.Fremote, f1, f2)
// As currently there is no Fs interface providing number of chunks
// in a file, this test depends on the well-known names of test remotes.
remote := r.Fremote.Name()
// Union remote can Move but returns CantMove error.
moveAsCopyDelete := r.Fremote.Features().Move == nil || remote == "TestUnion"
chunker := strings.HasPrefix(remote, "TestChunker")
wrappedMoveAsCopyDelete := chunker && strings.HasSuffix(remote, "S3")
chunk3b := chunker && strings.Contains(remote, "Chunk3b") // chunker with 3 byte chunks
chunk50b := chunker && strings.Contains(remote, "Chunk50b") // chunker with 50 byte chunks
chunkDefault := chunker && !strings.Contains(remote, "ChunkerChunk") // default big chunk size
chunkBig := chunk50b || chunkDefault // file is smaller than chunk size
// Verify number of checks for a toy 14 byte file.
// The order of cases matters!
var checks int
switch {
case canTrackRenames && chunk3b:
checks = 8 // chunker makes extra checks for each small chunk
case canTrackRenames && chunkBig:
checks = 4 // chunker makes 1 extra check for a single big chunk
case canTrackRenames && moveAsCopyDelete:
checks = 4 // 2 file checks + 1 move + 1 delete
case canTrackRenames:
checks = 3 // 2 file checks + 1 move
case !chunker:
checks = 2 // 2 file checks on a generic non-chunking remote
case chunk3b:
checks = 6 // chunker makes extra checks for each small chunk
case chunkBig && wrappedMoveAsCopyDelete:
checks = 4 // one more extra check because S3 emulates Move as Copy+Delete
case chunkBig:
checks = 3 // chunker makes 1 extra check for a single big chunk
default:
checks = -1 // skip verification for chunker with unknown chunk size
}
if checks != -1 { // "-1" allows remotes to bypass this check
assert.Equal(t, int64(checks), accounting.GlobalStats().GetChecks())
}
// Verify number of copy operations for a toy 14 byte file.
// The order of cases matters!
var copies int64
switch {
case canTrackRenames && moveAsCopyDelete:
copies = 1 // 1 copy
case canTrackRenames:
copies = 0 // 0 copy
case chunkBig && wrappedMoveAsCopyDelete:
copies = 2 // extra Copy because S3 emulates Move as Copy+Delete.
default:
copies = 1
}
if copies != -1 { // "-1" allows remotes to bypass this check
assert.Equal(t, copies, accounting.GlobalStats().GetTransfers())
}
// Check we renamed something
renames := accounting.GlobalStats().Renames(0)
assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames))
}
func TestParseRenamesStrategyModtime(t *testing.T) {
@ -1191,64 +1136,9 @@ func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) {
fstest.CheckItems(t, r.Fremote, f1, f2)
// As currently there is no Fs interface providing number of chunks
// in a file, this test depends on the well-known names of test remotes.
remote := r.Fremote.Name()
// Union remote can Move but returns CantMove error.
moveAsCopyDelete := r.Fremote.Features().Move == nil || remote == "TestUnion"
chunker := strings.HasPrefix(remote, "TestChunker")
wrappedMoveAsCopyDelete := chunker && strings.HasSuffix(remote, "S3")
chunk3b := chunker && strings.Contains(remote, "Chunk3b") // chunker with 3 byte chunks
chunk50b := chunker && strings.Contains(remote, "Chunk50b") // chunker with 50 byte chunks
chunkDefault := chunker && !strings.Contains(remote, "ChunkerChunk") // default big chunk size
chunkBig := chunk50b || chunkDefault // file is smaller than chunk size
// Verify number of checks for a toy 14 byte file.
// The order of cases matters!
var checks int
switch {
case canTrackRenames && chunk3b:
checks = 8 // chunker makes extra checks for each small chunk
case canTrackRenames && chunkBig:
checks = 4 // chunker makes 1 extra check for a single big chunk
case canTrackRenames && moveAsCopyDelete:
checks = 4 // 2 file checks + 1 move + 1 delete
case canTrackRenames:
checks = 3 // 2 file checks + 1 move
case !chunker:
checks = 2 // 2 file checks on a generic non-chunking remote
case chunk3b:
checks = 6 // chunker makes extra checks for each small chunk
case chunkBig && wrappedMoveAsCopyDelete:
checks = 4 // one more extra check because S3 emulates Move as Copy+Delete
case chunkBig:
checks = 3 // chunker makes 1 extra check for a single big chunk
default:
checks = -1 // skip verification for chunker with unknown chunk size
}
if checks != -1 { // "-1" allows remotes to bypass this check
assert.Equal(t, int64(checks), accounting.GlobalStats().GetChecks())
}
// Verify number of copy operations for a toy 14 byte file.
// The order of cases matters!
var copies int64
switch {
case canTrackRenames && moveAsCopyDelete:
copies = 1 // 1 copy
case canTrackRenames:
copies = 0 // 0 copy
case chunkBig && wrappedMoveAsCopyDelete:
copies = 2 // extra Copy because S3 emulates Move as Copy+Delete.
default:
copies = 1
}
if copies != -1 { // "-1" allows remotes to bypass this check
assert.Equal(t, copies, accounting.GlobalStats().GetTransfers())
}
// Check we renamed something
renames := accounting.GlobalStats().Renames(0)
assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames))
}
func toyFileTransfers(r *fstest.Run) int64 {