mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 16:34:30 +01:00
sync: factor accumulating the rename checks
This commit is contained in:
parent
c3b2b89473
commit
86bec20b56
44
fs/sync.go
44
fs/sync.go
@ -41,6 +41,9 @@ type syncCopyMove struct {
|
|||||||
renameMap map[string][]Object // dst files by hash - only used by trackRenames
|
renameMap map[string][]Object // dst files by hash - only used by trackRenames
|
||||||
renamerWg sync.WaitGroup // wait for renamers
|
renamerWg sync.WaitGroup // wait for renamers
|
||||||
toBeRenamed ObjectPairChan // renamers channel
|
toBeRenamed ObjectPairChan // renamers channel
|
||||||
|
trackRenamesWg sync.WaitGroup // wg for background track renames
|
||||||
|
trackRenamesCh chan Object // objects are pumped in here
|
||||||
|
renameCheck []Object // accumulate files to check for rename here
|
||||||
backupDir Fs // place to store overwrites/deletes
|
backupDir Fs // place to store overwrites/deletes
|
||||||
suffix string // suffix to add to files placed in backupDir
|
suffix string // suffix to add to files placed in backupDir
|
||||||
}
|
}
|
||||||
@ -63,6 +66,7 @@ func newSyncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) (*syncCopyMove, er
|
|||||||
trackRenames: Config.TrackRenames,
|
trackRenames: Config.TrackRenames,
|
||||||
commonHash: fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(),
|
commonHash: fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(),
|
||||||
toBeRenamed: make(ObjectPairChan, Config.Transfers),
|
toBeRenamed: make(ObjectPairChan, Config.Transfers),
|
||||||
|
trackRenamesCh: make(chan Object, Config.Checkers),
|
||||||
}
|
}
|
||||||
if s.noTraverse && s.Delete {
|
if s.noTraverse && s.Delete {
|
||||||
Debug(s.fdst, "Ignoring --no-traverse with sync")
|
Debug(s.fdst, "Ignoring --no-traverse with sync")
|
||||||
@ -419,6 +423,29 @@ func (s *syncCopyMove) stopRenamers() {
|
|||||||
s.renamerWg.Wait()
|
s.renamerWg.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// This starts the collection of possible renames
|
||||||
|
func (s *syncCopyMove) startTrackRenames() {
|
||||||
|
if !s.trackRenames {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.trackRenamesWg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer s.trackRenamesWg.Done()
|
||||||
|
for o := range s.trackRenamesCh {
|
||||||
|
s.renameCheck = append(s.renameCheck, o)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// This stops the background rename collection
|
||||||
|
func (s *syncCopyMove) stopTrackRenames() {
|
||||||
|
if !s.trackRenames {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
close(s.trackRenamesCh)
|
||||||
|
s.trackRenamesWg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// This deletes the files in the dstFiles map. If checkSrcMap is set
|
// This deletes the files in the dstFiles map. If checkSrcMap is set
|
||||||
// then it checks to see if they exist first in srcFiles the source
|
// then it checks to see if they exist first in srcFiles the source
|
||||||
// file map, otherwise it unconditionally deletes them. If
|
// file map, otherwise it unconditionally deletes them. If
|
||||||
@ -491,13 +518,13 @@ func (s *syncCopyMove) popRenameMap(hash string) (dst Object) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// makeRenameMap builds a map of the destination files by hash that
|
// makeRenameMap builds a map of the destination files by hash that
|
||||||
// match sizes in the slice of objects in renameCheck
|
// match sizes in the slice of objects in s.renameCheck
|
||||||
func (s *syncCopyMove) makeRenameMap(renameCheck []Object) {
|
func (s *syncCopyMove) makeRenameMap() {
|
||||||
Debug(s.fdst, "Making map for --track-renames")
|
Debug(s.fdst, "Making map for --track-renames")
|
||||||
|
|
||||||
// first make a map of possible sizes we need to check
|
// first make a map of possible sizes we need to check
|
||||||
possibleSizes := map[int64]struct{}{}
|
possibleSizes := map[int64]struct{}{}
|
||||||
for _, obj := range renameCheck {
|
for _, obj := range s.renameCheck {
|
||||||
possibleSizes[obj.Size()] = struct{}{}
|
possibleSizes[obj.Size()] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -626,7 +653,7 @@ func (s *syncCopyMove) run() error {
|
|||||||
s.startTransfers()
|
s.startTransfers()
|
||||||
|
|
||||||
// Do the transfers
|
// Do the transfers
|
||||||
var renameCheck []Object
|
s.startTrackRenames()
|
||||||
for src := range s.srcFilesChan {
|
for src := range s.srcFilesChan {
|
||||||
remote := src.Remote()
|
remote := src.Remote()
|
||||||
var dst Object
|
var dst Object
|
||||||
@ -652,19 +679,20 @@ func (s *syncCopyMove) run() error {
|
|||||||
if dst != nil {
|
if dst != nil {
|
||||||
s.toBeChecked <- ObjectPair{src, dst}
|
s.toBeChecked <- ObjectPair{src, dst}
|
||||||
} else if s.trackRenames {
|
} else if s.trackRenames {
|
||||||
// save object until all matches transferred
|
// Save object to check for a rename later
|
||||||
renameCheck = append(renameCheck, src)
|
s.trackRenamesCh <- src
|
||||||
} else {
|
} else {
|
||||||
// No need to check since doesn't exist
|
// No need to check since doesn't exist
|
||||||
s.toBeUploaded <- ObjectPair{src, nil}
|
s.toBeUploaded <- ObjectPair{src, nil}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s.stopTrackRenames()
|
||||||
if s.trackRenames {
|
if s.trackRenames {
|
||||||
// Build the map of the remaining dstFiles by hash
|
// Build the map of the remaining dstFiles by hash
|
||||||
s.makeRenameMap(renameCheck)
|
s.makeRenameMap()
|
||||||
// Attempt renames for all the files which don't have a matching dst
|
// Attempt renames for all the files which don't have a matching dst
|
||||||
for _, src := range renameCheck {
|
for _, src := range s.renameCheck {
|
||||||
s.toBeRenamed <- ObjectPair{src, nil}
|
s.toBeRenamed <- ObjectPair{src, nil}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user