mirror of
https://github.com/rclone/rclone.git
synced 2024-12-22 23:22:08 +01:00
Make download work properly skipping not changed files
Reorganise FsObject and factor more stuff into methods Note that os.Chtimes() isn't setting the same precision (us) as we are reading from os.Lstat() (ns) so the modification times don't roundtrip properly yet.
This commit is contained in:
Nick Craig-Wood
parent
a3e28e6b97
commit
25a1b96537
40
notes.txt
40
notes.txt
@ -45,3 +45,43 @@ Check the locking in swift module!
|
||||
|
||||
Need to make directory objects otherwise can't upload an empty directory
|
||||
* Or could upload empty directories only?
|
||||
|
||||
Make a fs.Errorf and count errors and log them at a different level
|
||||
|
||||
Seem to be able to read stats to ns precision (using struct stat
|
||||
defined in asm-generic/stat.h)
|
||||
|
||||
https://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html
|
||||
|
||||
but only write them to us precision.
|
||||
This is confirmed by looking at the source for Chtimes
|
||||
|
||||
func Chtimes(name string, atime time.Time, mtime time.Time) error {
|
||||
var utimes [2]syscall.Timeval
|
||||
atime_ns := atime.Unix()*1e9 + int64(atime.Nanosecond())
|
||||
mtime_ns := mtime.Unix()*1e9 + int64(mtime.Nanosecond())
|
||||
utimes[0] = syscall.NsecToTimeval(atime_ns)
|
||||
utimes[1] = syscall.NsecToTimeval(mtime_ns)
|
||||
if e := syscall.Utimes(name, utimes[0:]); e != nil {
|
||||
return &PathError{"chtimes", name, e}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
And
|
||||
|
||||
func NsecToTimeval(nsec int64) (tv Timeval) {
|
||||
nsec += 999 // round up to microsecond
|
||||
tv.Sec = nsec / 1e9
|
||||
tv.Usec = nsec % 1e9 / 1e3
|
||||
return
|
||||
}
|
||||
|
||||
Seems likely Go should be using
|
||||
utimensat() and futimens()
|
||||
Instead of utimes() - the syscall is defined already
|
||||
SYS_UTIMENSAT
|
||||
Also for all the *bsd so likely they could be fixed too
|
||||
|
||||
Can also fix Futime which isn't using the syscall under linux
|
||||
|
||||
|
||||
181
swiftsync.go
181
swiftsync.go
@ -17,6 +17,7 @@ import (
|
||||
"runtime/pprof"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Globals
|
||||
@ -39,15 +40,12 @@ var (
|
||||
uploaders = flag.Int("uploaders", 4, "Number of uploaders to run in parallel.")
|
||||
)
|
||||
|
||||
// Make this an interface?
|
||||
//
|
||||
// Have an implementation for SwiftObjects and FsObjects?
|
||||
//
|
||||
// This represents an object in the filesystem
|
||||
// A filesystem like object which can either be a remote object or a
|
||||
// local file/directory or both
|
||||
type FsObject struct {
|
||||
rel string
|
||||
path string
|
||||
info os.FileInfo
|
||||
remote string // The remote path
|
||||
path string // The local path
|
||||
info os.FileInfo // Interface for file info
|
||||
}
|
||||
|
||||
type FsObjectsChan chan *FsObject
|
||||
@ -57,7 +55,7 @@ type FsObjects []FsObject
|
||||
// Write debuging output for this FsObject
|
||||
func (fs *FsObject) Debugf(text string, args ...interface{}) {
|
||||
out := fmt.Sprintf(text, args...)
|
||||
log.Printf("%s: %s", fs.rel, out)
|
||||
log.Printf("%s: %s", fs.remote, out)
|
||||
}
|
||||
|
||||
// md5sum calculates the md5sum of a file returning a lowercase hex string
|
||||
@ -77,41 +75,56 @@ func (fs *FsObject) md5sum() (string, error) {
|
||||
return fmt.Sprintf("%x", hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// Checks to see if an object has changed or not by looking at its size, mtime and MD5SUM
|
||||
// Sets the modification time of the local fs object
|
||||
func (fs *FsObject) SetModTime(modTime time.Time) {
|
||||
err := os.Chtimes(fs.path, modTime, modTime)
|
||||
if err != nil {
|
||||
fs.Debugf("Failed to set mtime on file: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Checks to see if the remote and local objects are equal by looking
|
||||
// at size, mtime and MD5SUM
|
||||
//
|
||||
// If the remote object size is different then it is considered to be
|
||||
// changed.
|
||||
// If the remote and local size are different then it is considered to
|
||||
// be not equal.
|
||||
//
|
||||
// If the size is the same and the mtime is the same then it is
|
||||
// considered to be unchanged. This is the heuristic rsync uses when
|
||||
// considered to be equal. This is the heuristic rsync uses when
|
||||
// not using --checksum.
|
||||
//
|
||||
// If the size is the same and and mtime is different or unreadable
|
||||
// and the MD5SUM is the same then the file is considered to be
|
||||
// unchanged. In this case the mtime on the object is updated.
|
||||
// equal. In this case the mtime on the object is updated. If
|
||||
// upload is set then the remote object is changed otherwise the local
|
||||
// object.
|
||||
//
|
||||
// Otherwise the file is considered to be changed.
|
||||
func (fs *FsObject) changed(c *swift.Connection, container string) bool {
|
||||
obj, h, err := c.Object(container, fs.rel)
|
||||
// Otherwise the file is considered to be not equal including if there
|
||||
// were errors reading info.
|
||||
func (fs *FsObject) Equal(c *swift.Connection, container string, upload bool) bool {
|
||||
// FIXME could pass in an Object here if we have one which
|
||||
// will mean we could do the Size and Hash checks without a
|
||||
// remote call if we wanted
|
||||
obj, h, err := c.Object(container, fs.remote)
|
||||
if err != nil {
|
||||
fs.Debugf("Failed to read info: %s", err)
|
||||
return true
|
||||
return false
|
||||
}
|
||||
if obj.Bytes != fs.info.Size() {
|
||||
fs.Debugf("Sizes differ")
|
||||
return true
|
||||
return false
|
||||
}
|
||||
|
||||
// Size the same so check the mtime
|
||||
m := h.ObjectMetadata()
|
||||
t, err := m.GetModTime()
|
||||
remoteModTime, err := m.GetModTime()
|
||||
if err != nil {
|
||||
fs.Debugf("Failed to read mtime: %s", err)
|
||||
} else if !t.Equal(fs.info.ModTime()) {
|
||||
} else if !remoteModTime.Equal(fs.info.ModTime()) {
|
||||
fs.Debugf("Modification times differ")
|
||||
} else {
|
||||
fs.Debugf("Size and modification time the same - skipping")
|
||||
return false
|
||||
fs.Debugf("Size and modification time the same")
|
||||
return true
|
||||
}
|
||||
|
||||
// mtime is unreadable or different but size is the same so
|
||||
@ -119,29 +132,36 @@ func (fs *FsObject) changed(c *swift.Connection, container string) bool {
|
||||
localMd5, err := fs.md5sum()
|
||||
if err != nil {
|
||||
fs.Debugf("Failed to calculate md5: %s", err)
|
||||
return true
|
||||
return false
|
||||
}
|
||||
// fs.Debugf("Local MD5 %s", localMd5)
|
||||
// fs.Debugf("Remote MD5 %s", obj.Hash)
|
||||
if localMd5 != strings.ToLower(obj.Hash) {
|
||||
fs.Debugf("Md5sums differ")
|
||||
return true
|
||||
return false
|
||||
}
|
||||
|
||||
// Update the mtime of the remote object here
|
||||
m.SetModTime(fs.info.ModTime())
|
||||
err = c.ObjectUpdate(container, fs.rel, m.ObjectHeaders())
|
||||
if err != nil {
|
||||
fs.Debugf("Failed to update mtime: %s", err)
|
||||
// Size and MD5 the same but mtime different so update the
|
||||
// mtime of the local or remote object here
|
||||
if upload {
|
||||
m.SetModTime(fs.info.ModTime())
|
||||
err = c.ObjectUpdate(container, fs.remote, m.ObjectHeaders())
|
||||
if err != nil {
|
||||
fs.Debugf("Failed to update remote mtime: %s", err)
|
||||
}
|
||||
fs.Debugf("Updated mtime of remote object")
|
||||
} else {
|
||||
fmt.Printf("metadata %q, remoteModTime = %s\n", m, remoteModTime)
|
||||
fs.SetModTime(remoteModTime)
|
||||
fs.Debugf("Updated mtime of local object")
|
||||
}
|
||||
fs.Debugf("Updated mtime")
|
||||
|
||||
fs.Debugf("Size and MD5SUM identical - skipping")
|
||||
return false
|
||||
fs.Debugf("Size and MD5SUM of local and remote objects identical")
|
||||
return true
|
||||
}
|
||||
|
||||
// Is this object storable
|
||||
func (fs *FsObject) storable(c *swift.Connection, container string) bool {
|
||||
func (fs *FsObject) storable() bool {
|
||||
mode := fs.info.Mode()
|
||||
if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
|
||||
fs.Debugf("Can't transfer non file/directory")
|
||||
@ -165,7 +185,7 @@ func (fs *FsObject) put(c *swift.Connection, container string) {
|
||||
defer in.Close()
|
||||
m := swift.Metadata{}
|
||||
m.SetModTime(fs.info.ModTime())
|
||||
_, err = c.ObjectPut(container, fs.rel, in, true, "", "", m.ObjectHeaders())
|
||||
_, err = c.ObjectPut(container, fs.remote, in, true, "", "", m.ObjectHeaders())
|
||||
if err != nil {
|
||||
fs.Debugf("Failed to upload: %s", err)
|
||||
return
|
||||
@ -173,24 +193,32 @@ func (fs *FsObject) put(c *swift.Connection, container string) {
|
||||
fs.Debugf("Uploaded")
|
||||
}
|
||||
|
||||
// Stat a FsObject info info
|
||||
func (fs *FsObject) lstat() error {
|
||||
info, err := os.Lstat(fs.path)
|
||||
fs.info = info
|
||||
return err
|
||||
}
|
||||
|
||||
// Return an FsObject from a path
|
||||
//
|
||||
// May return nil if an error occurred
|
||||
func NewFsObject(root, path string) *FsObject {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
log.Printf("Failed to stat %s: %s", path, err)
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
remote, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
log.Printf("Failed to get relative path %s: %s", path, err)
|
||||
return nil
|
||||
}
|
||||
if rel == "." {
|
||||
rel = ""
|
||||
if remote == "." {
|
||||
remote = ""
|
||||
}
|
||||
return &FsObject{rel: rel, path: path, info: info}
|
||||
fs := &FsObject{remote: remote, path: path}
|
||||
err = fs.lstat()
|
||||
if err != nil {
|
||||
log.Printf("Failed to stat %s: %s", path, err)
|
||||
return nil
|
||||
}
|
||||
return fs
|
||||
}
|
||||
|
||||
// Walk the path returning a channel of FsObjects
|
||||
@ -252,11 +280,11 @@ func checker(c *swift.Connection, container string, in, out FsObjectsChan, wg *s
|
||||
defer wg.Done()
|
||||
for fs := range in {
|
||||
// Check to see if can store this
|
||||
if !fs.storable(c, container) {
|
||||
if !fs.storable() {
|
||||
continue
|
||||
}
|
||||
// Check to see if changed or not
|
||||
if !fs.changed(c, container) {
|
||||
if fs.Equal(c, container, true) {
|
||||
fs.Debugf("Unchanged skipping")
|
||||
continue
|
||||
}
|
||||
@ -296,54 +324,48 @@ func upload(c *swift.Connection, root, container string) {
|
||||
uploaderWg.Wait()
|
||||
}
|
||||
|
||||
// FIXME should be an FsOject method
|
||||
//
|
||||
// Get an object to the filepath making directories if necessary
|
||||
func get(c *swift.Connection, container, name, filepath string) {
|
||||
log.Printf("Download %s to %s", name, filepath)
|
||||
func (fs *FsObject) get(c *swift.Connection, container string) {
|
||||
log.Printf("Download %s to %s", fs.remote, fs.path)
|
||||
|
||||
dir := path.Dir(filepath)
|
||||
dir := path.Dir(fs.path)
|
||||
err := os.MkdirAll(dir, 0770)
|
||||
if err != nil {
|
||||
log.Printf("Couldn't make directory %q: %s", dir, err)
|
||||
fs.Debugf("Couldn't make directory: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
out, err := os.Create(filepath)
|
||||
out, err := os.Create(fs.path)
|
||||
if err != nil {
|
||||
log.Printf("Failed to open %q: %s", filepath, err)
|
||||
fs.Debugf("Failed to open: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
h, getErr := c.ObjectGet(container, name, out, true, nil)
|
||||
h, getErr := c.ObjectGet(container, fs.remote, out, true, nil)
|
||||
if getErr != nil {
|
||||
log.Printf("Failed to download %q: %s", name, getErr)
|
||||
fs.Debugf("Failed to download: %s", getErr)
|
||||
}
|
||||
|
||||
closeErr := out.Close()
|
||||
if closeErr != nil {
|
||||
log.Printf("Error closing %q: %s", filepath, closeErr)
|
||||
fs.Debugf("Error closing: %s", closeErr)
|
||||
}
|
||||
|
||||
if getErr != nil || closeErr != nil {
|
||||
log.Printf("Removing failed download %q", filepath)
|
||||
err = os.Remove(filepath)
|
||||
fs.Debugf("Removing failed download")
|
||||
err = os.Remove(fs.path)
|
||||
if err != nil {
|
||||
log.Printf("Failed to remove %q: %s", filepath, err)
|
||||
fs.Debugf("Failed to remove failed download: %s", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Set the mtime
|
||||
// FIXME should be a FsObject method?
|
||||
modTime, err := h.ObjectMetadata().GetModTime()
|
||||
if err != nil {
|
||||
log.Printf("Failed to read mtime from object: %s", err)
|
||||
fs.Debugf("Failed to read mtime from object: %s", err)
|
||||
} else {
|
||||
err = os.Chtimes(filepath, modTime, modTime)
|
||||
if err != nil {
|
||||
log.Printf("Failed to set mtime on file: %s", err)
|
||||
}
|
||||
fs.SetModTime(modTime)
|
||||
}
|
||||
}
|
||||
|
||||
@ -369,17 +391,22 @@ func download(c *swift.Connection, container, root string) {
|
||||
for i := range objects {
|
||||
object := &objects[i]
|
||||
filepath := path.Join(root, object.Name)
|
||||
// fs := NewFsObject(root, filepath)
|
||||
// if fs == nil {
|
||||
// log.Printf("%s: Download: not found", object.Name)
|
||||
// } else if !fs.storable(c, container) {
|
||||
// fs.Debugf("Skip: not storable")
|
||||
// continue
|
||||
// } else if !fs.changed(c, container) {
|
||||
// fs.Debugf("Skip: not changed")
|
||||
// continue
|
||||
// }
|
||||
get(c, container, object.Name, filepath)
|
||||
fs := FsObject{remote: object.Name, path: filepath}
|
||||
_ = fs.lstat()
|
||||
if fs.info == nil {
|
||||
fs.Debugf("Couldn't find local file - download")
|
||||
} else {
|
||||
fs.Debugf("Found local file - checking")
|
||||
if !fs.storable() {
|
||||
fs.Debugf("Not overwriting different type local file")
|
||||
continue
|
||||
}
|
||||
if fs.Equal(c, container, false) {
|
||||
fs.Debugf("Skip: not changed")
|
||||
continue
|
||||
}
|
||||
}
|
||||
fs.get(c, container)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user