extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-02-02 11:39:33 +01:00
Code Issues Packages Projects Releases Wiki Activity

local: don't calculate any hashes by default #3419

Browse Source 
Before this change, if the caller didn't provide a hint, we would
calculate all hashes for reads and writes.

The new whirlpool hash is particularly expensive and that has become noticeable.

Now we don't calculate any hashes on upload or download unless hints are provided.

This means that some operations may run slower and these will need to be discovered!

It does not affect anything calling operations.Copy which already puts
the corrects hints in.
This commit is contained in:
Nick Craig-Wood 2019-08-10 10:26:29 +01:00
parent 106cf1852d
commit 402aaca7fe
1 changed files with 28 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
backend/local/local.go
View File

@ -917,7 +917,7 @@ func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err
// Open an object for read // Open an object for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
var offset, limit int64 = 0, -1 var offset, limit int64 = 0, -1
hashes := hash.Supported var hasher *hash.MultiHasher
for _, option := range options { for _, option := range options {
switch x := option.(type) { switch x := option.(type) {
case *fs.SeekOption: case *fs.SeekOption:
@ -925,7 +925,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
case *fs.RangeOption: case *fs.RangeOption:
offset, limit = x.Decode(o.size) offset, limit = x.Decode(o.size)
case *fs.HashesOption: case *fs.HashesOption:
hashes = x.Hashes if x.Hashes.Count() > 0 {
hasher, err = hash.NewMultiHasherTypes(x.Hashes)
if err != nil {
return nil, err
}
}
default: default:
if option.Mandatory() { if option.Mandatory() {
fs.Logf(o, "Unsupported mandatory option: %v", option) fs.Logf(o, "Unsupported mandatory option: %v", option)
@ -949,15 +954,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
// don't attempt to make checksums // don't attempt to make checksums
return wrappedFd, err return wrappedFd, err
} }
hash, err := hash.NewMultiHasherTypes(hashes) if hasher == nil {
if err != nil { // no need to wrap since we don't need checksums
return nil, err return wrappedFd, nil
} }
// Update the md5sum as we go along // Update the hashes as we go along
in = &localOpenFile{ in = &localOpenFile{
o: o, o: o,
in: wrappedFd, in: wrappedFd,
hash: hash, hash: hasher,
fd: fd, fd: fd,
} }
return in, nil return in, nil
@ -979,18 +984,23 @@ func (nwc nopWriterCloser) Close() error {
} }
// Update the object from in with modTime and size // Update the object from in with modTime and size
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
var out io.WriteCloser var out io.WriteCloser
var hasher *hash.MultiHasher
hashes := hash.Supported
for _, option := range options { for _, option := range options {
switch x := option.(type) { switch x := option.(type) {
case *fs.HashesOption: case *fs.HashesOption:
hashes = x.Hashes if x.Hashes.Count() > 0 {
hasher, err = hash.NewMultiHasherTypes(x.Hashes)
if err != nil {
return err
}
}
} }
} }
err := o.mkdirAll() err = o.mkdirAll()
if err != nil { if err != nil {
return err return err
} }
@ -1015,11 +1025,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
// Calculate the hash of the object we are reading as we go along // Calculate the hash of the object we are reading as we go along
hash, err := hash.NewMultiHasherTypes(hashes) if hasher != nil {
if err != nil { in = io.TeeReader(in, hasher)
return err
} }
in = io.TeeReader(in, hash)
_, err = io.Copy(out, in) _, err = io.Copy(out, in)
closeErr := out.Close() closeErr := out.Close()
@ -1055,9 +1063,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} }
// All successful so update the hashes // All successful so update the hashes
if hasher != nil {
o.fs.objectHashesMu.Lock() o.fs.objectHashesMu.Lock()
o.hashes = hash.Sums() o.hashes = hasher.Sums()
o.fs.objectHashesMu.Unlock() o.fs.objectHashesMu.Unlock()
}
// Set the mtime // Set the mtime
err = o.SetModTime(ctx, src.ModTime(ctx)) err = o.SetModTime(ctx, src.ModTime(ctx))