extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-23 00:43:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

rcat: directly upload small files without streaming them

Browse Source
This commit is contained in:
Stefan Breunig 2017-09-11 08:25:34 +02:00
parent 10fa2a7806
commit 57817397a0
2 changed files with 37 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
fs/operations.go
View File

@ -1587,8 +1587,28 @@ func Rcat(fdst Fs, dstFileName string, in0 io.ReadCloser, modTime time.Time) (er
Stats.Transferring(dstFileName) Stats.Transferring(dstFileName)
defer func() { defer func() {
Stats.DoneTransferring(dstFileName, err == nil) Stats.DoneTransferring(dstFileName, err == nil)
if err := in0.Close(); err != nil {
Debugf(fdst, "Rcat: failed to close source: %v", err)
}
}() }()
hashOption := &HashesOption{Hashes: NewHashSet()}
in := in0
buf := make([]byte, 100*1024)
if n, err := io.ReadFull(in0, buf); err != nil {
Debugf(fdst, "File to upload is small, uploading instead of streaming")
in = ioutil.NopCloser(bytes.NewReader(buf[:n]))
in = NewAccountSizeName(in, int64(n), dstFileName).WithBuffer()
if !Config.SizeOnly {
hashOption = &HashesOption{Hashes: HashSet(fdst.Hashes().GetOne())}
}
objInfo := NewStaticObjectInfo(dstFileName, modTime, int64(n), false, nil, nil)
_, err := fdst.Put(in, objInfo, hashOption)
return err
}
in = ioutil.NopCloser(io.MultiReader(bytes.NewReader(buf), in0))
fStreamTo := fdst fStreamTo := fdst
canStream := fdst.Features().PutStream != nil canStream := fdst.Features().PutStream != nil
if !canStream { if !canStream {
@ -1606,21 +1626,11 @@ func Rcat(fdst Fs, dstFileName string, in0 io.ReadCloser, modTime time.Time) (er
fStreamTo = tmpLocalFs fStreamTo = tmpLocalFs
} }
objInfo := NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, nil)
// work out which hash to use - limit to 1 hash in common
var common HashSet
hashType := HashNone
if !Config.SizeOnly { if !Config.SizeOnly {
common = fStreamTo.Hashes().Overlap(SupportedHashes) hashOption = &HashesOption{Hashes: HashSet(fStreamTo.Hashes().GetOne())}
if common.Count() > 0 {
hashType = common.GetOne()
common = HashSet(hashType)
}
} }
hashOption := &HashesOption{Hashes: common}
in := NewAccountSizeName(in0, -1, dstFileName).WithBuffer() in = NewAccountSizeName(in, -1, dstFileName).WithBuffer()
if Config.DryRun { if Config.DryRun {
Logf("stdin", "Not copying as --dry-run") Logf("stdin", "Not copying as --dry-run")
@ -1629,6 +1639,7 @@ func Rcat(fdst Fs, dstFileName string, in0 io.ReadCloser, modTime time.Time) (er
return err return err
} }
objInfo := NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, nil)
tmpObj, err := fStreamTo.Features().PutStream(in, objInfo, hashOption) tmpObj, err := fStreamTo.Features().PutStream(in, objInfo, hashOption)
if err == nil && !canStream { if err == nil && !canStream {
err = Copy(fdst, nil, dstFileName, tmpObj) err = Copy(fdst, nil, dstFileName, tmpObj)

20
fs/operations_test.go
View File

@ -737,15 +737,23 @@ func TestRcat(t *testing.T) {
fstest.CheckListing(t, r.fremote, []fstest.Item{}) fstest.CheckListing(t, r.fremote, []fstest.Item{})
data := "this is some really nice test data" data1 := "this is some really nice test data"
path := "file_from_pipe" path1 := "small_file_from_pipe"
in := ioutil.NopCloser(strings.NewReader(data)) data2 := string(make([]byte, 100*1024+1))
err := fs.Rcat(r.fremote, path, in, t1) path2 := "big_file_from_pipe"
in := ioutil.NopCloser(strings.NewReader(data1))
err := fs.Rcat(r.fremote, path1, in, t1)
require.NoError(t, err) require.NoError(t, err)
file := fstest.NewItem(path, data, t1) in = ioutil.NopCloser(strings.NewReader(data2))
fstest.CheckItems(t, r.fremote, file) err = fs.Rcat(r.fremote, path2, in, t2)
require.NoError(t, err)
file1 := fstest.NewItem(path1, data1, t1)
file2 := fstest.NewItem(path2, data2, t2)
fstest.CheckItems(t, r.fremote, file1, file2)
} }
func TestRmdirs(t *testing.T) { func TestRmdirs(t *testing.T) {