lib/multipart: fix accounting for multipart transfers

This change makes sure the accouting is done when bytes are taken out
of the buffer rather than put in.

See: https://forum.rclone.org/t/improve-transfer-stats-calculation-for-multipart-uploads/41172
This commit is contained in:
Nick Craig-Wood 2023-08-27 23:10:58 +01:00
parent b95bda1e92
commit 967fc6d7f4
2 changed files with 19 additions and 0 deletions

View File

@ -632,3 +632,15 @@ func UnWrap(in io.Reader) (unwrapped io.Reader, wrap WrapFn) {
}
return acc.OldStream(), acc.WrapStream
}
// UnWrapAccounting unwraps a reader returning unwrapped and acc a
// pointer to the accounting.
//
// The caller is expected to manage the accounting at this point.
func UnWrapAccounting(in io.Reader) (unwrapped io.Reader, acc *Account) {
a, ok := in.(*accountStream)
if !ok {
return in, nil
}
return a.in, a.acc
}

View File

@ -8,6 +8,7 @@ import (
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/pool"
@ -87,10 +88,16 @@ func UploadMultipart(ctx context.Context, src fs.ObjectInfo, in io.Reader, opt U
size = src.Size()
)
// Do the accounting manually
in, acc := accounting.UnWrapAccounting(in)
for partNum := int64(0); !finished; partNum++ {
// Get a block of memory from the pool and token which limits concurrency.
tokens.Get()
rw := NewRW()
if acc != nil {
rw.SetAccounting(acc.AccountRead)
}
free := func() {
// return the memory and token