From 269f90c1e4d908d6ce1d22a8c94a84c8bf60b14b Mon Sep 17 00:00:00 2001 From: YenForYang Date: Mon, 12 Jul 2021 17:29:31 -0500 Subject: [PATCH] drive: Fix buffering for single request upload for files smaller than `--drive-upload-cutoff` I discovered that `rclone` always upload in chunks of 16MiB whenever uploading a file smaller than `--drive-upload-cutoff`. This is undesirable since the purpose of the flag `--drive-upload-cutoff` is to *prevent* chunking below a certain file size. I realized that it wasn't `rclone` forcing the 16MiB chunks. The `google-api-go-client` forces a chunk size default of [`googleapi.DefaultUploadChunkSize`](https://github.com/googleapis/google-api-go-client/blob/32bf29c2e17105d5f285adac4531846c57847f11/googleapi/googleapi.go#L55-L57) bytes for resumable type uploads. This means that all requests that use `*drive.Service` directly for upload without specifying a `googleapi.ChunkSize` will be forced to use a *`resumable`* `uploadType` (rather than `multipart`) for files less than `googleapi.DefaultUploadChunkSize`. This is also noted directly in the Drive API client documentation [here](https://pkg.go.dev/google.golang.org/api/drive/v3@v0.44.0#FilesUpdateCall.Media). This fixes the problem by passing `googleapi.ChunkSize(0)` to `Media()` method calls, which is the only way to disable chunking completely. This is mentioned in the API docs [here](https://pkg.go.dev/google.golang.org/api/googleapi@v0.44.0#ChunkSize). The other alternative would be to pass `googleapi.ChunkSize(f.opt.ChunkSize)` -- however, I'm *strongly* in favor of *not* doing this for performance reasons. By not explicitly passing a `googleapi.ChunkSize(0)`, we effectively allow [`PrepareUpload()`](https://pkg.go.dev/google.golang.org/api/internal/gensupport@v0.44.0#PrepareUpload) to create a [`NewMediaBuffer`](https://pkg.go.dev/google.golang.org/api/internal/gensupport@v0.44.0#NewMediaBuffer) that copies the original `io.Reader` passed to `Media()` in order to check that its size is less than `ChunkSize`, which will unnecessarily consume time and memory. `minChunkSize` is also changed to be `googleapi.MinUploadChunkSize`, as it is something specified we have no control over. --- backend/drive/drive.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 75d305c0c..72207c381 100755 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -68,7 +68,7 @@ const ( defaultScope = "drive" // chunkSize is the size of the chunks created during a resumable upload and should be a power of two. // 1<<18 is the minimum size supported by the Google uploader, and there is no maximum. - minChunkSize = 256 * fs.Kibi + minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize) defaultChunkSize = 8 * fs.Mebi partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks" listRGrouping = 50 // number of IDs to search at once when using ListR @@ -2145,7 +2145,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, // Don't retry, return a retry error instead err = f.pacer.CallNoRetry(func() (bool, error) { info, err = f.svc.Files.Create(createInfo). - Media(in, googleapi.ContentType(srcMimeType)). + Media(in, googleapi.ContentType(srcMimeType), googleapi.ChunkSize(0)). Fields(partialFields). SupportsAllDrives(true). KeepRevisionForever(f.opt.KeepRevisionForever). @@ -3638,7 +3638,7 @@ func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadM // Don't retry, return a retry error instead err = o.fs.pacer.CallNoRetry(func() (bool, error) { info, err = o.fs.svc.Files.Update(actualID(o.id), updateInfo). - Media(in, googleapi.ContentType(uploadMimeType)). + Media(in, googleapi.ContentType(uploadMimeType), googleapi.ChunkSize(0)). Fields(partialFields). SupportsAllDrives(true). KeepRevisionForever(o.fs.opt.KeepRevisionForever).