mirror of
https://github.com/rclone/rclone.git
synced 2025-06-21 04:08:02 +02:00
onedrive: re-add --onedrive-upload-cutoff flag
This was removed as part of #1716 to fix rclone uploads taking double the space. 7f744033d8eae0fa onedrive: Removed upload cutoff and always do session uploads As far as I can see, two revisions are still being created for single part uploads so the default for this flag is set to -1, off. However it may be useful for experimentation. See: #8545
This commit is contained in:
parent
b31659904f
commit
c37fe733df
@ -56,6 +56,7 @@ const (
|
|||||||
driveTypeSharepoint = "documentLibrary"
|
driveTypeSharepoint = "documentLibrary"
|
||||||
defaultChunkSize = 10 * fs.Mebi
|
defaultChunkSize = 10 * fs.Mebi
|
||||||
chunkSizeMultiple = 320 * fs.Kibi
|
chunkSizeMultiple = 320 * fs.Kibi
|
||||||
|
maxSinglePartSize = 4 * fs.Mebi
|
||||||
|
|
||||||
regionGlobal = "global"
|
regionGlobal = "global"
|
||||||
regionUS = "us"
|
regionUS = "us"
|
||||||
@ -138,6 +139,21 @@ func init() {
|
|||||||
Help: "Azure and Office 365 operated by Vnet Group in China",
|
Help: "Azure and Office 365 operated by Vnet Group in China",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
}, {
|
||||||
|
Name: "upload_cutoff",
|
||||||
|
Help: `Cutoff for switching to chunked upload.
|
||||||
|
|
||||||
|
Any files larger than this will be uploaded in chunks of chunk_size.
|
||||||
|
|
||||||
|
This is disabled by default as uploading using single part uploads
|
||||||
|
causes rclone to use twice the storage on Onedrive business as when
|
||||||
|
rclone sets the modification time after the upload Onedrive creates a
|
||||||
|
new version.
|
||||||
|
|
||||||
|
See: https://github.com/rclone/rclone/issues/1716
|
||||||
|
`,
|
||||||
|
Default: fs.SizeSuffix(-1),
|
||||||
|
Advanced: true,
|
||||||
}, {
|
}, {
|
||||||
Name: "chunk_size",
|
Name: "chunk_size",
|
||||||
Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
|
Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
|
||||||
@ -746,6 +762,7 @@ Examples:
|
|||||||
// Options defines the configuration for this backend
|
// Options defines the configuration for this backend
|
||||||
type Options struct {
|
type Options struct {
|
||||||
Region string `config:"region"`
|
Region string `config:"region"`
|
||||||
|
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||||
DriveID string `config:"drive_id"`
|
DriveID string `config:"drive_id"`
|
||||||
DriveType string `config:"drive_type"`
|
DriveType string `config:"drive_type"`
|
||||||
@ -1022,6 +1039,13 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func checkUploadCutoff(cs fs.SizeSuffix) error {
|
||||||
|
if cs > maxSinglePartSize {
|
||||||
|
return fmt.Errorf("%v is greater than %v", cs, maxSinglePartSize)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// NewFs constructs an Fs from the path, container:path
|
// NewFs constructs an Fs from the path, container:path
|
||||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||||
// Parse config into Options struct
|
// Parse config into Options struct
|
||||||
@ -1035,6 +1059,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("onedrive: chunk size: %w", err)
|
return nil, fmt.Errorf("onedrive: chunk size: %w", err)
|
||||||
}
|
}
|
||||||
|
err = checkUploadCutoff(opt.UploadCutoff)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("onedrive: upload cutoff: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
if opt.DriveID == "" || opt.DriveType == "" {
|
if opt.DriveID == "" || opt.DriveType == "" {
|
||||||
return nil, errors.New("unable to get drive_id and drive_type - if you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")
|
return nil, errors.New("unable to get drive_id and drive_type - if you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")
|
||||||
@ -2567,8 +2595,8 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
|
|||||||
// This function will set modtime and metadata after uploading, which will create a new version for the remote file
|
// This function will set modtime and metadata after uploading, which will create a new version for the remote file
|
||||||
func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (info *api.Item, err error) {
|
func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (info *api.Item, err error) {
|
||||||
size := src.Size()
|
size := src.Size()
|
||||||
if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
|
if size < 0 || size > int64(maxSinglePartSize) {
|
||||||
return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4 MiB")
|
return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and <= %v", maxSinglePartSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
fs.Debugf(o, "Starting singlepart upload")
|
fs.Debugf(o, "Starting singlepart upload")
|
||||||
@ -2621,9 +2649,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||||||
size := src.Size()
|
size := src.Size()
|
||||||
|
|
||||||
var info *api.Item
|
var info *api.Item
|
||||||
if size > 0 {
|
if size > 0 && size >= int64(o.fs.opt.UploadCutoff) {
|
||||||
info, err = o.uploadMultipart(ctx, in, src, options...)
|
info, err = o.uploadMultipart(ctx, in, src, options...)
|
||||||
} else if size == 0 {
|
} else if size >= 0 {
|
||||||
info, err = o.uploadSinglepart(ctx, in, src, options...)
|
info, err = o.uploadSinglepart(ctx, in, src, options...)
|
||||||
} else {
|
} else {
|
||||||
return errors.New("unknown-sized upload not supported")
|
return errors.New("unknown-sized upload not supported")
|
||||||
|
Loading…
x
Reference in New Issue
Block a user