rclone/fs/chunksize/chunksize.go

// Package chunksize calculates a suitable chunk size for large uploads
package chunksize

import (
	"github.com/rclone/rclone/fs"
)

/*
 Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi

 For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest
 possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption
 that some backends may only allow integer type parameters when specifying the chunk size.

 Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the
 smallest chunk size necessary to allow the upload to proceed.
*/
func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix {
	fileSize := fs.SizeSuffix(objInfo.Size())
	requiredChunks := fileSize / defaultChunkSize
	if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) {
		return defaultChunkSize
	}

	minChunk := fileSize / fs.SizeSuffix(maxParts)
	remainder := minChunk % fs.Mebi
	if remainder != 0 {
		minChunk += fs.Mebi - remainder
	}
	if fileSize/minChunk == fs.SizeSuffix(maxParts) && fileSize%fs.SizeSuffix(maxParts) != 0 { // when right on the boundary, we need to add a MiB
		minChunk += fs.Mebi
	}

	fs.Debugf(objInfo, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk)
	return minChunk
}