mirror of
https://github.com/rclone/rclone.git
synced 2024-11-30 04:15:26 +01:00
9cc8ff4dd4
Some storage providers e.g. S3 don't have an efficient rename operation. Before this change, when chunker finished an upload, the server-side copy and delete operations that renamed temporary chunks to their final names could take a significant amount of time. This PR records transaction identifier (versioning) in the metadata of chunker composite objects striving to remove the need for rename operations on such backends. This approach will be triggered be the new "transactions" configuration option, which can be "rename" (the default) or "norename". We implement the new approach for uploads (Put operations). The chunker Move operation still uses the rename operation of underlying backend. Filling this gap is left for a later PR. Co-authored-by: Ivan Andreev <ivandeex@gmail.com>
326 lines
8.3 KiB
YAML
326 lines
8.3 KiB
YAML
tests:
|
|
- path: backend
|
|
addbackend: true
|
|
nobinary: true
|
|
short: true
|
|
- path: fs/operations
|
|
fastlist: true
|
|
- path: fs/sync
|
|
fastlist: true
|
|
- path: vfs
|
|
- path: cmd/serve/restic
|
|
localonly: true
|
|
backends:
|
|
# - backend: "amazonclouddrive"
|
|
# remote: "TestAmazonCloudDrive:"
|
|
# fastlist: false
|
|
- backend: "local"
|
|
remote: ""
|
|
fastlist: false
|
|
- backend: "b2"
|
|
remote: "TestB2:"
|
|
fastlist: true
|
|
listretries: 5
|
|
- backend: "crypt"
|
|
remote: "TestCryptDrive:"
|
|
fastlist: true
|
|
- backend: "crypt"
|
|
remote: "TestCryptSwift:"
|
|
fastlist: false
|
|
## chunker
|
|
- backend: "chunker"
|
|
remote: "TestChunkerLocal:"
|
|
fastlist: true
|
|
- backend: "chunker"
|
|
remote: "TestChunkerNometaLocal:"
|
|
fastlist: true
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk3bLocal:"
|
|
fastlist: true
|
|
maxfile: 6k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk3bNometaLocal:"
|
|
fastlist: true
|
|
maxfile: 6k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk3bNoRenameLocal:"
|
|
fastlist: true
|
|
maxfile: 6k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerMailru:"
|
|
fastlist: true
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bMailru:"
|
|
fastlist: true
|
|
maxfile: 10k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bYandex:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bBox:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerS3:"
|
|
fastlist: true
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bS3:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bMD5HashS3:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bSHA1HashS3:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerOverCrypt:"
|
|
fastlist: true
|
|
maxfile: 6k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bMD5QuickS3:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bSHA1QuickS3:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
## end chunker
|
|
## begin compress
|
|
- backend: "compress"
|
|
remote: "TestCompress:"
|
|
fastlist: false
|
|
- backend: "compress"
|
|
remote: "TestCompressSwift:"
|
|
fastlist: false
|
|
- backend: "compress"
|
|
remote: "TestCompressDrive:"
|
|
fastlist: false
|
|
- backend: "compress"
|
|
remote: "TestCompressS3:"
|
|
fastlist: false
|
|
## end compress
|
|
- backend: "drive"
|
|
remote: "TestDrive:"
|
|
fastlist: true
|
|
- backend: "dropbox"
|
|
remote: "TestDropbox:"
|
|
fastlist: false
|
|
- backend: "filefabric"
|
|
remote: "TestFileFabric:"
|
|
fastlist: false
|
|
- backend: "googlecloudstorage"
|
|
remote: "TestGoogleCloudStorage:"
|
|
fastlist: true
|
|
- backend: "googlephotos"
|
|
remote: "TestGooglePhotos:"
|
|
tests:
|
|
- backend
|
|
- backend: "hubic"
|
|
remote: "TestHubic:"
|
|
fastlist: false
|
|
- backend: "jottacloud"
|
|
remote: "TestJottacloud:"
|
|
fastlist: true
|
|
- backend: "memory"
|
|
remote: ":memory:"
|
|
fastlist: true
|
|
- backend: "onedrive"
|
|
remote: "TestOneDrive:"
|
|
fastlist: false
|
|
- backend: "onedrive"
|
|
remote: "TestOneDriveBusiness:"
|
|
fastlist: false
|
|
- backend: "onedrive"
|
|
remote: "TestOneDriveCn:"
|
|
fastlist: false
|
|
- backend: "s3"
|
|
remote: "TestS3:"
|
|
fastlist: true
|
|
- backend: "s3"
|
|
remote: "TestS3Minio:"
|
|
fastlist: true
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsPutFiles/SetTier
|
|
- TestIntegration/FsMkdir/FsEncoding/control_chars
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_LF
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_VT
|
|
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
- TestIntegration/FsMkdir/FsEncoding/trailing_LF
|
|
- TestIntegration/FsMkdir/FsEncoding/trailing_VT
|
|
- backend: "s3"
|
|
remote: "TestS3MinioEdge:"
|
|
fastlist: true
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsPutFiles/SetTier
|
|
- backend: "s3"
|
|
remote: "TestS3Wasabi:"
|
|
fastlist: true
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_CR
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_LF
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_HT
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_VT
|
|
- TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0
|
|
# Disabled due to excessive rate limiting at DO which cause the tests never to pass
|
|
# This hits the rate limit as documented here: https://www.digitalocean.com/docs/spaces/#limits
|
|
# 2 COPYs per 5 minutes on any individual object in a Space
|
|
# - backend: "s3"
|
|
# remote: "TestS3DigitalOcean:"
|
|
# fastlist: true
|
|
# ignore:
|
|
# - TestIntegration/FsMkdir/FsPutFiles/FsCopy
|
|
# - TestIntegration/FsMkdir/FsPutFiles/SetTier
|
|
# - backend: "s3"
|
|
# remote: "TestS3Ceph:"
|
|
# fastlist: true
|
|
# ignore:
|
|
# - TestIntegration/FsMkdir/FsPutFiles/FsCopy
|
|
# - TestIntegration/FsMkdir/FsPutFiles/SetTier
|
|
- backend: "s3"
|
|
remote: "TestS3Alibaba:"
|
|
fastlist: true
|
|
- backend: "sftp"
|
|
remote: "TestSFTPOpenssh:"
|
|
fastlist: false
|
|
- backend: "sftp"
|
|
remote: "TestSFTPRclone:"
|
|
fastlist: false
|
|
- backend: "sugarsync"
|
|
remote: "TestSugarSync:Test"
|
|
fastlist: false
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsPutFiles/PublicLink
|
|
- backend: "swift"
|
|
remote: "TestSwiftAIO:"
|
|
fastlist: true
|
|
- backend: "swift"
|
|
remote: "TestSwift:"
|
|
fastlist: true
|
|
# - backend: "swift"
|
|
# remote: "TestSwiftCeph:"
|
|
# fastlist: true
|
|
# ignore:
|
|
# - TestIntegration/FsMkdir/FsPutFiles/FsCopy
|
|
- backend: "yandex"
|
|
remote: "TestYandex:"
|
|
fastlist: false
|
|
- backend: "ftp"
|
|
remote: "TestFTPProftpd:"
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
fastlist: false
|
|
# - backend: "ftp"
|
|
# remote: "TestFTPVsftpd:"
|
|
# ignore:
|
|
# - TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
# fastlist: false
|
|
- backend: "ftp"
|
|
remote: "TestFTPPureftpd:"
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
fastlist: false
|
|
- backend: "ftp"
|
|
remote: "TestFTPRclone:"
|
|
ignore:
|
|
- "TestMultithreadCopy/{size:131071_streams:2}"
|
|
- "TestMultithreadCopy/{size:131072_streams:2}"
|
|
- "TestMultithreadCopy/{size:131073_streams:2}"
|
|
fastlist: false
|
|
- backend: "box"
|
|
remote: "TestBox:"
|
|
fastlist: false
|
|
- backend: "fichier"
|
|
remote: "TestFichier:"
|
|
fastlist: false
|
|
tests:
|
|
- backend
|
|
- backend: "qingstor"
|
|
remote: "TestQingStor:"
|
|
fastlist: false
|
|
oneonly: true
|
|
cleanup: true
|
|
- backend: "azureblob"
|
|
remote: "TestAzureBlob:"
|
|
fastlist: true
|
|
- backend: "pcloud"
|
|
remote: "TestPcloud:"
|
|
fastlist: false
|
|
- backend: "webdav"
|
|
remote: "TestWebdavNextcloud:"
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
- TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8
|
|
fastlist: false
|
|
- backend: "webdav"
|
|
remote: "TestWebdavOwncloud:"
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
- TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8
|
|
- TestIntegration/FsMkdir/FsPutFiles/FsCopy
|
|
- TestCopyFileCopyDest
|
|
- TestServerSideCopy
|
|
- TestSyncCopyDest
|
|
fastlist: false
|
|
- backend: "webdav"
|
|
remote: "TestWebdavRclone:"
|
|
ignore:
|
|
- TestFileReadAtZeroLength
|
|
fastlist: false
|
|
- backend: "cache"
|
|
remote: "TestCache:"
|
|
fastlist: false
|
|
- backend: "mega"
|
|
remote: "TestMega:"
|
|
fastlist: false
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsPutFiles/PublicLink
|
|
- TestDirRename
|
|
- backend: "opendrive"
|
|
remote: "TestOpenDrive:"
|
|
fastlist: false
|
|
- backend: "union"
|
|
remote: "TestUnion:"
|
|
fastlist: false
|
|
- backend: "koofr"
|
|
remote: "TestKoofr:"
|
|
fastlist: false
|
|
- backend: "premiumizeme"
|
|
remote: "TestPremiumizeMe:"
|
|
fastlist: false
|
|
- backend: "putio"
|
|
remote: "TestPutio:"
|
|
fastlist: false
|
|
- backend: "sharefile"
|
|
remote: "TestSharefile:"
|
|
fastlist: false
|
|
- backend: "mailru"
|
|
remote: "TestMailru:"
|
|
subdir: false
|
|
fastlist: false
|
|
- backend: "seafile"
|
|
remote: "TestSeafileV6:"
|
|
fastlist: false
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsPutFiles/FsDirMove
|
|
- backend: "seafile"
|
|
remote: "TestSeafile:"
|
|
fastlist: true
|
|
- backend: "seafile"
|
|
remote: "TestSeafileEncrypted:"
|
|
fastlist: true
|
|
- backend: "tardigrade"
|
|
remote: "TestTardigrade:"
|
|
fastlist: true
|
|
- backend: "zoho"
|
|
remote: "TestZoho:"
|
|
fastlist: false
|
|
- backend: "hdfs"
|
|
remote: "TestHdfs:"
|
|
fastlist: false
|
|
ignore:
|
|
- TestSyncUTFNorm
|