azureblob: add support cold tier

This commit is contained in:
Ivan Yanitra 2023-10-18 15:01:11 +08:00 committed by Nick Craig-Wood
parent 4ac4597afb
commit 0ee6d0b4bf
5 changed files with 18 additions and 14 deletions

View File

@ -295,10 +295,10 @@ avoid the time out.`,
Advanced: true, Advanced: true,
}, { }, {
Name: "access_tier", Name: "access_tier",
Help: `Access tier of blob: hot, cool or archive. Help: `Access tier of blob: hot, cool, cold or archive.
Archived blobs can be restored by setting access tier to hot or Archived blobs can be restored by setting access tier to hot, cool or
cool. Leave blank if you intend to use default access tier, which is cold. Leave blank if you intend to use default access tier, which is
set at account level set at account level
If there is no "access tier" specified, rclone doesn't apply any tier. If there is no "access tier" specified, rclone doesn't apply any tier.
@ -306,7 +306,7 @@ rclone performs "Set Tier" operation on blobs while uploading, if objects
are not modified, specifying "access tier" to new one will have no effect. are not modified, specifying "access tier" to new one will have no effect.
If blobs are in "archive tier" at remote, trying to perform data transfer If blobs are in "archive tier" at remote, trying to perform data transfer
operations from remote will not be allowed. User should first restore by operations from remote will not be allowed. User should first restore by
tiering blob to "Hot" or "Cool".`, tiering blob to "Hot", "Cool" or "Cold".`,
Advanced: true, Advanced: true,
}, { }, {
Name: "archive_tier_delete", Name: "archive_tier_delete",
@ -520,6 +520,7 @@ func (o *Object) split() (container, containerPath string) {
func validateAccessTier(tier string) bool { func validateAccessTier(tier string) bool {
return strings.EqualFold(tier, string(blob.AccessTierHot)) || return strings.EqualFold(tier, string(blob.AccessTierHot)) ||
strings.EqualFold(tier, string(blob.AccessTierCool)) || strings.EqualFold(tier, string(blob.AccessTierCool)) ||
strings.EqualFold(tier, string(blob.AccessTierCold)) ||
strings.EqualFold(tier, string(blob.AccessTierArchive)) strings.EqualFold(tier, string(blob.AccessTierArchive))
} }
@ -649,8 +650,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if opt.AccessTier == "" { if opt.AccessTier == "" {
opt.AccessTier = string(defaultAccessTier) opt.AccessTier = string(defaultAccessTier)
} else if !validateAccessTier(opt.AccessTier) { } else if !validateAccessTier(opt.AccessTier) {
return nil, fmt.Errorf("supported access tiers are %s, %s and %s", return nil, fmt.Errorf("supported access tiers are %s, %s, %s and %s",
string(blob.AccessTierHot), string(blob.AccessTierCool), string(blob.AccessTierArchive)) string(blob.AccessTierHot), string(blob.AccessTierCool), string(blob.AccessTierCold), string(blob.AccessTierArchive))
} }
if !validatePublicAccess((opt.PublicAccess)) { if !validatePublicAccess((opt.PublicAccess)) {
@ -1899,7 +1900,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
var offset int64 var offset int64
var count int64 var count int64
if o.AccessTier() == blob.AccessTierArchive { if o.AccessTier() == blob.AccessTierArchive {
return nil, fmt.Errorf("blob in archive tier, you need to set tier to hot or cool first") return nil, fmt.Errorf("blob in archive tier, you need to set tier to hot, cool, cold first")
} }
fs.FixRangeOption(options, o.size) fs.FixRangeOption(options, o.size)
for _, option := range options { for _, option := range options {

View File

@ -19,7 +19,7 @@ func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{ fstests.Run(t, &fstests.Opt{
RemoteName: "TestAzureBlob:", RemoteName: "TestAzureBlob:",
NilObject: (*Object)(nil), NilObject: (*Object)(nil),
TiersToTest: []string{"Hot", "Cool"}, TiersToTest: []string{"Hot", "Cool", "Cold"},
ChunkedUpload: fstests.ChunkedUploadConfig{ ChunkedUpload: fstests.ChunkedUploadConfig{
MinChunkSize: defaultChunkSize, MinChunkSize: defaultChunkSize,
}, },
@ -35,7 +35,7 @@ func TestIntegration2(t *testing.T) {
fstests.Run(t, &fstests.Opt{ fstests.Run(t, &fstests.Opt{
RemoteName: name + ":", RemoteName: name + ":",
NilObject: (*Object)(nil), NilObject: (*Object)(nil),
TiersToTest: []string{"Hot", "Cool"}, TiersToTest: []string{"Hot", "Cool", "Cold"},
ChunkedUpload: fstests.ChunkedUploadConfig{ ChunkedUpload: fstests.ChunkedUploadConfig{
MinChunkSize: defaultChunkSize, MinChunkSize: defaultChunkSize,
}, },
@ -62,6 +62,7 @@ func TestValidateAccessTier(t *testing.T) {
"HOT": {"HOT", true}, "HOT": {"HOT", true},
"Hot": {"Hot", true}, "Hot": {"Hot", true},
"cool": {"cool", true}, "cool": {"cool", true},
"cold": {"cold", true},
"archive": {"archive", true}, "archive": {"archive", true},
"empty": {"", false}, "empty": {"", false},
"unknown": {"unknown", false}, "unknown": {"unknown", false},

View File

@ -675,10 +675,10 @@ Properties:
#### --azureblob-access-tier #### --azureblob-access-tier
Access tier of blob: hot, cool or archive. Access tier of blob: hot, cool, cold or archive.
Archived blobs can be restored by setting access tier to hot or Archived blobs can be restored by setting access tier to hot, cool or
cool. Leave blank if you intend to use default access tier, which is cold. Leave blank if you intend to use default access tier, which is
set at account level set at account level
If there is no "access tier" specified, rclone doesn't apply any tier. If there is no "access tier" specified, rclone doesn't apply any tier.
@ -686,7 +686,7 @@ rclone performs "Set Tier" operation on blobs while uploading, if objects
are not modified, specifying "access tier" to new one will have no effect. are not modified, specifying "access tier" to new one will have no effect.
If blobs are in "archive tier" at remote, trying to perform data transfer If blobs are in "archive tier" at remote, trying to perform data transfer
operations from remote will not be allowed. User should first restore by operations from remote will not be allowed. User should first restore by
tiering blob to "Hot" or "Cool". tiering blob to "Hot", "Cool" or "Cold".
Properties: Properties:

2
go.mod
View File

@ -6,7 +6,7 @@ require (
bazil.org/fuse v0.0.0-20221209211307-2abb8038c751 bazil.org/fuse v0.0.0-20221209211307-2abb8038c751
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd
github.com/Unknwon/goconfig v1.0.0 github.com/Unknwon/goconfig v1.0.0

2
go.sum
View File

@ -46,6 +46,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aov
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=