extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-03-04 18:32:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

s3: split the GCS quirks into -s3-use-x-id and -s3-sign-accept-encoding #8373

Browse Source 
Before this we applied both these quirks if provider == "GCS".

Splitting them like this makes them applicable for other providers
such as ActiveScale.
This commit is contained in:
Nick Craig-Wood 2025-02-11 20:22:03 +00:00
parent 64cd8ae0f0
commit e175c863aa
1 changed files with 81 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
backend/s3/s3.go
View File

@ -2680,6 +2680,34 @@ knows about - please make a bug report if not.
You can change this if you want to disable the use of multipart uploads.
This shouldn't be necessary in normal operation.
This should be automatically set correctly for all providers rclone
knows about - please make a bug report if not.
`,
Default: fs.Tristate{},
Advanced: true,
}, {
Name: "use_x_id",
Help: `Set if rclone should add x-id URL parameters.
You can change this if you want to disable the AWS SDK from
adding x-id URL parameters.
This shouldn't be necessary in normal operation.
This should be automatically set correctly for all providers rclone
knows about - please make a bug report if not.
`,
Default: fs.Tristate{},
Advanced: true,
}, {
Name: "sign_accept_encoding",
Help: `Set if rclone should include Accept-Encoding as part of the signature.
You can change this if you want to stop rclone including
Accept-Encoding as part of the signature.
This shouldn't be necessary in normal operation.
This should be automatically set correctly for all providers rclone
knows about - please make a bug report if not.
`,
@ -2901,6 +2929,8 @@ type Options struct {
DirectoryBucket bool `config:"directory_bucket"`
IBMAPIKey string `config:"ibm_api_key"`
IBMInstanceID string `config:"ibm_resource_instance_id"`
UseXID fs.Tristate `config:"use_x_id"`
SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`
}
// Fs represents a remote s3 server
@ -3085,26 +3115,29 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
}
}
// Fixup the request if needed.
//
// Google Cloud Storage alters the Accept-Encoding header, which
// breaks the v2 request signature
// breaks the v2 request signature. This is set with opt.SignAcceptEncoding.
//
// It also doesn't like the x-id URL parameter SDKv2 puts in so we
// remove that too.
// remove that too. This is set with opt.UseXID.Value.
//
// See https://github.com/aws/aws-sdk-go-v2/issues/1816.
// Adapted from: https://github.com/aws/aws-sdk-go-v2/issues/1816#issuecomment-1927281540
func fixupGCS(o *s3.Options) {
func fixupRequest(o *s3.Options, opt *Options) {
type ignoredHeadersKey struct{}
headers := []string{"Accept-Encoding"}
fixup := middleware.FinalizeMiddlewareFunc(
"FixupGCS",
"FixupRequest",
func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
req, ok := in.Request.(*smithyhttp.Request)
if !ok {
return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)
return out, metadata, fmt.Errorf("fixupRequest: unexpected request middleware type %T", in.Request)
}
if !opt.SignAcceptEncoding.Value {
// Delete headers from being signed - will restore later
ignored := make(map[string]string, len(headers))
for _, h := range headers {
@ -3112,14 +3145,17 @@ func fixupGCS(o *s3.Options) {
req.Header.Del(h)
}
// Remove x-id because Google doesn't like them
// Store ignored on context
ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)
}
if !opt.UseXID.Value {
// Remove x-id
if query := req.URL.Query(); query.Has("x-id") {
query.Del("x-id")
req.URL.RawQuery = query.Encode()
}
// Store ignored on context
ctx = middleware.WithStackValue(ctx, ignoredHeadersKey{}, ignored)
}
return next.HandleFinalize(ctx, in)
},
@ -3127,18 +3163,20 @@ func fixupGCS(o *s3.Options) {
// Restore headers if necessary
restore := middleware.FinalizeMiddlewareFunc(
"FixupGCSRestoreHeaders",
"FixupRequestRestoreHeaders",
func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (out middleware.FinalizeOutput, metadata middleware.Metadata, err error) {
req, ok := in.Request.(*smithyhttp.Request)
if !ok {
return out, metadata, fmt.Errorf("fixupGCS: unexpected request middleware type %T", in.Request)
return out, metadata, fmt.Errorf("fixupRequest: unexpected request middleware type %T", in.Request)
}
if !opt.SignAcceptEncoding.Value {
// Restore ignored from ctx
ignored, _ := middleware.GetStackValue(ctx, ignoredHeadersKey{}).(map[string]string)
for k, v := range ignored {
req.Header.Set(k, v)
}
}
return next.HandleFinalize(ctx, in)
},
@ -3267,9 +3305,10 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
}
}
if opt.Provider == "GCS" {
// Fixup the request if needed
if !opt.UseXID.Value || !opt.SignAcceptEncoding.Value {
options = append(options, func(o *s3.Options) {
fixupGCS(o)
fixupRequest(o, opt)
})
}
@ -3384,6 +3423,8 @@ func setQuirks(opt *Options) {
useAlreadyExists = true // Set if provider returns AlreadyOwnedByYou or no error if you try to remake your own bucket
useMultipartUploads = true // Set if provider supports multipart uploads
useUnsignedPayload = true // Do we need to use unsigned payloads to avoid seeking in PutObject
useXID = true // Add x-id URL parameter into requests
signAcceptEncoding = true // If we should include AcceptEncoding in the signature
)
switch opt.Provider {
case "AWS":
@ -3528,11 +3569,14 @@ func setQuirks(opt *Options) {
// Google break request Signature by mutating accept-encoding HTTP header
// https://github.com/rclone/rclone/issues/6670
useAcceptEncodingGzip = false
signAcceptEncoding = false
useAlreadyExists = true // returns BucketNameUnavailable instead of BucketAlreadyExists but good enough!
// GCS S3 doesn't support multi-part server side copy:
// See: https://issuetracker.google.com/issues/323465186
// So make cutoff very large which it does seem to support
opt.CopyCutoff = math.MaxInt64
// GCS doesn't like the x-id URL parameter the SDKv2 inserts
useXID = false
default: //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid defaultCaseOrder: consider to make `default` case as first or as last case
fs.Logf("s3", "s3 provider %q not known - please set correctly", opt.Provider)
fallthrough
@ -3602,6 +3646,18 @@ func setQuirks(opt *Options) {
opt.UseUnsignedPayload.Valid = true
opt.UseUnsignedPayload.Value = useUnsignedPayload
}
// Set the correct use UseXID if not manually set
if !opt.UseXID.Valid {
opt.UseXID.Valid = true
opt.UseXID.Value = useXID
}
// Set the correct SignAcceptEncoding if not manually set
if !opt.SignAcceptEncoding.Valid {
opt.SignAcceptEncoding.Valid = true
opt.SignAcceptEncoding.Value = signAcceptEncoding
}
}
// setRoot changes the root of the Fs