From 00a76dca83cb4627694409ea99440f958ddf3ef6 Mon Sep 17 00:00:00 2001 From: Alexander Minbaev Date: Wed, 30 Oct 2024 21:03:16 -0500 Subject: [PATCH] add IBM IAM signer --- backend/ibmcos/ibmSigner.go | 33 +++++++++++++++++++++++++++++++++ backend/s3/s3.go | 19 +++++++++++++++---- go.mod | 12 ++++++++++++ go.sum | 14 ++++++++++++++ 4 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 backend/ibmcos/ibmSigner.go diff --git a/backend/ibmcos/ibmSigner.go b/backend/ibmcos/ibmSigner.go new file mode 100644 index 000000000..b07ff2b52 --- /dev/null +++ b/backend/ibmcos/ibmSigner.go @@ -0,0 +1,33 @@ +package ibmcos + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/IBM/go-sdk-core/v5/core" + "github.com/aws/aws-sdk-go-v2/aws" + v4signer "github.com/aws/aws-sdk-go-v2/aws/signer/v4" +) + +// IBM IAM signer structure. Requeres APIKye and Resource InstanceID +type IbmIamSigner struct { + APIKey string + InstanceID string +} + +// SignHTTP signs requests using IBM IAM token +func (signer *IbmIamSigner) SignHTTP(ctx context.Context, credentials aws.Credentials, req *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*v4signer.SignerOptions)) error { + fmt.Println("SignHTTP called") // Debug print + authenticator := &core.IamAuthenticator{ + ApiKey: signer.APIKey, + } + token, err := authenticator.GetToken() + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("ibm-service-instance-id", signer.InstanceID) + return nil +} diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 3c62dd3c5..1e69b4707 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -36,8 +36,9 @@ import ( "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" smithyhttp "github.com/aws/smithy-go/transport/http" - "github.com/ncw/swift/v2" + + "github.com/rclone/rclone/backend/ibmcos" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/chunksize" @@ -2878,6 +2879,8 @@ type Options struct { UseUnsignedPayload fs.Tristate `config:"use_unsigned_payload"` SDKLogMode sdkLogMode `config:"sdk_log_mode"` DirectoryBucket bool `config:"directory_bucket"` + APIKey string `config:"api_key"` + InstanceID string `config:"resource_instance_id"` } // Fs represents a remote s3 server @@ -3160,6 +3163,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli // Try to fill in the config from the environment if env_auth=true if opt.EnvAuth && opt.AccessKeyID == "" && opt.SecretAccessKey == "" { + configOpts := []func(*awsconfig.LoadOptions) error{} // Set the name of the profile if supplied if opt.Profile != "" { @@ -3173,6 +3177,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli if err != nil { return nil, fmt.Errorf("couldn't load configuration with env_auth=true: %w", err) } + } else { switch { case opt.AccessKeyID == "" && opt.SecretAccessKey == "": @@ -3228,9 +3233,15 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli if opt.V2Auth || opt.Region == "other-v2-signature" { fs.Debugf(nil, "Using v2 auth") - options = append(options, func(s3Opt *s3.Options) { - s3Opt.HTTPSignerV4 = &v2Signer{opt: opt} - }) + if opt.Provider == "IBMCOS" { + options = append(options, func(s3Opt *s3.Options) { + s3Opt.HTTPSignerV4 = &ibmcos.IbmIamSigner{APIKey: opt.APIKey, InstanceID: opt.InstanceID} + }) + } else { + options = append(options, func(s3Opt *s3.Options) { + s3Opt.HTTPSignerV4 = &v2Signer{opt: opt} + }) + } } if opt.Provider == "GCS" { diff --git a/go.mod b/go.mod index fd6c96bc8..9de1f98ce 100644 --- a/go.mod +++ b/go.mod @@ -89,6 +89,7 @@ require ( gopkg.in/validator.v2 v2.0.1 gopkg.in/yaml.v2 v2.4.0 storj.io/uplink v1.13.1 + ) require ( @@ -107,6 +108,7 @@ require ( github.com/anacrolix/generics v0.0.1 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect @@ -142,6 +144,11 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/errors v0.21.0 // indirect + github.com/go-openapi/strfmt v0.22.1 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-resty/resty/v2 v2.11.0 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/gofrs/flock v0.8.1 // indirect @@ -168,11 +175,14 @@ require ( github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/lpar/date v1.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/md5-simd v1.1.2 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/oklog/ulid v1.3.1 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/panjf2000/ants/v2 v2.9.1 // indirect github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect @@ -199,6 +209,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/blake3 v0.2.3 // indirect github.com/zeebo/errs v1.3.0 // indirect + go.mongodb.org/mongo-driver v1.14.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect @@ -221,6 +232,7 @@ require ( ) require ( + github.com/IBM/go-sdk-core/v5 v5.17.5 github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.0.0 github.com/golang-jwt/jwt/v4 v4.5.0 diff --git a/go.sum b/go.sum index e1dff78eb..b97a10a04 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Files-com/files-sdk-go/v3 v3.2.34 h1:j6gSzu6BF1wWH1z4itRe7eKhQSCrx/I78SDNiBBUtvI= github.com/Files-com/files-sdk-go/v3 v3.2.34/go.mod h1:Y/bCHoPJNPKz2hw1ADXjQXJP378HODwK+g/5SR2gqfU= +github.com/IBM/go-sdk-core/v5 v5.17.5 h1:AjGC7xNee5tgDIjndekBDW5AbypdERHSgib3EZ1KNsA= +github.com/IBM/go-sdk-core/v5 v5.17.5/go.mod h1:KsAAI7eStAWwQa4F96MLy+whYSh39JzNjklZRbN/8ns= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= @@ -102,6 +104,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er+LKDHDUfU5tRvFwxH0LjPpZN8OoG8Ll+liGU= github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc/go.mod h1:w648aMHEgFYS6xb0KVMMtZ2uMeemhiKCuD2vj6gY52A= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= @@ -256,6 +260,10 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/errors v0.21.0 h1:FhChC/duCnfoLj1gZ0BgaBmzhJC2SL/sJr8a2vAobSY= +github.com/go-openapi/errors v0.21.0/go.mod h1:jxNTMUxRCKj65yb/okJGEtahVd7uvWnuWfj53bse4ho= +github.com/go-openapi/strfmt v0.22.1 h1:5Ky8cybT4576C6Ffc+8gYji/wRXCo6Ozm8RaWjPI6jc= +github.com/go-openapi/strfmt v0.22.1/go.mod h1:OfVoytIXJasDkkGvkb1Cceb3BPyMOwk1FgmyyEw7NYg= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -454,6 +462,8 @@ github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= @@ -466,6 +476,8 @@ github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfb github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -636,6 +648,8 @@ github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= +go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=