From 4b981100db7196fe221e8daa7e77a9dce1a171d7 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 28 Jul 2022 11:49:19 +0100 Subject: [PATCH] s3: refactor to use generated code instead of reflection to copy structs --- backend/s3/gen_setfrom.go | 101 ++++++++++++++++ backend/s3/s3.go | 39 ++++--- backend/s3/setfrom.go | 239 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 366 insertions(+), 13 deletions(-) create mode 100644 backend/s3/gen_setfrom.go create mode 100644 backend/s3/setfrom.go diff --git a/backend/s3/gen_setfrom.go b/backend/s3/gen_setfrom.go new file mode 100644 index 000000000..b3e0247f2 --- /dev/null +++ b/backend/s3/gen_setfrom.go @@ -0,0 +1,101 @@ +// Generate boilerplate code for setting similar structs from each other + +//go:build ignore +// +build ignore + +package main + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "reflect" + "strings" + + "github.com/aws/aws-sdk-go/service/s3" +) + +// flags +var ( + outputFile = flag.String("o", "", "Output file name, stdout if unset") +) + +// globals +var ( + out io.Writer = os.Stdout +) + +// genSetFrom generates code to set the public members of a from b +// +// a and b should be pointers to structs +// +// a can be a different type from b +// +// Only the Fields which have the same name and assignable type on a +// and b will be set. +// +// This is useful for copying between almost identical structures that +// are frequently present in auto-generated code for cloud storage +// interfaces. +func genSetFrom(a, b interface{}) { + name := fmt.Sprintf("setFrom_%T_%T", a, b) + name = strings.Replace(name, ".", "", -1) + name = strings.Replace(name, "*", "", -1) + fmt.Fprintf(out, "\n// %s copies matching elements from a to b\n", name) + fmt.Fprintf(out, "func %s(a %T, b %T) {\n", name, a, b) + ta := reflect.TypeOf(a).Elem() + tb := reflect.TypeOf(b).Elem() + va := reflect.ValueOf(a).Elem() + vb := reflect.ValueOf(b).Elem() + for i := 0; i < tb.NumField(); i++ { + bField := vb.Field(i) + tbField := tb.Field(i) + name := tbField.Name + aField := va.FieldByName(name) + taField, found := ta.FieldByName(name) + if found && aField.IsValid() && bField.IsValid() && aField.CanSet() && tbField.Type.AssignableTo(taField.Type) { + fmt.Fprintf(out, "\ta.%s = b.%s\n", name, name) + } + } + fmt.Fprintf(out, "}\n") +} + +func main() { + flag.Parse() + + if *outputFile != "" { + fd, err := os.Create(*outputFile) + if err != nil { + log.Fatal(err) + } + defer func() { + err := fd.Close() + if err != nil { + log.Fatal(err) + } + }() + out = fd + } + + fmt.Fprintf(out, `// Code generated by "go run gen_setfrom.go"; DO NOT EDIT. + +package s3 + +import "github.com/aws/aws-sdk-go/service/s3" +`) + + genSetFrom(new(s3.ListObjectsInput), new(s3.ListObjectsV2Input)) + genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectsOutput)) + genSetFrom(new(s3.ListObjectVersionsInput), new(s3.ListObjectsV2Input)) + genSetFrom(new(s3.ObjectVersion), new(s3.DeleteMarkerEntry)) + genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectVersionsOutput)) + genSetFrom(new(s3.Object), new(s3.ObjectVersion)) + genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.HeadObjectOutput)) + genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.CopyObjectInput)) + genSetFrom(new(s3.UploadPartCopyInput), new(s3.CopyObjectInput)) + genSetFrom(new(s3.HeadObjectOutput), new(s3.GetObjectOutput)) + genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.PutObjectInput)) + genSetFrom(new(s3.HeadObjectOutput), new(s3.PutObjectInput)) +} diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 4f7dd8aee..78f23b3ed 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1,6 +1,8 @@ // Package s3 provides an interface to Amazon S3 oject storage package s3 +//go:generate go run gen_setfrom.go -o setfrom.go + import ( "bytes" "context" @@ -54,7 +56,6 @@ import ( "github.com/rclone/rclone/lib/pool" "github.com/rclone/rclone/lib/readers" "github.com/rclone/rclone/lib/rest" - "github.com/rclone/rclone/lib/structs" "github.com/rclone/rclone/lib/version" "golang.org/x/sync/errgroup" ) @@ -2866,7 +2867,8 @@ func (f *Fs) newV1List(req *s3.ListObjectsV2Input) bucketLister { f: f, } // Convert v2 req into v1 req - structs.SetFrom(&l.req, req) + //structs.SetFrom(&l.req, req) + setFrom_s3ListObjectsInput_s3ListObjectsV2Input(&l.req, req) return l } @@ -2898,7 +2900,8 @@ func (ls *v1List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versi // convert v1 resp into v2 resp resp = new(s3.ListObjectsV2Output) - structs.SetFrom(resp, respv1) + //structs.SetFrom(resp, respv1) + setFrom_s3ListObjectsV2Output_s3ListObjectsOutput(resp, respv1) return resp, nil, nil } @@ -2961,7 +2964,8 @@ func (f *Fs) newVersionsList(req *s3.ListObjectsV2Input, hidden bool, versionAt hidden: hidden, } // Convert v2 req into withVersions req - structs.SetFrom(&l.req, req) + //structs.SetFrom(&l.req, req) + setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input(&l.req, req) return l } @@ -3011,7 +3015,8 @@ func mergeDeleteMarkers(oldVersions []*s3.ObjectVersion, deleteMarkers []*s3.Del newVersions = make([]*s3.ObjectVersion, 0, len(oldVersions)+len(deleteMarkers)) for _, deleteMarker := range deleteMarkers { var obj = new(s3.ObjectVersion) - structs.SetFrom(obj, deleteMarker) + //structs.SetFrom(obj, deleteMarker) + setFrom_s3ObjectVersion_s3DeleteMarkerEntry(obj, deleteMarker) obj.Size = isDeleteMarker for len(oldVersions) > 0 && versionLess(oldVersions[0], obj) { newVersions = append(newVersions, oldVersions[0]) @@ -3045,7 +3050,8 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output, // convert Versions resp into v2 resp resp = new(s3.ListObjectsV2Output) - structs.SetFrom(resp, respVersions) + //structs.SetFrom(resp, respVersions) + setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(resp, respVersions) // Merge in delete Markers as s3.ObjectVersion if we need them if ls.hidden || ls.usingVersionAt { @@ -3074,7 +3080,8 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output, continue } var obj = new(s3.Object) - structs.SetFrom(obj, objVersion) + //structs.SetFrom(obj, objVersion) + setFrom_s3Object_s3ObjectVersion(obj, objVersion) // Adjust the file names if !ls.usingVersionAt && !aws.BoolValue(objVersion.IsLatest) { if obj.Key != nil && objVersion.LastModified != nil { @@ -3611,7 +3618,8 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst req := &s3.CreateMultipartUploadInput{} // Fill in the request from the head info - structs.SetFrom(req, info) + //structs.SetFrom(req, info) + setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(req, info) // If copy metadata was set then set the Metadata to that read // from the head request @@ -3620,7 +3628,8 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst } // Overwrite any from the copyReq - structs.SetFrom(req, copyReq) + //structs.SetFrom(req, copyReq) + setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(req, copyReq) req.Bucket = &dstBucket req.Key = &dstPath @@ -3660,7 +3669,8 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst if err := f.pacer.Call(func() (bool, error) { partNum := partNum uploadPartReq := &s3.UploadPartCopyInput{} - structs.SetFrom(uploadPartReq, copyReq) + //structs.SetFrom(uploadPartReq, copyReq) + setFrom_s3UploadPartCopyInput_s3CopyObjectInput(uploadPartReq, copyReq) uploadPartReq.Bucket = &dstBucket uploadPartReq.Key = &dstPath uploadPartReq.PartNumber = &partNum @@ -4632,7 +4642,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } } var head s3.HeadObjectOutput - structs.SetFrom(&head, resp) + //structs.SetFrom(&head, resp) + setFrom_s3HeadObjectOutput_s3GetObjectOutput(&head, resp) head.ContentLength = size o.setMetaData(&head) return resp.Body, nil @@ -4675,7 +4686,8 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si memPool := f.getMemoryPool(int64(partSize)) var mReq s3.CreateMultipartUploadInput - structs.SetFrom(&mReq, req) + //structs.SetFrom(&mReq, req) + setFrom_s3CreateMultipartUploadInput_s3PutObjectInput(&mReq, req) var cout *s3.CreateMultipartUploadOutput err = f.pacer.Call(func() (bool, error) { var err error @@ -5144,7 +5156,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // uploaded properly. If size < 0 then we need to do the HEAD. if o.fs.opt.NoHead && size >= 0 { var head s3.HeadObjectOutput - structs.SetFrom(&head, &req) + //structs.SetFrom(&head, &req) + setFrom_s3HeadObjectOutput_s3PutObjectInput(&head, &req) head.ETag = &md5sumHex // doesn't matter quotes are misssing head.ContentLength = &size // If we have done a single part PUT request then we can read these diff --git a/backend/s3/setfrom.go b/backend/s3/setfrom.go new file mode 100644 index 000000000..1f7000b24 --- /dev/null +++ b/backend/s3/setfrom.go @@ -0,0 +1,239 @@ +// Code generated by "go run gen_setfrom.go"; DO NOT EDIT. + +package s3 + +import "github.com/aws/aws-sdk-go/service/s3" + +// setFrom_s3ListObjectsInput_s3ListObjectsV2Input copies matching elements from a to b +func setFrom_s3ListObjectsInput_s3ListObjectsV2Input(a *s3.ListObjectsInput, b *s3.ListObjectsV2Input) { + a.Bucket = b.Bucket + a.Delimiter = b.Delimiter + a.EncodingType = b.EncodingType + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.MaxKeys = b.MaxKeys + a.Prefix = b.Prefix + a.RequestPayer = b.RequestPayer +} + +// setFrom_s3ListObjectsV2Output_s3ListObjectsOutput copies matching elements from a to b +func setFrom_s3ListObjectsV2Output_s3ListObjectsOutput(a *s3.ListObjectsV2Output, b *s3.ListObjectsOutput) { + a.CommonPrefixes = b.CommonPrefixes + a.Contents = b.Contents + a.Delimiter = b.Delimiter + a.EncodingType = b.EncodingType + a.IsTruncated = b.IsTruncated + a.MaxKeys = b.MaxKeys + a.Name = b.Name + a.Prefix = b.Prefix +} + +// setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input copies matching elements from a to b +func setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input(a *s3.ListObjectVersionsInput, b *s3.ListObjectsV2Input) { + a.Bucket = b.Bucket + a.Delimiter = b.Delimiter + a.EncodingType = b.EncodingType + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.MaxKeys = b.MaxKeys + a.Prefix = b.Prefix +} + +// setFrom_s3ObjectVersion_s3DeleteMarkerEntry copies matching elements from a to b +func setFrom_s3ObjectVersion_s3DeleteMarkerEntry(a *s3.ObjectVersion, b *s3.DeleteMarkerEntry) { + a.IsLatest = b.IsLatest + a.Key = b.Key + a.LastModified = b.LastModified + a.Owner = b.Owner + a.VersionId = b.VersionId +} + +// setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput copies matching elements from a to b +func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsV2Output, b *s3.ListObjectVersionsOutput) { + a.CommonPrefixes = b.CommonPrefixes + a.Delimiter = b.Delimiter + a.EncodingType = b.EncodingType + a.IsTruncated = b.IsTruncated + a.MaxKeys = b.MaxKeys + a.Name = b.Name + a.Prefix = b.Prefix +} + +// setFrom_s3Object_s3ObjectVersion copies matching elements from a to b +func setFrom_s3Object_s3ObjectVersion(a *s3.Object, b *s3.ObjectVersion) { + a.ChecksumAlgorithm = b.ChecksumAlgorithm + a.ETag = b.ETag + a.Key = b.Key + a.LastModified = b.LastModified + a.Owner = b.Owner + a.Size = b.Size + a.StorageClass = b.StorageClass +} + +// setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput copies matching elements from a to b +func setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(a *s3.CreateMultipartUploadInput, b *s3.HeadObjectOutput) { + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentType = b.ContentType + a.Metadata = b.Metadata + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +} + +// setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput copies matching elements from a to b +func setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(a *s3.CreateMultipartUploadInput, b *s3.CopyObjectInput) { + a.ACL = b.ACL + a.Bucket = b.Bucket + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ChecksumAlgorithm = b.ChecksumAlgorithm + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentType = b.ContentType + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.Expires = b.Expires + a.GrantFullControl = b.GrantFullControl + a.GrantRead = b.GrantRead + a.GrantReadACP = b.GrantReadACP + a.GrantWriteACP = b.GrantWriteACP + a.Key = b.Key + a.Metadata = b.Metadata + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.RequestPayer = b.RequestPayer + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKey = b.SSECustomerKey + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSEncryptionContext = b.SSEKMSEncryptionContext + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.Tagging = b.Tagging + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +} + +// setFrom_s3UploadPartCopyInput_s3CopyObjectInput copies matching elements from a to b +func setFrom_s3UploadPartCopyInput_s3CopyObjectInput(a *s3.UploadPartCopyInput, b *s3.CopyObjectInput) { + a.Bucket = b.Bucket + a.CopySource = b.CopySource + a.CopySourceIfMatch = b.CopySourceIfMatch + a.CopySourceIfModifiedSince = b.CopySourceIfModifiedSince + a.CopySourceIfNoneMatch = b.CopySourceIfNoneMatch + a.CopySourceIfUnmodifiedSince = b.CopySourceIfUnmodifiedSince + a.CopySourceSSECustomerAlgorithm = b.CopySourceSSECustomerAlgorithm + a.CopySourceSSECustomerKey = b.CopySourceSSECustomerKey + a.CopySourceSSECustomerKeyMD5 = b.CopySourceSSECustomerKeyMD5 + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.ExpectedSourceBucketOwner = b.ExpectedSourceBucketOwner + a.Key = b.Key + a.RequestPayer = b.RequestPayer + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKey = b.SSECustomerKey + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 +} + +// setFrom_s3HeadObjectOutput_s3GetObjectOutput copies matching elements from a to b +func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.GetObjectOutput) { + a.AcceptRanges = b.AcceptRanges + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ChecksumCRC32 = b.ChecksumCRC32 + a.ChecksumCRC32C = b.ChecksumCRC32C + a.ChecksumSHA1 = b.ChecksumSHA1 + a.ChecksumSHA256 = b.ChecksumSHA256 + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentLength = b.ContentLength + a.ContentType = b.ContentType + a.DeleteMarker = b.DeleteMarker + a.ETag = b.ETag + a.Expiration = b.Expiration + a.Expires = b.Expires + a.LastModified = b.LastModified + a.Metadata = b.Metadata + a.MissingMeta = b.MissingMeta + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.PartsCount = b.PartsCount + a.ReplicationStatus = b.ReplicationStatus + a.RequestCharged = b.RequestCharged + a.Restore = b.Restore + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.VersionId = b.VersionId + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +} + +// setFrom_s3CreateMultipartUploadInput_s3PutObjectInput copies matching elements from a to b +func setFrom_s3CreateMultipartUploadInput_s3PutObjectInput(a *s3.CreateMultipartUploadInput, b *s3.PutObjectInput) { + a.ACL = b.ACL + a.Bucket = b.Bucket + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ChecksumAlgorithm = b.ChecksumAlgorithm + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentType = b.ContentType + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.Expires = b.Expires + a.GrantFullControl = b.GrantFullControl + a.GrantRead = b.GrantRead + a.GrantReadACP = b.GrantReadACP + a.GrantWriteACP = b.GrantWriteACP + a.Key = b.Key + a.Metadata = b.Metadata + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.RequestPayer = b.RequestPayer + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKey = b.SSECustomerKey + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSEncryptionContext = b.SSEKMSEncryptionContext + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.Tagging = b.Tagging + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +} + +// setFrom_s3HeadObjectOutput_s3PutObjectInput copies matching elements from a to b +func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.PutObjectInput) { + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ChecksumCRC32 = b.ChecksumCRC32 + a.ChecksumCRC32C = b.ChecksumCRC32C + a.ChecksumSHA1 = b.ChecksumSHA1 + a.ChecksumSHA256 = b.ChecksumSHA256 + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentLength = b.ContentLength + a.ContentType = b.ContentType + a.Metadata = b.Metadata + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +}