mirror of
https://github.com/rclone/rclone.git
synced 2025-02-02 03:29:51 +01:00
Update to AWS SDK 0.10.0
Tested with S3 and Dreamhost Here is a link to the release notes: http://aws.amazon.com/releasenotes/5476699172355228
This commit is contained in:
parent
b872ff0237
commit
345c98ed62
42
s3/s3.go
42
s3/s3.go
@ -27,6 +27,7 @@ import (
|
|||||||
"github.com/aws/aws-sdk-go/aws/corehandlers"
|
"github.com/aws/aws-sdk-go/aws/corehandlers"
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
"github.com/aws/aws-sdk-go/aws/request"
|
"github.com/aws/aws-sdk-go/aws/request"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
"github.com/aws/aws-sdk-go/service/s3"
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||||
"github.com/ncw/rclone/fs"
|
"github.com/ncw/rclone/fs"
|
||||||
@ -129,12 +130,13 @@ const (
|
|||||||
|
|
||||||
// FsS3 represents a remote s3 server
|
// FsS3 represents a remote s3 server
|
||||||
type FsS3 struct {
|
type FsS3 struct {
|
||||||
name string // the name of the remote
|
name string // the name of the remote
|
||||||
c *s3.S3 // the connection to the s3 server
|
c *s3.S3 // the connection to the s3 server
|
||||||
bucket string // the bucket we are working on
|
ses *session.Session // the s3 session
|
||||||
perm string // permissions for new buckets / objects
|
bucket string // the bucket we are working on
|
||||||
root string // root of the bucket - ignore all objects above this
|
perm string // permissions for new buckets / objects
|
||||||
locationConstraint string // location constraint of new buckets
|
root string // root of the bucket - ignore all objects above this
|
||||||
|
locationConstraint string // location constraint of new buckets
|
||||||
}
|
}
|
||||||
|
|
||||||
// FsObjectS3 describes a s3 object
|
// FsObjectS3 describes a s3 object
|
||||||
@ -190,7 +192,7 @@ func s3ParsePath(path string) (bucket, directory string, err error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// s3Connection makes a connection to s3
|
// s3Connection makes a connection to s3
|
||||||
func s3Connection(name string) (*s3.S3, error) {
|
func s3Connection(name string) (*s3.S3, *session.Session, error) {
|
||||||
// Make the auth
|
// Make the auth
|
||||||
accessKeyID := fs.ConfigFile.MustValue(name, "access_key_id")
|
accessKeyID := fs.ConfigFile.MustValue(name, "access_key_id")
|
||||||
secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key")
|
secretAccessKey := fs.ConfigFile.MustValue(name, "secret_access_key")
|
||||||
@ -200,9 +202,9 @@ func s3Connection(name string) (*s3.S3, error) {
|
|||||||
fs.Debug(name, "Using anonymous access for S3")
|
fs.Debug(name, "Using anonymous access for S3")
|
||||||
auth = credentials.AnonymousCredentials
|
auth = credentials.AnonymousCredentials
|
||||||
case accessKeyID == "":
|
case accessKeyID == "":
|
||||||
return nil, errors.New("access_key_id not found")
|
return nil, nil, errors.New("access_key_id not found")
|
||||||
case secretAccessKey == "":
|
case secretAccessKey == "":
|
||||||
return nil, errors.New("secret_access_key not found")
|
return nil, nil, errors.New("secret_access_key not found")
|
||||||
default:
|
default:
|
||||||
auth = credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "")
|
auth = credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "")
|
||||||
}
|
}
|
||||||
@ -223,12 +225,13 @@ func s3Connection(name string) (*s3.S3, error) {
|
|||||||
WithHTTPClient(fs.Config.Client()).
|
WithHTTPClient(fs.Config.Client()).
|
||||||
WithS3ForcePathStyle(true)
|
WithS3ForcePathStyle(true)
|
||||||
// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
|
// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
|
||||||
c := s3.New(awsConfig)
|
ses := session.New()
|
||||||
|
c := s3.New(ses, awsConfig)
|
||||||
if region == "other-v2-signature" {
|
if region == "other-v2-signature" {
|
||||||
fs.Debug(name, "Using v2 auth")
|
fs.Debug(name, "Using v2 auth")
|
||||||
signer := func(req *request.Request) {
|
signer := func(req *request.Request) {
|
||||||
// Ignore AnonymousCredentials object
|
// Ignore AnonymousCredentials object
|
||||||
if req.Service.Config.Credentials == credentials.AnonymousCredentials {
|
if req.Config.Credentials == credentials.AnonymousCredentials {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
sign(accessKeyID, secretAccessKey, req.HTTPRequest)
|
sign(accessKeyID, secretAccessKey, req.HTTPRequest)
|
||||||
@ -241,7 +244,7 @@ func s3Connection(name string) (*s3.S3, error) {
|
|||||||
c.Handlers.Build.PushBack(func(r *request.Request) {
|
c.Handlers.Build.PushBack(func(r *request.Request) {
|
||||||
r.HTTPRequest.Header.Set("User-Agent", fs.UserAgent)
|
r.HTTPRequest.Header.Set("User-Agent", fs.UserAgent)
|
||||||
})
|
})
|
||||||
return c, nil
|
return c, ses, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFs contstructs an FsS3 from the path, bucket:path
|
// NewFs contstructs an FsS3 from the path, bucket:path
|
||||||
@ -250,7 +253,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
c, err := s3Connection(name)
|
c, ses, err := s3Connection(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -258,6 +261,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
|||||||
name: name,
|
name: name,
|
||||||
c: c,
|
c: c,
|
||||||
bucket: bucket,
|
bucket: bucket,
|
||||||
|
ses: ses,
|
||||||
// FIXME perm: s3.Private, // FIXME need user to specify
|
// FIXME perm: s3.Private, // FIXME need user to specify
|
||||||
root: directory,
|
root: directory,
|
||||||
locationConstraint: fs.ConfigFile.MustValue(name, "location_constraint"),
|
locationConstraint: fs.ConfigFile.MustValue(name, "location_constraint"),
|
||||||
@ -674,13 +678,11 @@ func (o *FsObjectS3) Open() (in io.ReadCloser, err error) {
|
|||||||
|
|
||||||
// Update the Object from in with modTime and size
|
// Update the Object from in with modTime and size
|
||||||
func (o *FsObjectS3) Update(in io.Reader, modTime time.Time, size int64) error {
|
func (o *FsObjectS3) Update(in io.Reader, modTime time.Time, size int64) error {
|
||||||
opts := s3manager.UploadOptions{
|
uploader := s3manager.NewUploader(o.s3.ses, func(u *s3manager.Uploader) {
|
||||||
// PartSize: 64 * 1024 * 1024, use default
|
u.Concurrency = 2
|
||||||
Concurrency: 2, // limit concurrency
|
u.LeavePartsOnError = false
|
||||||
LeavePartsOnError: false,
|
u.S3 = o.s3.c
|
||||||
S3: o.s3.c,
|
})
|
||||||
}
|
|
||||||
uploader := s3manager.NewUploader(&opts)
|
|
||||||
|
|
||||||
// Set the mtime in the meta data
|
// Set the mtime in the meta data
|
||||||
metadata := map[string]*string{
|
metadata := map[string]*string{
|
||||||
|
Loading…
Reference in New Issue
Block a user