extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-07 09:04:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

fshttp: read config from ctx not passed in ConfigInfo #4685

Browse Source
This commit is contained in:
Nick Craig-Wood 2020-11-13 15:24:43 +00:00
parent 2e21c58e6a
commit 9d574c0d63
30 changed files with 52 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/amazonclouddrive/amazonclouddrive.go
View File

@ -248,7 +248,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, err
}
root = parsePath(root)
baseClient := fshttp.NewClient(fs.GetConfig(ctx))
baseClient := fshttp.NewClient(ctx)
if do, ok := baseClient.Transport.(interface {
SetRequestFilter(f func(req *http.Request))
}); ok {
@ -270,7 +270,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci: ci,
c: c,
pacer: fs.NewPacer(ctx, pacer.NewAmazonCloudDrive(pacer.MinSleep(minSleep))),
noAuthClient: fshttp.NewClient(ci),
noAuthClient: fshttp.NewClient(ctx),
}
f.features = (&fs.Features{
CaseInsensitive: true,

2
backend/azureblob/azureblob.go
View File

@ -417,7 +417,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci: ci,
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
uploadToken: pacer.NewTokenDispenser(ci.Transfers),
client: fshttp.NewClient(fs.GetConfig(ctx)),
client: fshttp.NewClient(ctx),
cache: bucket.NewCache(),
cntURLcache: make(map[string]*azblob.ContainerURL, 1),
pool: pool.New(

2
backend/b2/b2.go
View File

@ -421,7 +421,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
name: name,
opt: *opt,
ci: ci,
srv: rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetErrorHandler(errorHandler),
srv: rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
cache: bucket.NewCache(),
_bucketID: make(map[string]string, 1),
_bucketType: make(map[string]string, 1),

4
backend/box/box.go
View File

@ -169,7 +169,7 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
}
signingHeaders := getSigningHeaders(boxConfig)
queryParams := getQueryParams(boxConfig)
client := fshttp.NewClient(fs.GetConfig(ctx))
client := fshttp.NewClient(ctx)
err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client)
return err
}
@ -386,7 +386,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
root = parsePath(root)
client := fshttp.NewClient(fs.GetConfig(ctx))
client := fshttp.NewClient(ctx)
var ts *oauthutil.TokenSource
// If not using an accessToken, create an oauth client and tokensource
if opt.AccessToken == "" {

2
backend/drive/drive.go
View File

@ -983,7 +983,7 @@ func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name
// getClient makes an http client according to the options
func getClient(ctx context.Context, opt *Options) *http.Client {
t := fshttp.NewTransportCustom(fs.GetConfig(ctx), func(t *http.Transport) {
t := fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
if opt.DisableHTTP2 {
t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
}

2
backend/fichier/fichier.go
View File

@ -195,7 +195,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
CanHaveEmptyDirectories: true,
}).Fill(ctx, f)
client := fshttp.NewClient(fs.GetConfig(ctx))
client := fshttp.NewClient(ctx)
f.rest = rest.NewClient(client).SetRoot(apiBaseURL)

2
backend/filefabric/filefabric.go
View File

@ -425,7 +425,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
root = parsePath(root)
client := fshttp.NewClient(fs.GetConfig(ctx))
client := fshttp.NewClient(ctx)
f := &Fs{
name: name,

2
backend/googlecloudstorage/googlecloudstorage.go
View File

@ -375,7 +375,7 @@ func getServiceAccountClient(ctx context.Context, credentialsData []byte) (*http
if err != nil {
return nil, errors.Wrap(err, "error processing credentials")
}
ctxWithSpecialClient := oauthutil.Context(ctx, fshttp.NewClient(fs.GetConfig(ctx)))
ctxWithSpecialClient := oauthutil.Context(ctx, fshttp.NewClient(ctx))
return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
}

2
backend/googlephotos/googlephotos.go
View File

@ -254,7 +254,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, err
}
baseClient := fshttp.NewClient(fs.GetConfig(ctx))
baseClient := fshttp.NewClient(ctx)
oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
if err != nil {
return nil, errors.Wrap(err, "failed to configure Box")

2
backend/http/http.go
View File

@ -172,7 +172,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, err
}
client := fshttp.NewClient(fs.GetConfig(ctx))
client := fshttp.NewClient(ctx)
var isFile = false
if !strings.HasSuffix(u.String(), "/") {

2
backend/hubic/hubic.go
View File

@ -162,7 +162,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
Auth: newAuth(f),
ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
Timeout: 10 * ci.Timeout, // Use the timeouts in the transport
Transport: fshttp.NewTransport(fs.GetConfig(ctx)),
Transport: fshttp.NewTransport(ctx),
}
err = c.Authenticate()
if err != nil {

6
backend/jottacloud/jottacloud.go
View File

@ -230,7 +230,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
// v1config configure a jottacloud backend using legacy authentication
func v1config(ctx context.Context, name string, m configmap.Mapper) {
srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx)))
srv := rest.NewClient(fshttp.NewClient(ctx))
fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
if config.Confirm(false) {
@ -365,7 +365,7 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
// v2config configure a jottacloud backend using the modern JottaCli token based authentication
func v2config(ctx context.Context, name string, m configmap.Mapper) {
srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx)))
srv := rest.NewClient(fshttp.NewClient(ctx))
fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
fmt.Printf("Login Token> ")
@ -661,7 +661,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, errors.New("Outdated config - please reconfigure this backend")
}
baseClient := fshttp.NewClient(fs.GetConfig(ctx))
baseClient := fshttp.NewClient(ctx)
if ver == configVersion {
oauthConfig.ClientID = "jottacli"

2
backend/koofr/koofr.go
View File

@ -267,7 +267,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
return nil, err
}
httpClient := httpclient.New()
httpClient.Client = fshttp.NewClient(fs.GetConfig(ctx))
httpClient.Client = fshttp.NewClient(ctx)
client := koofrclient.NewKoofrClientWithHTTPClient(opt.Endpoint, httpClient)
basicAuth := fmt.Sprintf("Basic %s",
base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass)))

4
backend/mailru/mailru.go
View File

@ -338,12 +338,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}).Fill(ctx, f)
// Override few config settings and create a client
clientConfig := *fs.GetConfig(ctx)
newCtx, clientConfig := fs.AddConfig(ctx)
if opt.UserAgent != "" {
clientConfig.UserAgent = opt.UserAgent
}
clientConfig.NoGzip = true // Mimic official client, skip sending "Accept-Encoding: gzip"
f.cli = fshttp.NewClient(&clientConfig)
f.cli = fshttp.NewClient(newCtx)
f.srv = rest.NewClient(f.cli)
f.srv.SetRoot(api.APIServerURL)

2
backend/mega/mega.go
View File

@ -205,7 +205,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
defer megaCacheMu.Unlock()
srv := megaCache[opt.User]
if srv == nil {
srv = mega.New().SetClient(fshttp.NewClient(fs.GetConfig(ctx)))
srv = mega.New().SetClient(fshttp.NewClient(ctx))
srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
srv.SetLogger(func(format string, v ...interface{}) {
fs.Infof("*go-mega*", format, v...)

2
backend/opendrive/opendrive.go
View File

@ -187,7 +187,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
name: name,
root: root,
opt: *opt,
srv: rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetErrorHandler(errorHandler),
srv: rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}

2
backend/premiumizeme/premiumizeme.go
View File

@ -252,7 +252,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, errors.Wrap(err, "failed to configure premiumize.me")
}
} else {
client = fshttp.NewClient(fs.GetConfig(ctx))
client = fshttp.NewClient(ctx)
}
f := &Fs{

2
backend/putio/fs.go
View File

@ -77,7 +77,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (f fs.Fs,
return nil, err
}
root = parsePath(root)
httpClient := fshttp.NewClient(fs.GetConfig(ctx))
httpClient := fshttp.NewClient(ctx)
oAuthClient, _, err := oauthutil.NewClientWithBaseClient(ctx, name, m, putioConfig, httpClient)
if err != nil {
return nil, errors.Wrap(err, "failed to configure putio")

2
backend/qingstor/qingstor.go
View File

@ -277,7 +277,7 @@ func qsServiceConnection(ctx context.Context, opt *Options) (*qs.Service, error)
cf.Host = host
cf.Port = port
// unsupported in v3.1: cf.ConnectionRetries = opt.ConnectionRetries
cf.Connection = fshttp.NewClient(fs.GetConfig(ctx))
cf.Connection = fshttp.NewClient(ctx)
return qs.Init(cf)
}

2
backend/s3/s3.go
View File

@ -1405,7 +1405,7 @@ func (o *Object) split() (bucket, bucketPath string) {
// getClient makes an http client according to the options
func getClient(ctx context.Context, opt *Options) *http.Client {
// TODO: Do we need cookies too?
t := fshttp.NewTransportCustom(fs.GetConfig(ctx), func(t *http.Transport) {
t := fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
if opt.DisableHTTP2 {
t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
}

4
backend/seafile/seafile.go
View File

@ -197,7 +197,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
opt: *opt,
endpoint: u,
endpointURL: u.String(),
srv: rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(u.String()),
srv: rest.NewClient(fshttp.NewClient(ctx)).SetRoot(u.String()),
pacer: getPacer(ctx, opt.URL),
}
f.features = (&fs.Features{
@ -343,7 +343,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper) {
if !strings.HasPrefix(url, "/") {
url += "/"
}
srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(url)
srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(url)
// We loop asking for a 2FA code
for {

2
backend/sftp/sftp.go
View File

@ -254,7 +254,7 @@ type Object struct {
// convenience function that connects to the given network address,
// initiates the SSH handshake, and then sets up a Client.
func (f *Fs) dial(ctx context.Context, network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
dialer := fshttp.NewDialer(fs.GetConfig(ctx))
dialer := fshttp.NewDialer(ctx)
conn, err := dialer.Dial(network, addr)
if err != nil {
return nil, err

4
backend/sugarsync/sugarsync.go
View File

@ -106,7 +106,7 @@ func init() {
Method: "POST",
Path: "/app-authorization",
}
srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(rootURL) // FIXME
srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(rootURL) // FIXME
// FIXME
//err = f.pacer.Call(func() (bool, error) {
@ -403,7 +403,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
root = parsePath(root)
client := fshttp.NewClient(fs.GetConfig(ctx))
client := fshttp.NewClient(ctx)
f := &Fs{
name: name,
root: root,

2
backend/swift/swift.go
View File

@ -363,7 +363,7 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
EndpointType: swift.EndpointType(opt.EndpointType),
ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
Timeout: 10 * ci.Timeout, // Use the timeouts in the transport
Transport: fshttp.NewTransport(fs.GetConfig(ctx)),
Transport: fshttp.NewTransport(ctx),
}
if opt.EnvAuth {
err := c.ApplyEnvironment()

2
backend/webdav/odrvcookie/fetch.go
View File

@ -182,7 +182,7 @@ func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SharepointSuccessRe
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
client := fshttp.NewClient(fs.GetConfig(ctx))
client := fshttp.NewClient(ctx)
resp, err := client.Do(req)
if err != nil {
return nil, errors.Wrap(err, "Error while logging in to endpoint")

2
backend/webdav/webdav.go
View File

@ -336,7 +336,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
opt: *opt,
endpoint: u,
endpointURL: u.String(),
srv: rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(u.String()),
srv: rest.NewClient(fshttp.NewClient(ctx)).SetRoot(u.String()),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
precision: fs.ModTimeNotSupported,
}

2
cmd/rc/rc.go
View File

@ -177,7 +177,7 @@ func doCall(ctx context.Context, path string, in rc.Params) (out rc.Params, err
}
// Do HTTP request
client := fshttp.NewClient(fs.GetConfig(ctx))
client := fshttp.NewClient(ctx)
url += path
data, err := json.Marshal(in)
if err != nil {

26
fs/fshttp/http.go
View File

@ -35,13 +35,14 @@ var (
// StartHTTPTokenBucket starts the token bucket if necessary
func StartHTTPTokenBucket(ctx context.Context) {
if fs.GetConfig(ctx).TPSLimit > 0 {
tpsBurst := fs.GetConfig(ctx).TPSLimitBurst
ci := fs.GetConfig(ctx)
if ci.TPSLimit > 0 {
tpsBurst := ci.TPSLimitBurst
if tpsBurst < 1 {
tpsBurst = 1
}
tpsBucket = rate.NewLimiter(rate.Limit(fs.GetConfig(ctx).TPSLimit), tpsBurst)
fs.Infof(nil, "Starting HTTP transaction limiter: max %g transactions/s with burst %d", fs.GetConfig(ctx).TPSLimit, tpsBurst)
tpsBucket = rate.NewLimiter(rate.Limit(ci.TPSLimit), tpsBurst)
fs.Infof(nil, "Starting HTTP transaction limiter: max %g transactions/s with burst %d", ci.TPSLimit, tpsBurst)
}
}
@ -94,7 +95,7 @@ func (c *timeoutConn) Write(b []byte) (n int, err error) {
// dial with context and timeouts
func dialContextTimeout(ctx context.Context, network, address string, ci *fs.ConfigInfo) (net.Conn, error) {
dialer := NewDialer(ci)
dialer := NewDialer(ctx)
c, err := dialer.DialContext(ctx, network, address)
if err != nil {
return c, err
@ -111,7 +112,8 @@ func ResetTransport() {
// NewTransportCustom returns an http.RoundTripper with the correct timeouts.
// The customize function is called if set to give the caller an opportunity to
// customize any defaults in the Transport.
func NewTransportCustom(ci *fs.ConfigInfo, customize func(*http.Transport)) http.RoundTripper {
func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) http.RoundTripper {
ci := fs.GetConfig(ctx)
// Start with a sensible set of defaults then override.
// This also means we get new stuff when it gets added to go
t := new(http.Transport)
@ -178,17 +180,18 @@ func NewTransportCustom(ci *fs.ConfigInfo, customize func(*http.Transport)) http
}
// NewTransport returns an http.RoundTripper with the correct timeouts
func NewTransport(ci *fs.ConfigInfo) http.RoundTripper {
func NewTransport(ctx context.Context) http.RoundTripper {
(*noTransport).Do(func() {
transport = NewTransportCustom(ci, nil)
transport = NewTransportCustom(ctx, nil)
})
return transport
}
// NewClient returns an http.Client with the correct timeouts
func NewClient(ci *fs.ConfigInfo) *http.Client {
func NewClient(ctx context.Context) *http.Client {
ci := fs.GetConfig(ctx)
client := &http.Client{
Transport: NewTransport(ci),
Transport: NewTransport(ctx),
}
if ci.Cookie {
client.Jar = cookieJar
@ -355,7 +358,8 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
// NewDialer creates a net.Dialer structure with Timeout, Keepalive
// and LocalAddr set from rclone flags.
func NewDialer(ci *fs.ConfigInfo) *net.Dialer {
func NewDialer(ctx context.Context) *net.Dialer {
ci := fs.GetConfig(ctx)
dialer := &net.Dialer{
Timeout: ci.ConnectTimeout,
KeepAlive: 30 * time.Second,

2
fs/operations/operations.go
View File

@ -1502,7 +1502,7 @@ type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser,
// copyURLFn copies the data from the url to the function supplied
func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameFromURL bool, fn copyURLFunc) (err error) {
client := fshttp.NewClient(fs.GetConfig(ctx))
client := fshttp.NewClient(ctx)
resp, err := client.Get(url)
if err != nil {
return err

4
lib/oauthutil/oauthutil.go
View File

@ -353,7 +353,7 @@ func NewClientWithBaseClient(ctx context.Context, name string, m configmap.Mappe
// NewClient gets a token from the config file and configures a Client
// with it. It returns the client and a TokenSource which Invalidate may need to be called on
func NewClient(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config) (*http.Client, *TokenSource, error) {
return NewClientWithBaseClient(ctx, name, m, oauthConfig, fshttp.NewClient(fs.GetConfig(ctx)))
return NewClientWithBaseClient(ctx, name, m, oauthConfig, fshttp.NewClient(ctx))
}
// AuthResult is returned from the web server after authorization
@ -526,7 +526,7 @@ version recommended):
}
// Exchange the code for a token
ctx = Context(ctx, fshttp.NewClient(fs.GetConfig(ctx)))
ctx = Context(ctx, fshttp.NewClient(ctx))
token, err := oauthConfig.Exchange(ctx, auth.Code)
if err != nil {
return errors.Wrap(err, "failed to get token")