From a9178cab8c58e3571370a1e947db38aadab7880f Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 2 Jul 2025 17:20:58 +0100 Subject: [PATCH] fs: allow setting of --http_proxy from command line This in turn allows `override.http_proxy` to be set in backend configs to set an http proxy for a single backend. --- docs/content/docs.md | 9 +++++++++ docs/content/faq.md | 10 ++++++++++ fs/config.go | 6 ++++++ fs/fshttp/http.go | 15 ++++++++++++++- 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 15ee1a22f..94ebec258 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -1249,6 +1249,15 @@ rclone sync --interactive ~/src s3:test/dst --header-upload "Content-Disposition See the GitHub issue [here](https://github.com/rclone/rclone/issues/59) for currently supported backends. +### --http-proxy string + +Use this option to set an HTTP proxy for all HTTP based services to +use. + +Rclone also supports the standard HTTP proxy environment variables +which it will pick up automatically. The is the way the HTTP proxy +will normally be set but this flag can be used to override it. + ### --human-readable ### Rclone commands output values for sizes (e.g. number of bytes) and diff --git a/docs/content/faq.md b/docs/content/faq.md index 10667c4c9..ea751336e 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -141,6 +141,16 @@ e.g. Note that the FTP backend does not support `ftp_proxy` yet. +You can use the command line argument `--http-proxy` to set the proxy, +and in turn use an override in the config file if you want it set for +a single backend, eg `override.http_proxy = http://...` in the config +file. + +The FTP and SFTP backends have their own `http_proxy` settings to +support an HTTP CONNECT proxy ( +[--ftp-http-proxy](https://rclone.org/ftp/#ftp-http-proxy) and +[--sftp-http-proxy](https://rclone.org/ftp/#sftp-http-proxy) ) + ### Rclone gives x509: failed to load system roots and no roots provided error ### This means that `rclone` can't find the SSL root certificates. Likely diff --git a/fs/config.go b/fs/config.go index aaae5defc..230a01483 100644 --- a/fs/config.go +++ b/fs/config.go @@ -555,6 +555,11 @@ var ConfigOptionsInfo = Options{{ Default: []string{}, Help: "Transform paths during the copy process.", Groups: "Copy", +}, { + Name: "http_proxy", + Default: "", + Help: "HTTP proxy URL.", + Groups: "Networking", }} // ConfigInfo is filesystem config options @@ -667,6 +672,7 @@ type ConfigInfo struct { MetadataMapper SpaceSepList `config:"metadata_mapper"` MaxConnections int `config:"max_connections"` NameTransform []string `config:"name_transform"` + HTTPProxy string `config:"http_proxy"` } func init() { diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go index 97e0728af..cd4488c28 100644 --- a/fs/fshttp/http.go +++ b/fs/fshttp/http.go @@ -6,10 +6,12 @@ import ( "context" "crypto/tls" "crypto/x509" + "fmt" "net" "net/http" "net/http/cookiejar" "net/http/httputil" + "net/url" "os" "sync" "time" @@ -55,7 +57,18 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht // This also means we get new stuff when it gets added to go t := new(http.Transport) structs.SetDefaults(t, http.DefaultTransport.(*http.Transport)) - t.Proxy = http.ProxyFromEnvironment + if ci.HTTPProxy != "" { + proxyURL, err := url.Parse(ci.HTTPProxy) + if err != nil { + t.Proxy = func(*http.Request) (*url.URL, error) { + return nil, fmt.Errorf("failed to set --http-proxy from %q: %w", ci.HTTPProxy, err) + } + } else { + t.Proxy = http.ProxyURL(proxyURL) + } + } else { + t.Proxy = http.ProxyFromEnvironment + } t.MaxIdleConnsPerHost = 2 * (ci.Checkers + ci.Transfers + 1) t.MaxIdleConns = 2 * t.MaxIdleConnsPerHost t.TLSHandshakeTimeout = ci.ConnectTimeout