diff --git a/docs/content/docs.md b/docs/content/docs.md index 9996ff3fc..b247c2dc7 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -437,6 +437,103 @@ Do not use single character names on Windows as it creates ambiguity with Window drives' names, e.g.: remote called `C` is indistinguishable from `C` drive. Rclone will always assume that single letter name refers to a drive. +## Adding global configuration to a remote + +It is possible to add global configuration to the remote configuration which +will be applied just before the remote is created. + +This can be done in two ways. The first is to use `override.var = value` in the +config file or the connection string for a temporary change, and the second is +to use `global.var = value` in the config file or connection string for a +permanent change. + +This is explained fully below. + +### override.var + +This is used to override a global variable **just** for the duration of the +remote creation. It won't affect other remotes even if they are created at the +same time. + +This is very useful for overriding networking config needed for just for that +remote. For example, say you have a remote which needs `--no-check-certificate` +as it is running on test infrastructure without a proper certificate. You could +supply the `--no-check-certificate` flag to rclone, but this will affect **all** +the remotes. To make it just affect this remote you use an override. You could +put this in the config file: + +```ini +[remote] +type = XXX +... +override.no_check_certificate = true +``` + +or use it in the connection string `remote,override.no_check_certificate=true:` +(or just `remote,override.no_check_certificate:`). + +Note how the global flag name loses its initial `--` and gets `-` replaced with +`_` and gets an `override.` prefix. + +Not all global variables make sense to be overridden like this as the config is +only applied during the remote creation. Here is a non exhaustive list of ones +which might be useful: + +- `bind_addr` +- `ca_cert` +- `client_cert` +- `client_key` +- `connect_timeout` +- `disable_http2` +- `disable_http_keep_alives` +- `dump` +- `expect_continue_timeout` +- `headers` +- `http_proxy` +- `low_level_retries` +- `max_connections` +- `no_check_certificate` +- `no_gzip` +- `timeout` +- `traffic_class` +- `use_cookies` +- `use_server_modtime` +- `user_agent` + +An `override.var` will override all other config methods, but **just** for the +duration of the creation of the remote. + +### global.var + +This is used to set a global variable **for everything**. The global variable is +set just before the remote is created. + +This is useful for parameters (eg sync parameters) which can't be set as an +`override`. For example, say you have a remote where you would always like to +use the `--checksum` flag. You could supply the `--checksum` flag to rclone on +every command line, but instead you could put this in the config file: + +```ini +[remote] +type = XXX +... +global.checksum = true +``` + +or use it in the connection string `remote,global.checksum=true:` (or just +`remote,global.checksum:`). This is equivalent to using the `--checksum` flag. + +Note how the global flag name loses its initial `--` and gets `-` replaced with +`_` and gets a `global.` prefix. + +Any global variable can be set like this and it is exactly equivalent to using +the equivalent flag on the command line. This means it will affect all uses of +rclone. + +If two remotes set the same global variable then the first one instantiated will +be overridden by the second one. A `global.var` will override all other config +methods when the remote is created. + ## Quoting and the shell When you are typing commands to your computer you are using something diff --git a/fs/fspath/path.go b/fs/fspath/path.go index 09e71d08e..4cdd382c0 100644 --- a/fs/fspath/path.go +++ b/fs/fspath/path.go @@ -20,7 +20,7 @@ const ( var ( errInvalidCharacters = errors.New("config name contains invalid characters - may only contain numbers, letters, `_`, `-`, `.`, `+`, `@` and space, while not start with `-` or space, and not end with space") errCantBeEmpty = errors.New("can't use empty string as a path") - errBadConfigParam = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z` and `_`") + errBadConfigParam = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z`, `_` and `.`") errEmptyConfigParam = errors.New("config parameters can't be empty") errConfigNameEmpty = errors.New("config name can't be empty") errConfigName = errors.New("config name needs a trailing `:`") @@ -79,7 +79,8 @@ func isConfigParam(c rune) bool { return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || - c == '_') + c == '_' || + c == '.') } // Parsed is returned from Parse with the results of the connection string decomposition diff --git a/fs/newfs.go b/fs/newfs.go index f4f7dd346..290e4ac40 100644 --- a/fs/newfs.go +++ b/fs/newfs.go @@ -7,12 +7,15 @@ import ( "crypto/md5" "encoding/base64" "fmt" + "maps" "os" "path/filepath" + "slices" "strings" "sync" "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/fspath" ) @@ -65,6 +68,10 @@ func NewFs(ctx context.Context, path string) (Fs, error) { overriddenConfig[suffix] = extraConfig overriddenConfigMu.Unlock() } + ctx, err = addConfigToContext(ctx, configName, config) + if err != nil { + return nil, err + } f, err := fsInfo.NewFs(ctx, configName, fsPath, config) if f != nil && (err == nil || err == ErrorIsFile) { addReverse(f, fsInfo) @@ -72,6 +79,54 @@ func NewFs(ctx context.Context, path string) (Fs, error) { return f, err } +// Add "global" config or "override" to ctx and the global config if required. +// +// This looks through keys prefixed with "global." or "override." in +// config and sets ctx and optionally the global context if "global.". +func addConfigToContext(ctx context.Context, configName string, config configmap.Getter) (newCtx context.Context, err error) { + overrideConfig := make(configmap.Simple) + globalConfig := make(configmap.Simple) + for i := range ConfigOptionsInfo { + opt := &ConfigOptionsInfo[i] + globalName := "global." + opt.Name + value, isSet := config.Get(globalName) + if isSet { + // Set both override and global if global + overrideConfig[opt.Name] = value + globalConfig[opt.Name] = value + } + overrideName := "override." + opt.Name + value, isSet = config.Get(overrideName) + if isSet { + overrideConfig[opt.Name] = value + } + } + if len(overrideConfig) == 0 && len(globalConfig) == 0 { + return ctx, nil + } + newCtx, ci := AddConfig(ctx) + overrideKeys := slices.Collect(maps.Keys(overrideConfig)) + slices.Sort(overrideKeys) + globalKeys := slices.Collect(maps.Keys(globalConfig)) + slices.Sort(globalKeys) + // Set the config in the newCtx + err = configstruct.Set(overrideConfig, ci) + if err != nil { + return ctx, fmt.Errorf("failed to set override config variables %q: %w", overrideKeys, err) + } + Debugf(configName, "Set overridden config %q for backend startup", overrideKeys) + // Set the global context only + if len(globalConfig) != 0 { + globalCI := GetConfig(context.Background()) + err = configstruct.Set(globalConfig, globalCI) + if err != nil { + return ctx, fmt.Errorf("failed to set global config variables %q: %w", globalKeys, err) + } + Debugf(configName, "Set global config %q at backend startup", overrideKeys) + } + return newCtx, nil +} + // ConfigFs makes the config for calling NewFs with. // // It parses the path which is of the form remote:path diff --git a/fs/newfs_internal_test.go b/fs/newfs_internal_test.go new file mode 100644 index 000000000..6428a5bc8 --- /dev/null +++ b/fs/newfs_internal_test.go @@ -0,0 +1,55 @@ +package fs + +import ( + "context" + "testing" + + "github.com/rclone/rclone/fs/config/configmap" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// When no override/global keys exist, ctx must be returned unchanged. +func TestAddConfigToContext_NoChanges(t *testing.T) { + ctx := context.Background() + newCtx, err := addConfigToContext(ctx, "unit-test", configmap.Simple{}) + require.NoError(t, err) + assert.Equal(t, newCtx, ctx) +} + +// A single override.key must create a new ctx, but leave the +// background ctx untouched. +func TestAddConfigToContext_OverrideOnly(t *testing.T) { + override := configmap.Simple{ + "override.user_agent": "potato", + } + ctx := context.Background() + globalCI := GetConfig(ctx) + original := globalCI.UserAgent + newCtx, err := addConfigToContext(ctx, "unit-test", override) + require.NoError(t, err) + assert.NotEqual(t, newCtx, ctx) + assert.Equal(t, original, globalCI.UserAgent) + ci := GetConfig(newCtx) + assert.Equal(t, "potato", ci.UserAgent) +} + +// A single global.key must create a new ctx and update the +// background/global config. +func TestAddConfigToContext_GlobalOnly(t *testing.T) { + global := configmap.Simple{ + "global.user_agent": "potato2", + } + ctx := context.Background() + globalCI := GetConfig(ctx) + original := globalCI.UserAgent + defer func() { + globalCI.UserAgent = original + }() + newCtx, err := addConfigToContext(ctx, "unit-test", global) + require.NoError(t, err) + assert.NotEqual(t, newCtx, ctx) + assert.Equal(t, "potato2", globalCI.UserAgent) + ci := GetConfig(newCtx) + assert.Equal(t, "potato2", ci.UserAgent) +} diff --git a/fs/newfs_test.go b/fs/newfs_test.go index 4d5ef8520..19667401e 100644 --- a/fs/newfs_test.go +++ b/fs/newfs_test.go @@ -42,4 +42,21 @@ func TestNewFs(t *testing.T) { assert.Equal(t, ":mockfs{S_NHG}:/tmp", fs.ConfigString(f3)) assert.Equal(t, ":mockfs,potato='true':/tmp", fs.ConfigStringFull(f3)) + + // Check that the overrides work + globalCI := fs.GetConfig(ctx) + original := globalCI.UserAgent + defer func() { + globalCI.UserAgent = original + }() + + f4, err := fs.NewFs(ctx, ":mockfs,global.user_agent='julian':/tmp") + require.NoError(t, err) + assert.Equal(t, ":mockfs", f4.Name()) + assert.Equal(t, "/tmp", f4.Root()) + + assert.Equal(t, ":mockfs:/tmp", fs.ConfigString(f4)) + assert.Equal(t, ":mockfs:/tmp", fs.ConfigStringFull(f4)) + + assert.Equal(t, "julian", globalCI.UserAgent) }