fs: make sure backends with additional config have a different name #4996

Backends for which additional config is detected (in the config string
or on the command line or as environment variables) will gain a suffix
`{XXXXX}` where `XXXX` is a base64 encoded md5hash of the config
string.

This fixes backend caching with config string remotes.

This much requested feature now works properly:

    rclone copy -vv drive,shared_with_me:file.txt drive:

docs/content/docs.md

@@ -285,6 +285,31 @@ does not work on Windows.)
 
     rclone copy ':http,url="https://example.com":path/to/dir' /tmp/dir
 
+#### Connection strings, config and logging
+
+If you supply extra configuration to a backend by command line flag,
+environment variable or connection string then rclone will add a
+suffix based on the hash of the config to the name of the remote, eg
+
+    rclone -vv lsf --s3-chunk-size 20M s3:
+
+Has the log message
+
+    DEBUG : s3: detected overridden config - adding "{Srj1p}" suffix to name
+
+This is so rclone can tell the modified remote apart from the
+unmodified remote when caching the backends.
+
+This should only be noticeable in the logs.
+
+This means that on the fly backends such as
+
+    rclone -vv lsf :s3,env_auth:
+
+Will get their own names
+
+    DEBUG : :s3: detected overridden config - adding "{YTu53}" suffix to name
+
 ### Valid remote names
 
  - Remote names may only contain 0-9, A-Z ,a-z ,_ , - and space.

fs/fs.go

@@ -3,6 +3,8 @@ package fs
 
 import (
 	"context"
+	"crypto/md5"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -1379,6 +1381,34 @@ func NewFs(ctx context.Context, path string) (Fs, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Now discover which config items have been overridden,
+	// either by the config string, command line flags or
+	// environment variables
+	var overridden = configmap.Simple{}
+	for i := range fsInfo.Options {
+		opt := &fsInfo.Options[i]
+		value, isSet := config.GetOverride(opt.Name)
+		if isSet {
+			overridden.Set(opt.Name, value)
+		}
+	}
+	if len(overridden) > 0 {
+		extraConfig := overridden.String()
+		//Debugf(nil, "detected overriden config %q", extraConfig)
+		md5sumBinary := md5.Sum([]byte(extraConfig))
+		suffix := base64.RawStdEncoding.EncodeToString(md5sumBinary[:])
+		// 5 characters length is 5*6 = 30 bits of base64
+		const maxLength = 5
+		if len(suffix) > maxLength {
+			suffix = suffix[:maxLength]
+		}
+		suffix = "{" + suffix + "}"
+		Debugf(configName, "detected overridden config - adding %q suffix to name", suffix)
+		// Add the suffix to the config name
+		//
+		// These need to work as filesystem names as the VFS cache will use them
+		configName += suffix
+	}
 	return fsInfo.NewFs(ctx, configName, fsPath, config)
 }
 

fstest/fstests/fstests.go

@@ -318,6 +318,17 @@ func toUpperASCII(s string) string {
 	}, s)
 }
 
+// removeConfigID removes any {xyz} parts of the name put in for
+// config disambiguation
+func removeConfigID(s string) string {
+	bra := strings.IndexRune(s, '{')
+	ket := strings.IndexRune(s, '}')
+	if bra >= 0 && ket > bra {
+		s = s[:bra] + s[ket+1:]
+	}
+	return s
+}
+
 // Run runs the basic integration tests for a remote using the options passed in.
 //
 // They are structured in a hierarchical way so that dependencies for the tests can be created.
@@ -505,7 +516,7 @@ func Run(t *testing.T, opt *Opt) {
 	// TestFsName tests the Name method
 	t.Run("FsName", func(t *testing.T) {
 		skipIfNotOk(t)
-		got := f.Name()
+		got := removeConfigID(f.Name())
 		want := remoteName[:strings.LastIndex(remoteName, ":")+1]
 		if isLocalRemote {
 			want = "local:"
@@ -516,7 +527,7 @@ func Run(t *testing.T, opt *Opt) {
 	// TestFsRoot tests the Root method
 	t.Run("FsRoot", func(t *testing.T) {
 		skipIfNotOk(t)
-		name := f.Name() + ":"
+		name := removeConfigID(f.Name()) + ":"
 		root := f.Root()
 		if isLocalRemote {
 			// only check last path element on local