diff --git a/bin/test_proxy.py b/bin/test_proxy.py
new file mode 100755
index 000000000..6f00ec757
--- /dev/null
+++ b/bin/test_proxy.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+"""
+A demo proxy for rclone serve sftp/webdav/ftp etc
+
+This takes the incoming user/pass and converts it into an sftp backend
+running on localhost.
+"""
+
+import sys
+import json
+
+def main():
+    i = json.load(sys.stdin)
+    o = {
+        "type": "sftp",              # type of backend
+        "_root": "",                 # root of the fs
+        "_obscure": "pass",          # comma sep list of fields to obscure
+        "user": i["user"],
+        "pass": i["pass"],
+        "host": "127.0.0.1",
+    }
+    json.dump(o, sys.stdout, indent="\t")
+
+if __name__ == "__main__":
+    main()
diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.go
new file mode 100644
index 000000000..83a37bca2
--- /dev/null
+++ b/cmd/serve/proxy/proxy.go
@@ -0,0 +1,270 @@
+// Package proxy implements a programmable proxy for rclone serve
+package proxy
+
+import (
+	"bytes"
+	"encoding/json"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/obscure"
+	libcache "github.com/rclone/rclone/lib/cache"
+	"github.com/rclone/rclone/vfs"
+	"github.com/rclone/rclone/vfs/vfsflags"
+	"golang.org/x/crypto/bcrypt"
+)
+
+// Help contains text describing how to use the proxy
+var Help = strings.Replace(`
+### Auth Proxy
+
+If you supply the parameter |--auth-proxy /path/to/program| then
+rclone will use that program to generate backends on the fly which
+then are used to authenticate incoming requests.  This uses a simple
+JSON based protocl with input on STDIN and output on STDOUT.
+
+There is an example program
+[bin/test_proxy.py](https://github.com/rclone/rclone/blob/master/test_proxy.py)
+in the rclone source code.
+
+The program's job is to take a |user| and |pass| on the input and turn
+those into the config for a backend on STDOUT in JSON format.  This
+config will have any default parameters for the backend added, but it
+won't use configuration from environment variables or command line
+options - it is the job of the proxy program to make a complete
+config.
+
+This config generated must have this extra parameter
+- |_root| - root to use for the backend
+
+And it may have this parameter
+- |_obscure| - comma separated strings for parameters to obscure
+
+For example the program might take this on STDIN
+
+|||
+{
+	"user": "me",
+	"pass": "mypassword"
+}
+|||
+
+And return this on STDOUT
+
+|||
+{
+	"type": "sftp",
+	"_root": "",
+	"_obscure": "pass",
+	"user": "me",
+	"pass": "mypassword",
+	"host": "sftp.example.com"
+}
+|||
+
+This would mean that an SFTP backend would be created on the fly for
+the |user| and |pass| returned in the output to the host given.  Note
+that since |_obscure| is set to |pass|, rclone will obscure the |pass|
+parameter before creating the backend (which is required for sftp
+backends).
+
+The progam can manipulate the supplied |user| in any way, for example
+to make proxy to many different sftp backends, you could make the
+|user| be |user@example.com| and then set the |host| to |example.com|
+in the output and the user to |user|. For security you'd probably want
+to restrict the |host| to a limited list.
+
+Note that an internal cache is keyed on |user| so only use that for
+configuration, don't use |pass|.  This also means that if a user's
+password is changed the cache will need to expire (which takes 5 mins)
+before it takes effect.
+
+This can be used to build general purpose proxies to any kind of
+backend that rclone supports.  
+`, "|", "`", -1)
+
+// Options is options for creating the proxy
+type Options struct {
+	AuthProxy string
+}
+
+// DefaultOpt is the default values uses for Opt
+var DefaultOpt = Options{
+	AuthProxy: "",
+}
+
+// Proxy represents a proxy to turn auth requests into a VFS
+type Proxy struct {
+	cmdLine  []string // broken down command line
+	vfsCache *libcache.Cache
+	Opt      Options
+}
+
+// cacheEntry is what is stored in the vfsCache
+type cacheEntry struct {
+	vfs    *vfs.VFS // stored VFS
+	pwHash []byte   // bcrypt hash of the password
+}
+
+// New creates a new proxy with the Options passed in
+func New(opt *Options) *Proxy {
+	return &Proxy{
+		Opt:      *opt,
+		cmdLine:  strings.Fields(opt.AuthProxy),
+		vfsCache: libcache.New(),
+	}
+}
+
+// run the proxy command returning a config map
+func (p *Proxy) run(in map[string]string) (config configmap.Simple, err error) {
+	cmd := exec.Command(p.cmdLine[0], p.cmdLine[1:]...)
+	inBytes, err := json.MarshalIndent(in, "", "\t")
+	if err != nil {
+		return nil, errors.Wrap(err, "Proxy.Call failed to marshal input: %v")
+	}
+	var stdout, stderr bytes.Buffer
+	cmd.Stdin = bytes.NewBuffer(inBytes)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	start := time.Now()
+	err = cmd.Run()
+	fs.Debugf(nil, "Calling proxy %v", p.cmdLine)
+	duration := time.Since(start)
+	if err != nil {
+		return nil, errors.Wrapf(err, "proxy: failed on %v: %q", p.cmdLine, strings.TrimSpace(string(stderr.Bytes())))
+	}
+	err = json.Unmarshal(stdout.Bytes(), &config)
+	if err != nil {
+		return nil, errors.Wrapf(err, "proxy: failed to read output: %q", string(stdout.Bytes()))
+	}
+	fs.Debugf(nil, "Proxy returned in %v", duration)
+
+	// Obscure any values in the config map that need it
+	obscureFields, ok := config.Get("_obscure")
+	if ok {
+		for _, key := range strings.Split(obscureFields, ",") {
+			value, ok := config.Get(key)
+			if ok {
+				obscuredValue, err := obscure.Obscure(value)
+				if err != nil {
+					return nil, errors.Wrap(err, "proxy")
+				}
+				config.Set(key, obscuredValue)
+			}
+		}
+	}
+	return config, nil
+}
+
+// call runs the auth proxy and returns a cacheEntry and an error
+func (p *Proxy) call(user, pass string, passwordBytes []byte) (value interface{}, err error) {
+	// Contact the proxy
+	config, err := p.run(map[string]string{
+		"user": user,
+		"pass": pass,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Look for required fields in the answer
+	fsName, ok := config.Get("type")
+	if !ok {
+		return nil, errors.New("proxy: type not set in result")
+	}
+	root, ok := config.Get("_root")
+	if !ok {
+		return nil, errors.New("proxy: _root not set in result")
+	}
+
+	// Find the backend
+	fsInfo, err := fs.Find(fsName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "proxy: couldn't find backend for %q", fsName)
+	}
+
+	// base name of config on user name.  This may appear in logs
+	name := "proxy-" + user
+	fsString := name + ":" + root
+
+	// Look for fs in the VFS cache
+	value, err = p.vfsCache.Get(user, func(key string) (value interface{}, ok bool, err error) {
+		// Create the Fs from the cache
+		f, err := cache.GetFn(fsString, func(fsString string) (fs.Fs, error) {
+			// Update the config with the default values
+			for i := range fsInfo.Options {
+				o := &fsInfo.Options[i]
+				if _, found := config.Get(o.Name); !found && o.Default != nil && o.String() != "" {
+					config.Set(o.Name, o.String())
+				}
+			}
+			return fsInfo.NewFs(name, root, config)
+		})
+		if err != nil {
+			return nil, false, err
+		}
+		pwHash, err := bcrypt.GenerateFromPassword(passwordBytes, bcrypt.DefaultCost)
+		if err != nil {
+			return nil, false, err
+		}
+		entry := cacheEntry{
+			vfs:    vfs.New(f, &vfsflags.Opt),
+			pwHash: pwHash,
+		}
+		return entry, true, nil
+	})
+	if err != nil {
+		return nil, errors.Wrapf(err, "proxy: failed to create backend")
+	}
+	return value, nil
+}
+
+// Call runs the auth proxy with the given input, returning a *vfs.VFS
+// and the key used in the VFS cache.
+func (p *Proxy) Call(user, pass string) (VFS *vfs.VFS, vfsKey string, err error) {
+	var passwordBytes = []byte(pass)
+
+	// Look in the cache first
+	value, ok := p.vfsCache.GetMaybe(user)
+
+	// If not found then call the proxy for a fresh answer
+	if !ok {
+		value, err = p.call(user, pass, passwordBytes)
+		if err != nil {
+			return nil, "", err
+		}
+	}
+
+	// check we got what we were expecting
+	entry, ok := value.(cacheEntry)
+	if !ok {
+		return nil, "", errors.Errorf("proxy: value is not cache entry: %#v", value)
+	}
+
+	// Check the password is correct in the cached entry.  This
+	// prevents an attack where subsequent requests for the same
+	// user don't have their auth checked. It does mean that if
+	// the password is changed, the user will have to wait for
+	// cache expiry (5m) before trying again.
+	err = bcrypt.CompareHashAndPassword(entry.pwHash, passwordBytes)
+	if err != nil {
+		return nil, "", errors.Wrap(err, "proxy: incorrect password")
+	}
+
+	return entry.vfs, user, nil
+}
+
+// Get VFS from the cache using key - returns nil if not found
+func (p *Proxy) Get(key string) *vfs.VFS {
+	value, ok := p.vfsCache.GetMaybe(key)
+	if !ok {
+		return nil
+	}
+	entry := value.(cacheEntry)
+	return entry.vfs
+}
diff --git a/cmd/serve/proxy/proxy_code.go b/cmd/serve/proxy/proxy_code.go
new file mode 100644
index 000000000..1e2979602
--- /dev/null
+++ b/cmd/serve/proxy/proxy_code.go
@@ -0,0 +1,41 @@
+// +build ignore
+
+// A simple auth proxy for testing purposes
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+)
+
+func main() {
+	// Read the input
+	var in map[string]string
+	err := json.NewDecoder(os.Stdin).Decode(&in)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Write the output
+	var out = map[string]string{}
+	for k, v := range in {
+		switch k {
+		case "user":
+			v += "-test"
+		case "error":
+			log.Fatal(v)
+		}
+		out[k] = v
+	}
+	if out["type"] == "" {
+		out["type"] = "local"
+	}
+	if out["_root"] == "" {
+		out["_root"] = ""
+	}
+	json.NewEncoder(os.Stdout).Encode(&out)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
diff --git a/cmd/serve/proxy/proxy_test.go b/cmd/serve/proxy/proxy_test.go
new file mode 100644
index 000000000..135ed6aab
--- /dev/null
+++ b/cmd/serve/proxy/proxy_test.go
@@ -0,0 +1,145 @@
+package proxy
+
+import (
+	"strings"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/crypto/bcrypt"
+)
+
+func TestRun(t *testing.T) {
+	opt := DefaultOpt
+	cmd := "go run proxy_code.go"
+	opt.AuthProxy = cmd
+	p := New(&opt)
+
+	t.Run("Normal", func(t *testing.T) {
+		config, err := p.run(map[string]string{
+			"type": "ftp",
+			"user": "me",
+			"pass": "pass",
+			"host": "127.0.0.1",
+		})
+		require.NoError(t, err)
+		assert.Equal(t, configmap.Simple{
+			"type":  "ftp",
+			"user":  "me-test",
+			"pass":  "pass",
+			"host":  "127.0.0.1",
+			"_root": "",
+		}, config)
+	})
+
+	t.Run("Error", func(t *testing.T) {
+		config, err := p.run(map[string]string{
+			"error": "potato",
+		})
+		assert.Nil(t, config)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "potato")
+	})
+
+	t.Run("Obscure", func(t *testing.T) {
+		config, err := p.run(map[string]string{
+			"type":     "ftp",
+			"user":     "me",
+			"pass":     "pass",
+			"host":     "127.0.0.1",
+			"_obscure": "pass,user",
+		})
+		require.NoError(t, err)
+		config["user"] = obscure.MustReveal(config["user"])
+		config["pass"] = obscure.MustReveal(config["pass"])
+		assert.Equal(t, configmap.Simple{
+			"type":     "ftp",
+			"user":     "me-test",
+			"pass":     "pass",
+			"host":     "127.0.0.1",
+			"_obscure": "pass,user",
+			"_root":    "",
+		}, config)
+	})
+
+	const testUser = "testUser"
+	const testPass = "testPass"
+
+	t.Run("call", func(t *testing.T) {
+		// check cache empty
+		assert.Equal(t, 0, p.vfsCache.Entries())
+		defer p.vfsCache.Clear()
+
+		passwordBytes := []byte(testPass)
+		value, err := p.call(testUser, testPass, passwordBytes)
+		require.NoError(t, err)
+		entry, ok := value.(cacheEntry)
+		require.True(t, ok)
+
+		// check hash is correct in entry
+		err = bcrypt.CompareHashAndPassword(entry.pwHash, passwordBytes)
+		require.NoError(t, err)
+		require.NotNil(t, entry.vfs)
+		f := entry.vfs.Fs()
+		require.NotNil(t, f)
+		assert.Equal(t, "proxy-"+testUser, f.Name())
+		assert.True(t, strings.HasPrefix(f.String(), "Local file system"))
+
+		// check it is in the cache
+		assert.Equal(t, 1, p.vfsCache.Entries())
+		cacheValue, ok := p.vfsCache.GetMaybe(testUser)
+		assert.True(t, ok)
+		assert.Equal(t, value, cacheValue)
+	})
+
+	t.Run("Call", func(t *testing.T) {
+		// check cache empty
+		assert.Equal(t, 0, p.vfsCache.Entries())
+		defer p.vfsCache.Clear()
+
+		vfs, vfsKey, err := p.Call(testUser, testPass)
+		require.NoError(t, err)
+		require.NotNil(t, vfs)
+		assert.Equal(t, "proxy-"+testUser, vfs.Fs().Name())
+		assert.Equal(t, testUser, vfsKey)
+
+		// check it is in the cache
+		assert.Equal(t, 1, p.vfsCache.Entries())
+		cacheValue, ok := p.vfsCache.GetMaybe(testUser)
+		assert.True(t, ok)
+		cacheEntry, ok := cacheValue.(cacheEntry)
+		assert.True(t, ok)
+		assert.Equal(t, vfs, cacheEntry.vfs)
+
+		// Test Get works while we have something in the cache
+		t.Run("Get", func(t *testing.T) {
+			assert.Equal(t, vfs, p.Get(testUser))
+			assert.Nil(t, p.Get("unknown"))
+		})
+
+		// now try again from the cache
+		vfs, vfsKey, err = p.Call(testUser, testPass)
+		require.NoError(t, err)
+		require.NotNil(t, vfs)
+		assert.Equal(t, "proxy-"+testUser, vfs.Fs().Name())
+		assert.Equal(t, testUser, vfsKey)
+
+		// check cache is at the same level
+		assert.Equal(t, 1, p.vfsCache.Entries())
+
+		// now try again from the cache but with wrong password
+		vfs, vfsKey, err = p.Call(testUser, testPass+"wrong")
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "incorrect password")
+		require.Nil(t, vfs)
+		require.Equal(t, "", vfsKey)
+
+		// check cache is at the same level
+		assert.Equal(t, 1, p.vfsCache.Entries())
+
+	})
+
+}
diff --git a/cmd/serve/proxy/proxyflags/proxyflags.go b/cmd/serve/proxy/proxyflags/proxyflags.go
new file mode 100644
index 000000000..aff2af328
--- /dev/null
+++ b/cmd/serve/proxy/proxyflags/proxyflags.go
@@ -0,0 +1,18 @@
+// Package proxyflags implements command line flags to set up a proxy
+package proxyflags
+
+import (
+	"github.com/rclone/rclone/cmd/serve/proxy"
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/spf13/pflag"
+)
+
+// Options set by command line flags
+var (
+	Opt = proxy.DefaultOpt
+)
+
+// AddFlags adds the non filing system specific flags to the command
+func AddFlags(flagSet *pflag.FlagSet) {
+	flags.StringVarP(flagSet, &Opt.AuthProxy, "auth-proxy", "", Opt.AuthProxy, "A program to use to create the backend from the auth.")
+}