build: implement a framework for starting test servers during tests

Test servers are implemented by docker containers and run real servers
for rclone to test against.

backend/ftp/ftp_test.go

@@ -5,13 +5,44 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/ftp"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFTP:",
+		RemoteName: "TestFTPProftpd:",
 		NilObject:  (*ftp.Object)(nil),
 	})
 }
+
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFTPRclone:",
+		NilObject:  (*ftp.Object)(nil),
+	})
+}
+
+func TestIntegration3(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFTPPureftpd:",
+		NilObject:  (*ftp.Object)(nil),
+	})
+}
+
+// func TestIntegration4(t *testing.T) {
+// 	if *fstest.RemoteName != "" {
+// 		t.Skip("skipping as -remote is set")
+// 	}
+// 	fstests.Run(t, &fstests.Opt{
+// 		RemoteName: "TestFTPVsftpd:",
+// 		NilObject:  (*ftp.Object)(nil),
+// 	})
+// }

backend/sftp/sftp_test.go

@@ -8,13 +8,24 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/sftp"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestSftp:",
+		RemoteName: "TestSFTPOpenssh:",
+		NilObject:  (*sftp.Object)(nil),
+	})
+}
+
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSFTPRclone:",
 		NilObject:  (*sftp.Object)(nil),
 	})
 }

backend/swift/swift_test.go

@@ -20,7 +20,7 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestSwift:",
+		RemoteName: "TestSwiftAIO:",
 		NilObject:  (*Object)(nil),
 	})
 }

backend/webdav/webdav_test.go

@@ -5,13 +5,36 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/webdav"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestWebdav:",
+		RemoteName: "TestWebdavNexcloud:",
+		NilObject:  (*webdav.Object)(nil),
+	})
+}
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestWebdavOwncloud:",
+		NilObject:  (*webdav.Object)(nil),
+	})
+}
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration3(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestWebdavRclone:",
 		NilObject:  (*webdav.Object)(nil),
 	})
 }

fstest/fstests/fstests.go

@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/testserver"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
@@ -306,6 +307,10 @@ func Run(t *testing.T, opt *Opt) {
 		ctx           = context.Background()
 	)
 
+	if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" {
+		t.Skip("quicktest only")
+	}
+
 	// Skip the test if the remote isn't configured
 	skipIfNotOk := func(t *testing.T) {
 		if remote == nil {
@@ -352,7 +357,11 @@ func Run(t *testing.T, opt *Opt) {
 	if *fstest.RemoteName != "" {
 		remoteName = *fstest.RemoteName
 	}
+	oldFstestRemoteName := fstest.RemoteName
 	fstest.RemoteName = &remoteName
+	defer func() {
+		fstest.RemoteName = oldFstestRemoteName
+	}()
 	t.Logf("Using remote %q", remoteName)
 	var err error
 	if remoteName == "" {
@@ -361,6 +370,11 @@ func Run(t *testing.T, opt *Opt) {
 		isLocalRemote = true
 	}
 
+	// Start any test servers if required
+	finish, err := testserver.Start(remoteName)
+	require.NoError(t, err)
+	defer finish()
+
 	// Make the Fs we are testing with, initialising the local variables
 	// subRemoteName - name of the remote after the TestRemote:
 	// subRemoteLeaf - a subdirectory to use under that

fstest/test_all/config.yaml

@@ -144,13 +144,19 @@ backends:
    remote:   "TestS3Alibaba:"
    fastlist: true
  - backend:  "sftp"
-   remote:   "TestSftp:"
+   remote:   "TestSFTPOpenssh:"
+   fastlist: false
+ - backend:  "sftp"
+   remote:   "TestSFTPRclone:"
    fastlist: false
  - backend:  "sugarsync"
    remote:   "TestSugarSync:Test"
    fastlist: false
    ignore:
      - TestIntegration/FsMkdir/FsPutFiles/PublicLink
+ - backend:  "swift"
+   remote:   "TestSwiftAIO:"
+   fastlist: true
  - backend:  "swift"
    remote:   "TestSwift:"
    fastlist: true
@@ -163,10 +169,27 @@ backends:
    remote:   "TestYandex:"
    fastlist: false
  - backend:  "ftp"
-   remote:   "TestFTP:"
+   remote:   "TestFTPProftpd:"
    ignore:
      - TestIntegration/FsMkdir/FsEncoding/punctuation
    fastlist: false
+ # - backend:  "ftp"
+ #   remote:   "TestFTPVsftpd:"
+ #   ignore:
+ #     - TestIntegration/FsMkdir/FsEncoding/punctuation
+ #   fastlist: false
+ - backend:  "ftp"
+   remote:   "TestFTPPureftpd:"
+   ignore:
+     - TestIntegration/FsMkdir/FsEncoding/punctuation
+   fastlist: false
+ - backend:  "ftp"
+   remote:   "TestFTPRclone:"
+   ignore:
+     - "TestMultithreadCopy/{size:131071_streams:2}"
+     - "TestMultithreadCopy/{size:131072_streams:2}"
+     - "TestMultithreadCopy/{size:131073_streams:2}"
+   fastlist: false
  - backend:  "box"
    remote:   "TestBox:"
    fastlist: false
@@ -184,11 +207,26 @@ backends:
    remote:   "TestPcloud:"
    fastlist: false
  - backend:  "webdav"
-   remote:   "TestWebdav:"
+   remote:   "TestWebdavNextcloud:"
    ignore:
      - TestIntegration/FsMkdir/FsEncoding/punctuation
      - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8
    fastlist: false
+ - backend:  "webdav"
+   remote:   "TestWebdavOwncloud:"
+   ignore:
+     - TestIntegration/FsMkdir/FsEncoding/punctuation
+     - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8
+     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
+     - TestCopyFileCopyDest
+     - TestServerSideCopy
+     - TestSyncCopyDest
+   fastlist: false
+ - backend:  "webdav"
+   remote:   "TestWebdavRclone:"
+   ignore:
+     - TestFileReadAtZeroLength
+   fastlist: false
  - backend:  "cache"
    remote:   "TestCache:"
    fastlist: false

fstest/test_all/run.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/testserver"
 )
 
 // Control concurrency per backend if required
@@ -213,6 +214,16 @@ func (r *Run) trial() {
 		return
 	}
 
+	// Start the test server if required
+	finish, err := testserver.Start(r.Remote)
+	if err != nil {
+		log.Printf("%s: Failed to start test server: %v", r.Remote, err)
+		_, _ = fmt.Fprintf(out, "%s: Failed to start test server: %v\n", r.Remote, err)
+		r.err = err
+		return
+	}
+	defer finish()
+
 	// Internal buffer
 	var b bytes.Buffer
 	multiOut := io.MultiWriter(out, &b)

fstest/testserver/images/test-sftp-openssh/Dockerfile

@@ -0,0 +1,11 @@
+# A very minimal sftp server for integration testing rclone
+FROM alpine:latest
+
+# User rclone, password password
+RUN \
+    apk add openssh && \
+    ssh-keygen -A && \
+    adduser -D rclone && \
+    echo "rclone:password" | chpasswd 
+
+ENTRYPOINT [ "/usr/sbin/sshd", "-D" ]

fstest/testserver/images/test-sftp-openssh/README.md

@@ -0,0 +1,17 @@
+# Test SFTP Openssh
+
+This is a docker image for rclone's integration tests which runs an
+openssh server in a docker image.
+
+## Build
+
+```
+docker build --rm -t rclone/test-sftp-openssh .
+docker push rclone/test-sftp-openssh
+```
+
+# Test
+
+```
+rclone lsf -R --sftp-host 172.17.0.2 --sftp-user rclone --sftp-pass $(rclone obscure password) :sftp:
+```

fstest/testserver/init.d/README.md

@@ -0,0 +1,31 @@
+This directory contains scripts to start and stop servers for testing.
+
+The commands are named after the remotes in use.  They should be
+executable files with the following parameters:
+
+    start  - starts the server
+    stop   - stops the server
+    status - returns non-zero exit code if the server is not running
+
+These will be called automatically by test_all if that remote is
+required.
+
+When start is run it should output config parameters for that remote.
+If a `_connect` parameter is output then that will be used for a
+connection test.  For example if `_connect=127.0.0.1:80` then a TCP
+connection will be made to `127.0.0.1:80` and only when that succeeds
+will the test continue.
+
+`run.bash` contains boilerplate to be included in a bash script for
+interpreting the command line parameters.
+
+`docker.bash` contains library functions to help with docker
+implementations.
+
+## TODO
+
+- sftpd - https://github.com/panubo/docker-sshd ?
+- openstack swift - https://github.com/bouncestorage/docker-swift
+- ceph - https://github.com/ceph/cn
+- other ftp servers
+

fstest/testserver/init.d/TestFTPProftpd

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+NAME=proftpd
+USER=rclone
+PASS=RaidedBannedPokes5
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           -e "FTP_USERNAME=rclone" \
+           -e "FTP_PASSWORD=$PASS" \
+           hauptmedia/proftpd
+    
+    echo type=ftp
+    echo host=$(docker_ip)
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    echo _connect=$(docker_ip):21
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestFTPPureftpd

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+NAME=pureftpd
+USER=rclone
+PASS=AcridSpiesBooks2
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           -e "FTP_USER_NAME=rclone" \
+           -e "FTP_USER_PASS=$PASS" \
+           -e "FTP_USER_HOME=/data" \
+           -e "FTP_MAX_CLIENTS=50" \
+           -e "FTP_MAX_CONNECTIONS=50" \
+           -e "FTP_PASSIVE_PORTS=30000:40000" \
+           stilliard/pure-ftpd
+
+    echo type=ftp
+    echo host=$(docker_ip)
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    echo _connect=$(docker_ip):21
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestFTPRclone

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+NAME=rclone-serve-ftp
+USER=rclone
+PASS=FuddleIdlingJell5
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           rclone/rclone \
+           serve ftp --user $USER --pass $PASS --addr :21 /data
+    
+    echo type=ftp
+    echo host=$(docker_ip)
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    echo _connect=$(docker_ip):21
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestFTPVsftpd

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+NAME=vsftpd
+USER=rclone
+PASS=TiffedRestedSian4
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           -e "FTP_USER=rclone" \
+           -e "FTP_PASS=$PASS" \
+           fauria/vsftpd
+
+    echo type=ftp
+    echo host=$(docker_ip)
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    echo _connect=$(docker_ip):21
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestS3Minio

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+NAME=minio
+USER=rclone
+PASS=AxedBodedGinger7
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           -e "MINIO_ACCESS_KEY=$USER" \
+           -e "MINIO_SECRET_KEY=$PASS" \
+           minio/minio server /data
+    
+    echo type=s3
+    echo access_key_id=$USER
+    echo secret_access_key=$PASS
+    echo endpoint=http://$(docker_ip):9000/
+    echo _connect=$(docker_ip):9000
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestSFTPOpenssh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+NAME=rclone-sftp-openssh
+USER=rclone
+PASS=password
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name ${NAME} \
+           rclone/test-sftp-openssh
+    
+    echo type=sftp
+    echo host=$(docker_ip)
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    echo _connect=$(docker_ip):22
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestSFTPRclone

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+NAME=rclone-serve-sftp
+USER=rclone
+PASS=CranesBallotDorsey5
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           rclone/rclone \
+           serve sftp --user $USER --pass $PASS --addr :22 /data
+    
+    echo type=sftp
+    echo host=$(docker_ip)
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    echo _connect=$(docker_ip):22
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestSwiftAIO

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+NAME=swift-aio
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name ${NAME} \
+           bouncestorage/swift-aio
+    
+    echo type=swift
+    echo env_auth=false
+    echo user=test:tester
+    echo key=testing
+    echo auth=http://$(docker_ip):8080/auth/v1.0
+    echo _connect=$(docker_ip):8080
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestWebdavNextcloud

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+NAME=nextcloud
+USER=rclone
+PASS=ArmorAbleMale6
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           -e "SQLITE_DATABASE=nextcloud.db" \
+           -e "NEXTCLOUD_ADMIN_USER=rclone" \
+           -e "NEXTCLOUD_ADMIN_PASSWORD=$PASS" \
+           -e "NEXTCLOUD_TRUSTED_DOMAINS=*.*.*.*" \
+           nextcloud:latest
+    
+    echo type=webdav
+    echo url=http://$(docker_ip)/remote.php/webdav/
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    # the tests don't pass if we use the nextcloud features
+    # echo vendor=nextcloud
+    echo _connect=$(docker_ip):80
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestWebdavOwncloud

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+NAME=owncloud
+USER=rclone
+PASS=HarperGrayerFewest5
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           -e "OWNCLOUD_DOMAIN=${OWNCLOUD_DOMAIN}" \
+           -e "OWNCLOUD_DB_TYPE=sqlite" \
+           -e "OWNCLOUD_DB_NAME=oowncloud.db" \
+           -e "OWNCLOUD_ADMIN_USERNAME=$USER" \
+           -e "OWNCLOUD_ADMIN_PASSWORD=$PASS" \
+           -e "OWNCLOUD_MYSQL_UTF8MB4=true" \
+           -e "OWNCLOUD_REDIS_ENABLED=false" \
+           -e "OWNCLOUD_TRUSTED_DOMAINS=*.*.*.*" \
+           owncloud/server
+    
+    echo type=webdav
+    echo url=http://$(docker_ip):8080/remote.php/webdav/
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    echo vendor=owncloud
+    echo _connect=$(docker_ip):8080
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestWebdavRclone

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+NAME=rclone-serve-webdav
+USER=rclone
+PASS=PagansSwimExpiry9
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           rclone/rclone \
+           serve webdav --user $USER --pass $PASS --addr :80 /data
+    
+    echo type=webdav
+    echo url=http://$(docker_ip)/
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    echo _connect=$(docker_ip):80
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/docker.bash

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+stop() {
+    if status ; then
+        docker stop $NAME
+        echo "$NAME stopped"
+    fi
+}
+
+status() {
+    if docker ps --format "{{.Names}}" | grep ^${NAME}$ >/dev/null ; then
+        echo "$NAME running"
+    else
+        echo "$NAME not running"
+        return 1
+    fi
+    return 0
+}
+
+docker_ip() {
+    docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $NAME    
+}

fstest/testserver/init.d/run.bash

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+case "$1" in 
+    start)
+	start
+	;;
+    stop)
+	stop
+	;;
+    status)
+	status
+	;;
+    *)
+	echo "usage: $0 start|stop|status" >&2
+	exit 1
+	;;
+esac

fstest/testserver/testserver.go

@@ -0,0 +1,183 @@
+// Package testserver starts and stops test servers if required
+package testserver
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fspath"
+)
+
+var (
+	once      sync.Once
+	configDir string // where the config is stored
+	// Note of running servers
+	runningMu   sync.Mutex
+	running     = map[string]int{}
+	errNotFound = errors.New("command not found")
+)
+
+// Assume we are run somewhere within the rclone root
+func findConfig() (string, error) {
+	dir := filepath.Join("fstest", "testserver", "init.d")
+	for i := 0; i < 5; i++ {
+		fi, err := os.Stat(dir)
+		if err == nil && fi.IsDir() {
+			return filepath.Abs(dir)
+		} else if !os.IsNotExist(err) {
+			return "", err
+		}
+		dir = filepath.Join("..", dir)
+	}
+	return "", errors.New("couldn't find testserver config files - run from within rclone source")
+}
+
+// run the command returning the output and an error
+func run(name, command string) (out []byte, err error) {
+	cmdPath := filepath.Join(configDir, name)
+	fi, err := os.Stat(cmdPath)
+	if err != nil || fi.IsDir() {
+		return nil, errNotFound
+	}
+	cmd := exec.Command(cmdPath, command)
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		err = errors.Wrapf(err, "failed to run %s %s\n%s", cmdPath, command, string(out))
+	}
+	return out, err
+}
+
+// Check to see if the server is running
+func isRunning(name string) bool {
+	_, err := run(name, "status")
+	return err == nil
+}
+
+// envKey returns the environment variable name to set name, key
+func envKey(name, key string) string {
+	return fmt.Sprintf("RCLONE_CONFIG_%s_%s", strings.ToUpper(name), strings.ToUpper(key))
+}
+
+// match a line of config var=value
+var matchLine = regexp.MustCompile(`^([a-zA-Z_]+)=(.*)$`)
+
+// Start the server and set its env vars
+// Call with the mutex held
+func start(name string) error {
+	out, err := run(name, "start")
+	if err != nil {
+		return err
+	}
+	fs.Logf(name, "Starting server")
+	// parse the output and set environment vars from it
+	var connect string
+	for _, line := range bytes.Split(out, []byte("\n")) {
+		line = bytes.TrimSpace(line)
+		part := matchLine.FindSubmatch(line)
+		if part != nil {
+			key, value := part[1], part[2]
+			if string(key) == "_connect" {
+				connect = string(value)
+				continue
+			}
+
+			// fs.Debugf(name, "key = %q, envKey = %q, value = %q", key, envKey, value)
+			err = os.Setenv(envKey(name, string(key)), string(value))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	if connect == "" {
+		return nil
+	}
+	// If we got a _connect value then try to connect to it
+	const maxTries = 30
+	for i := 1; i <= maxTries; i++ {
+		fs.Debugf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries)
+		conn, err := net.Dial("tcp", connect)
+		if err == nil {
+			_ = conn.Close()
+			return nil
+		}
+		time.Sleep(time.Second)
+	}
+	return errors.Errorf("failed to connect to %q on %q", name, connect)
+}
+
+// Start starts the named test server which can be stopped by the
+// function returned.
+func Start(remoteName string) (fn func(), err error) {
+	var name string
+	name, _, err = fspath.Parse(remoteName)
+	if err != nil {
+		return nil, err
+	}
+	if name == "" {
+		// don't start the local backend
+		return func() {}, nil
+	}
+
+	// Make sure we know where the config is
+	once.Do(func() {
+		configDir, err = findConfig()
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	runningMu.Lock()
+	defer runningMu.Unlock()
+
+	if running[name] <= 0 {
+		// if server isn't running check to see if this server has
+		// been started already but not by us and stop it if so
+		if os.Getenv(envKey(name, "type")) == "" && isRunning(name) {
+			stop(name)
+		}
+		if !isRunning(name) {
+			err = start(name)
+			if err == errNotFound {
+				// if no file found then don't start or stop
+				return func() {}, nil
+			} else if err != nil {
+				return nil, err
+			}
+			running[name] = 0
+		} else {
+			running[name] = 1
+		}
+	}
+	running[name]++
+
+	return func() {
+		runningMu.Lock()
+		defer runningMu.Unlock()
+		stop(name)
+	}, nil
+
+}
+
+// Stops the named test server
+// Call with the mutex held
+func stop(name string) {
+	running[name]--
+	if running[name] <= 0 {
+		_, err := run(name, "stop")
+		if err != nil {
+			fs.Errorf(name, "Failed to stop server: %v", err)
+		}
+		running[name] = 0
+		fs.Logf(name, "Stopped server")
+	}
+}