serve: factor out common testing parts for ftp, sftp and webdav tests

cmd/serve/ftp/ftp_test.go

@@ -8,15 +8,15 @@
 package ftp
 
 import (
-	"context"
 	"fmt"
-	"os"
-	"os/exec"
 	"testing"
 
 	ftp "github.com/goftp/server"
 	_ "github.com/rclone/rclone/backend/local"
-	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/cmd/serve/servetest"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -25,68 +25,50 @@ const (
 	testHOST             = "localhost"
 	testPORT             = "51780"
 	testPASSIVEPORTRANGE = "30000-32000"
+	testUSER             = "rclone"
+	testPASS             = "password"
 )
 
 // TestFTP runs the ftp server then runs the unit tests for the
 // ftp remote against it.
 func TestFTP(t *testing.T) {
-	opt := DefaultOpt
-	opt.ListenAddr = testHOST + ":" + testPORT
-	opt.PassivePorts = testPASSIVEPORTRANGE
-	opt.BasicUser = "rclone"
-	opt.BasicPass = "password"
+	// Configure and start the server
+	start := func(f fs.Fs) (configmap.Simple, func()) {
+		opt := DefaultOpt
+		opt.ListenAddr = testHOST + ":" + testPORT
+		opt.PassivePorts = testPASSIVEPORTRANGE
+		opt.BasicUser = testUSER
+		opt.BasicPass = testPASS
 
-	fstest.Initialise()
-
-	fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
-	assert.NoError(t, err)
-	defer clean()
-
-	err = fremote.Mkdir(context.Background(), "")
-	assert.NoError(t, err)
-
-	// Start the server
-	w, err := newServer(fremote, &opt)
-	assert.NoError(t, err)
-
-	go func() {
-		err := w.serve()
-		if err != ftp.ErrServerClosed {
-			assert.NoError(t, err)
-		}
-	}()
-	defer func() {
-		err := w.close()
+		w, err := newServer(f, &opt)
 		assert.NoError(t, err)
-	}()
 
-	// Change directory to run the tests
-	err = os.Chdir("../../../backend/ftp")
-	assert.NoError(t, err, "failed to cd to ftp remote")
+		quit := make(chan struct{})
+		go func() {
+			err := w.serve()
+			close(quit)
+			if err != ftp.ErrServerClosed {
+				assert.NoError(t, err)
+			}
+		}()
 
-	// Run the ftp tests with an on the fly remote
-	args := []string{"test"}
-	if testing.Verbose() {
-		args = append(args, "-v")
+		// Config for the backend we'll use to connect to the server
+		config := configmap.Simple{
+			"type": "ftp",
+			"host": testHOST,
+			"port": testPORT,
+			"user": testUSER,
+			"pass": obscure.MustObscure(testPASS),
+		}
+
+		return config, func() {
+			err := w.close()
+			assert.NoError(t, err)
+			<-quit
+		}
 	}
-	if *fstest.Verbose {
-		args = append(args, "-verbose")
-	}
-	args = append(args, "-list-retries", fmt.Sprint(*fstest.ListRetries))
-	args = append(args, "-remote", "ftptest:")
-	cmd := exec.Command("go", args...)
-	cmd.Env = append(os.Environ(),
-		"RCLONE_CONFIG_FTPTEST_TYPE=ftp",
-		"RCLONE_CONFIG_FTPTEST_HOST="+testHOST,
-		"RCLONE_CONFIG_FTPTEST_PORT="+testPORT,
-		"RCLONE_CONFIG_FTPTEST_USER=rclone",
-		"RCLONE_CONFIG_FTPTEST_PASS=0HU5Hx42YiLoNGJxppOOP3QTbr-KB_MP", // ./rclone obscure password
-	)
-	out, err := cmd.CombinedOutput()
-	if len(out) != 0 {
-		t.Logf("\n----------\n%s----------\n", string(out))
-	}
-	assert.NoError(t, err, "Running ftp integration tests")
+
+	servetest.Run(t, "ftp", start)
 }
 
 func TestFindID(t *testing.T) {

cmd/serve/servetest/servetest.go

@@ -0,0 +1,69 @@
+// Package servetest provides infrastructure for running loopback
+// tests of "rclone serve backend:" against the backend integration
+// tests.
+package servetest
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+)
+
+// StartFn describes the callback which should start the server,
+// return a config and a clean up function
+type StartFn func(f fs.Fs) (configmap.Simple, func())
+
+// Run runs the server then runs the unit tests for the remote against
+// it.
+func Run(t *testing.T, name string, start StartFn) {
+	fstest.Initialise()
+
+	fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
+	assert.NoError(t, err)
+	defer clean()
+
+	err = fremote.Mkdir(context.Background(), "")
+	assert.NoError(t, err)
+
+	config, cleanup := start(fremote)
+	defer cleanup()
+
+	// Change directory to run the tests
+	err = os.Chdir("../../../backend/" + name)
+	assert.NoError(t, err, "failed to cd to "+name+" backend")
+
+	// Run the backend tests with an on the fly remote
+	args := []string{"test"}
+	if testing.Verbose() {
+		args = append(args, "-v")
+	}
+	if *fstest.Verbose {
+		args = append(args, "-verbose")
+	}
+	remoteName := name + "test:"
+	args = append(args, "-remote", remoteName)
+	args = append(args, "-list-retries", fmt.Sprint(*fstest.ListRetries))
+	cmd := exec.Command("go", args...)
+
+	// Configure the backend with environment variables
+	cmd.Env = os.Environ()
+	prefix := "RCLONE_CONFIG_" + strings.ToUpper(remoteName[:len(remoteName)-1]) + "_"
+	for k, v := range config {
+		cmd.Env = append(cmd.Env, prefix+strings.ToUpper(k)+"="+v)
+	}
+
+	// Run the test
+	out, err := cmd.CombinedOutput()
+	if len(out) != 0 {
+		t.Logf("\n----------\n%s----------\n", string(out))
+	}
+	assert.NoError(t, err, "Running "+name+" integration tests")
+}

cmd/serve/sftp/sftp_test.go

@@ -8,16 +8,15 @@
 package sftp
 
 import (
-	"context"
-	"os"
-	"os/exec"
 	"strings"
 	"testing"
 
 	"github.com/pkg/sftp"
 	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/cmd/serve/servetest"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fstest"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -38,58 +37,35 @@ var (
 // TestSftp runs the sftp server then runs the unit tests for the
 // sftp remote against it.
 func TestSftp(t *testing.T) {
-	fstest.Initialise()
+	// Configure and start the server
+	start := func(f fs.Fs) (configmap.Simple, func()) {
+		opt := DefaultOpt
+		opt.ListenAddr = testBindAddress
+		opt.User = testUser
+		opt.Pass = testPass
 
-	fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
-	assert.NoError(t, err)
-	defer clean()
+		w := newServer(f, &opt)
+		assert.NoError(t, w.serve())
 
-	err = fremote.Mkdir(context.Background(), "")
-	assert.NoError(t, err)
+		// Read the host and port we started on
+		addr := w.Addr()
+		colon := strings.LastIndex(addr, ":")
 
-	opt := DefaultOpt
-	opt.ListenAddr = testBindAddress
-	opt.User = testUser
-	opt.Pass = testPass
+		// Config for the backend we'll use to connect to the server
+		config := configmap.Simple{
+			"type": "sftp",
+			"user": testUser,
+			"pass": obscure.MustObscure(testPass),
+			"host": addr[:colon],
+			"port": addr[colon+1:],
+		}
 
-	// Start the server
-	w := newServer(fremote, &opt)
-	assert.NoError(t, w.serve())
-	defer func() {
-		w.Close()
-		w.Wait()
-	}()
-
-	// Change directory to run the tests
-	err = os.Chdir("../../../backend/sftp")
-	assert.NoError(t, err, "failed to cd to sftp backend")
-
-	// Run the sftp tests with an on the fly remote
-	args := []string{"test"}
-	if testing.Verbose() {
-		args = append(args, "-v")
+		// return a stop function
+		return config, func() {
+			w.Close()
+			w.Wait()
+		}
 	}
-	if *fstest.Verbose {
-		args = append(args, "-verbose")
-	}
-	args = append(args, "-remote", "sftptest:")
-	cmd := exec.Command("go", args...)
-	addr := w.Addr()
-	colon := strings.LastIndex(addr, ":")
-	if colon < 0 {
-		panic("need a : in the address: " + addr)
-	}
-	host, port := addr[:colon], addr[colon+1:]
-	cmd.Env = append(os.Environ(),
-		"RCLONE_CONFIG_SFTPTEST_TYPE=sftp",
-		"RCLONE_CONFIG_SFTPTEST_HOST="+host,
-		"RCLONE_CONFIG_SFTPTEST_PORT="+port,
-		"RCLONE_CONFIG_SFTPTEST_USER="+testUser,
-		"RCLONE_CONFIG_SFTPTEST_PASS="+obscure.MustObscure(testPass),
-	)
-	out, err := cmd.CombinedOutput()
-	if len(out) != 0 {
-		t.Logf("\n----------\n%s----------\n", string(out))
-	}
-	assert.NoError(t, err, "Running sftp integration tests")
+
+	servetest.Run(t, "sftp", start)
 }

cmd/serve/webdav/webdav_test.go

@@ -8,21 +8,22 @@
 package webdav
 
 import (
-	"context"
 	"flag"
 	"io/ioutil"
 	"net/http"
 	"os"
-	"os/exec"
 	"strings"
 	"testing"
 	"time"
 
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/cmd/serve/httplib"
+	"github.com/rclone/rclone/cmd/serve/servetest"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/filter"
-	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fs/hash"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/webdav"
@@ -30,6 +31,8 @@ import (
 
 const (
 	testBindAddress = "localhost:0"
+	testUser        = "user"
+	testPass        = "pass"
 )
 
 // check interfaces
@@ -42,50 +45,34 @@ var (
 // TestWebDav runs the webdav server then runs the unit tests for the
 // webdav remote against it.
 func TestWebDav(t *testing.T) {
-	opt := httplib.DefaultOpt
-	opt.ListenAddr = testBindAddress
+	// Configure and start the server
+	start := func(f fs.Fs) (configmap.Simple, func()) {
+		opt := httplib.DefaultOpt
+		opt.ListenAddr = testBindAddress
+		opt.BasicUser = testUser
+		opt.BasicPass = testPass
+		hashType = hash.MD5
 
-	fstest.Initialise()
+		// Start the server
+		w := newWebDAV(f, &opt)
+		assert.NoError(t, w.serve())
 
-	fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
-	assert.NoError(t, err)
-	defer clean()
+		// Config for the backend we'll use to connect to the server
+		config := configmap.Simple{
+			"type":   "webdav",
+			"vendor": "other",
+			"url":    w.Server.URL(),
+			"user":   testUser,
+			"pass":   obscure.MustObscure(testPass),
+		}
 
-	err = fremote.Mkdir(context.Background(), "")
-	assert.NoError(t, err)
-
-	// Start the server
-	w := newWebDAV(fremote, &opt)
-	assert.NoError(t, w.serve())
-	defer func() {
-		w.Close()
-		w.Wait()
-	}()
-
-	// Change directory to run the tests
-	err = os.Chdir("../../../backend/webdav")
-	assert.NoError(t, err, "failed to cd to webdav remote")
-
-	// Run the webdav tests with an on the fly remote
-	args := []string{"test"}
-	if testing.Verbose() {
-		args = append(args, "-v")
+		return config, func() {
+			w.Close()
+			w.Wait()
+		}
 	}
-	if *fstest.Verbose {
-		args = append(args, "-verbose")
-	}
-	args = append(args, "-remote", "webdavtest:")
-	cmd := exec.Command("go", args...)
-	cmd.Env = append(os.Environ(),
-		"RCLONE_CONFIG_WEBDAVTEST_TYPE=webdav",
-		"RCLONE_CONFIG_WEBDAVTEST_URL="+w.Server.URL(),
-		"RCLONE_CONFIG_WEBDAVTEST_VENDOR=other",
-	)
-	out, err := cmd.CombinedOutput()
-	if len(out) != 0 {
-		t.Logf("\n----------\n%s----------\n", string(out))
-	}
-	assert.NoError(t, err, "Running webdav integration tests")
+
+	servetest.Run(t, "webdav", start)
 }
 
 // Test serve http functionality in serve webdav
@@ -97,10 +84,6 @@ var (
 )
 
 func TestHTTPFunction(t *testing.T) {
-	// cd to correct directory for testing
-	err := os.Chdir("../../cmd/serve/webdav")
-	assert.NoError(t, err, "failed to cd to webdav cmd directory")
-
 	// exclude files called hidden.txt and directories called hidden
 	require.NoError(t, filter.Active.AddRule("- hidden.txt"))
 	require.NoError(t, filter.Active.AddRule("- hidden/**"))