From ee7101e6af11f2037852db6f8fbd0f4c2232939f Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 2 Aug 2019 10:56:27 +0100 Subject: [PATCH] serve: factor out common testing parts for ftp, sftp and webdav tests --- cmd/serve/ftp/ftp_test.go | 94 +++++++++++++------------------- cmd/serve/servetest/servetest.go | 69 +++++++++++++++++++++++ cmd/serve/sftp/sftp_test.go | 82 ++++++++++------------------ cmd/serve/webdav/webdav_test.go | 77 ++++++++++---------------- 4 files changed, 166 insertions(+), 156 deletions(-) create mode 100644 cmd/serve/servetest/servetest.go diff --git a/cmd/serve/ftp/ftp_test.go b/cmd/serve/ftp/ftp_test.go index e66824427..b411a69a1 100644 --- a/cmd/serve/ftp/ftp_test.go +++ b/cmd/serve/ftp/ftp_test.go @@ -8,15 +8,15 @@ package ftp import ( - "context" "fmt" - "os" - "os/exec" "testing" ftp "github.com/goftp/server" _ "github.com/rclone/rclone/backend/local" - "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/cmd/serve/servetest" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/obscure" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -25,68 +25,50 @@ const ( testHOST = "localhost" testPORT = "51780" testPASSIVEPORTRANGE = "30000-32000" + testUSER = "rclone" + testPASS = "password" ) // TestFTP runs the ftp server then runs the unit tests for the // ftp remote against it. func TestFTP(t *testing.T) { - opt := DefaultOpt - opt.ListenAddr = testHOST + ":" + testPORT - opt.PassivePorts = testPASSIVEPORTRANGE - opt.BasicUser = "rclone" - opt.BasicPass = "password" + // Configure and start the server + start := func(f fs.Fs) (configmap.Simple, func()) { + opt := DefaultOpt + opt.ListenAddr = testHOST + ":" + testPORT + opt.PassivePorts = testPASSIVEPORTRANGE + opt.BasicUser = testUSER + opt.BasicPass = testPASS - fstest.Initialise() - - fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) - assert.NoError(t, err) - defer clean() - - err = fremote.Mkdir(context.Background(), "") - assert.NoError(t, err) - - // Start the server - w, err := newServer(fremote, &opt) - assert.NoError(t, err) - - go func() { - err := w.serve() - if err != ftp.ErrServerClosed { - assert.NoError(t, err) - } - }() - defer func() { - err := w.close() + w, err := newServer(f, &opt) assert.NoError(t, err) - }() - // Change directory to run the tests - err = os.Chdir("../../../backend/ftp") - assert.NoError(t, err, "failed to cd to ftp remote") + quit := make(chan struct{}) + go func() { + err := w.serve() + close(quit) + if err != ftp.ErrServerClosed { + assert.NoError(t, err) + } + }() - // Run the ftp tests with an on the fly remote - args := []string{"test"} - if testing.Verbose() { - args = append(args, "-v") + // Config for the backend we'll use to connect to the server + config := configmap.Simple{ + "type": "ftp", + "host": testHOST, + "port": testPORT, + "user": testUSER, + "pass": obscure.MustObscure(testPASS), + } + + return config, func() { + err := w.close() + assert.NoError(t, err) + <-quit + } } - if *fstest.Verbose { - args = append(args, "-verbose") - } - args = append(args, "-list-retries", fmt.Sprint(*fstest.ListRetries)) - args = append(args, "-remote", "ftptest:") - cmd := exec.Command("go", args...) - cmd.Env = append(os.Environ(), - "RCLONE_CONFIG_FTPTEST_TYPE=ftp", - "RCLONE_CONFIG_FTPTEST_HOST="+testHOST, - "RCLONE_CONFIG_FTPTEST_PORT="+testPORT, - "RCLONE_CONFIG_FTPTEST_USER=rclone", - "RCLONE_CONFIG_FTPTEST_PASS=0HU5Hx42YiLoNGJxppOOP3QTbr-KB_MP", // ./rclone obscure password - ) - out, err := cmd.CombinedOutput() - if len(out) != 0 { - t.Logf("\n----------\n%s----------\n", string(out)) - } - assert.NoError(t, err, "Running ftp integration tests") + + servetest.Run(t, "ftp", start) } func TestFindID(t *testing.T) { diff --git a/cmd/serve/servetest/servetest.go b/cmd/serve/servetest/servetest.go new file mode 100644 index 000000000..f964b17dd --- /dev/null +++ b/cmd/serve/servetest/servetest.go @@ -0,0 +1,69 @@ +// Package servetest provides infrastructure for running loopback +// tests of "rclone serve backend:" against the backend integration +// tests. +package servetest + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "testing" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fstest" + "github.com/stretchr/testify/assert" +) + +// StartFn describes the callback which should start the server, +// return a config and a clean up function +type StartFn func(f fs.Fs) (configmap.Simple, func()) + +// Run runs the server then runs the unit tests for the remote against +// it. +func Run(t *testing.T, name string, start StartFn) { + fstest.Initialise() + + fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) + assert.NoError(t, err) + defer clean() + + err = fremote.Mkdir(context.Background(), "") + assert.NoError(t, err) + + config, cleanup := start(fremote) + defer cleanup() + + // Change directory to run the tests + err = os.Chdir("../../../backend/" + name) + assert.NoError(t, err, "failed to cd to "+name+" backend") + + // Run the backend tests with an on the fly remote + args := []string{"test"} + if testing.Verbose() { + args = append(args, "-v") + } + if *fstest.Verbose { + args = append(args, "-verbose") + } + remoteName := name + "test:" + args = append(args, "-remote", remoteName) + args = append(args, "-list-retries", fmt.Sprint(*fstest.ListRetries)) + cmd := exec.Command("go", args...) + + // Configure the backend with environment variables + cmd.Env = os.Environ() + prefix := "RCLONE_CONFIG_" + strings.ToUpper(remoteName[:len(remoteName)-1]) + "_" + for k, v := range config { + cmd.Env = append(cmd.Env, prefix+strings.ToUpper(k)+"="+v) + } + + // Run the test + out, err := cmd.CombinedOutput() + if len(out) != 0 { + t.Logf("\n----------\n%s----------\n", string(out)) + } + assert.NoError(t, err, "Running "+name+" integration tests") +} diff --git a/cmd/serve/sftp/sftp_test.go b/cmd/serve/sftp/sftp_test.go index 21661756c..f9fa92795 100644 --- a/cmd/serve/sftp/sftp_test.go +++ b/cmd/serve/sftp/sftp_test.go @@ -8,16 +8,15 @@ package sftp import ( - "context" - "os" - "os/exec" "strings" "testing" "github.com/pkg/sftp" _ "github.com/rclone/rclone/backend/local" + "github.com/rclone/rclone/cmd/serve/servetest" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fstest" "github.com/stretchr/testify/assert" ) @@ -38,58 +37,35 @@ var ( // TestSftp runs the sftp server then runs the unit tests for the // sftp remote against it. func TestSftp(t *testing.T) { - fstest.Initialise() + // Configure and start the server + start := func(f fs.Fs) (configmap.Simple, func()) { + opt := DefaultOpt + opt.ListenAddr = testBindAddress + opt.User = testUser + opt.Pass = testPass - fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) - assert.NoError(t, err) - defer clean() + w := newServer(f, &opt) + assert.NoError(t, w.serve()) - err = fremote.Mkdir(context.Background(), "") - assert.NoError(t, err) + // Read the host and port we started on + addr := w.Addr() + colon := strings.LastIndex(addr, ":") - opt := DefaultOpt - opt.ListenAddr = testBindAddress - opt.User = testUser - opt.Pass = testPass + // Config for the backend we'll use to connect to the server + config := configmap.Simple{ + "type": "sftp", + "user": testUser, + "pass": obscure.MustObscure(testPass), + "host": addr[:colon], + "port": addr[colon+1:], + } - // Start the server - w := newServer(fremote, &opt) - assert.NoError(t, w.serve()) - defer func() { - w.Close() - w.Wait() - }() - - // Change directory to run the tests - err = os.Chdir("../../../backend/sftp") - assert.NoError(t, err, "failed to cd to sftp backend") - - // Run the sftp tests with an on the fly remote - args := []string{"test"} - if testing.Verbose() { - args = append(args, "-v") + // return a stop function + return config, func() { + w.Close() + w.Wait() + } } - if *fstest.Verbose { - args = append(args, "-verbose") - } - args = append(args, "-remote", "sftptest:") - cmd := exec.Command("go", args...) - addr := w.Addr() - colon := strings.LastIndex(addr, ":") - if colon < 0 { - panic("need a : in the address: " + addr) - } - host, port := addr[:colon], addr[colon+1:] - cmd.Env = append(os.Environ(), - "RCLONE_CONFIG_SFTPTEST_TYPE=sftp", - "RCLONE_CONFIG_SFTPTEST_HOST="+host, - "RCLONE_CONFIG_SFTPTEST_PORT="+port, - "RCLONE_CONFIG_SFTPTEST_USER="+testUser, - "RCLONE_CONFIG_SFTPTEST_PASS="+obscure.MustObscure(testPass), - ) - out, err := cmd.CombinedOutput() - if len(out) != 0 { - t.Logf("\n----------\n%s----------\n", string(out)) - } - assert.NoError(t, err, "Running sftp integration tests") + + servetest.Run(t, "sftp", start) } diff --git a/cmd/serve/webdav/webdav_test.go b/cmd/serve/webdav/webdav_test.go index 33193085e..61fcd829e 100644 --- a/cmd/serve/webdav/webdav_test.go +++ b/cmd/serve/webdav/webdav_test.go @@ -8,21 +8,22 @@ package webdav import ( - "context" "flag" "io/ioutil" "net/http" "os" - "os/exec" "strings" "testing" "time" _ "github.com/rclone/rclone/backend/local" "github.com/rclone/rclone/cmd/serve/httplib" + "github.com/rclone/rclone/cmd/serve/servetest" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/filter" - "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/fs/hash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/webdav" @@ -30,6 +31,8 @@ import ( const ( testBindAddress = "localhost:0" + testUser = "user" + testPass = "pass" ) // check interfaces @@ -42,50 +45,34 @@ var ( // TestWebDav runs the webdav server then runs the unit tests for the // webdav remote against it. func TestWebDav(t *testing.T) { - opt := httplib.DefaultOpt - opt.ListenAddr = testBindAddress + // Configure and start the server + start := func(f fs.Fs) (configmap.Simple, func()) { + opt := httplib.DefaultOpt + opt.ListenAddr = testBindAddress + opt.BasicUser = testUser + opt.BasicPass = testPass + hashType = hash.MD5 - fstest.Initialise() + // Start the server + w := newWebDAV(f, &opt) + assert.NoError(t, w.serve()) - fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir) - assert.NoError(t, err) - defer clean() + // Config for the backend we'll use to connect to the server + config := configmap.Simple{ + "type": "webdav", + "vendor": "other", + "url": w.Server.URL(), + "user": testUser, + "pass": obscure.MustObscure(testPass), + } - err = fremote.Mkdir(context.Background(), "") - assert.NoError(t, err) - - // Start the server - w := newWebDAV(fremote, &opt) - assert.NoError(t, w.serve()) - defer func() { - w.Close() - w.Wait() - }() - - // Change directory to run the tests - err = os.Chdir("../../../backend/webdav") - assert.NoError(t, err, "failed to cd to webdav remote") - - // Run the webdav tests with an on the fly remote - args := []string{"test"} - if testing.Verbose() { - args = append(args, "-v") + return config, func() { + w.Close() + w.Wait() + } } - if *fstest.Verbose { - args = append(args, "-verbose") - } - args = append(args, "-remote", "webdavtest:") - cmd := exec.Command("go", args...) - cmd.Env = append(os.Environ(), - "RCLONE_CONFIG_WEBDAVTEST_TYPE=webdav", - "RCLONE_CONFIG_WEBDAVTEST_URL="+w.Server.URL(), - "RCLONE_CONFIG_WEBDAVTEST_VENDOR=other", - ) - out, err := cmd.CombinedOutput() - if len(out) != 0 { - t.Logf("\n----------\n%s----------\n", string(out)) - } - assert.NoError(t, err, "Running webdav integration tests") + + servetest.Run(t, "webdav", start) } // Test serve http functionality in serve webdav @@ -97,10 +84,6 @@ var ( ) func TestHTTPFunction(t *testing.T) { - // cd to correct directory for testing - err := os.Chdir("../../cmd/serve/webdav") - assert.NoError(t, err, "failed to cd to webdav cmd directory") - // exclude files called hidden.txt and directories called hidden require.NoError(t, filter.Active.AddRule("- hidden.txt")) require.NoError(t, filter.Active.AddRule("- hidden/**"))