From 5e91b93e59f3ac925189b5b883ebe9a605355216 Mon Sep 17 00:00:00 2001 From: Ole Frost <82263101+olefrost@users.noreply.github.com> Date: Fri, 4 Jun 2021 10:56:40 +0200 Subject: [PATCH] cmdtest: end-to-end test for commands, flags and environment variables There was no easy way to automatically test the end-to-end functionality of commands, flags, environment variables etc. The need for end-to-end testing was highlighted by the issues fixed in #5341. There was no automated test to continually verify current behaviour, nor a framework to quickly test the correctness of the fixes. This change adds an end-to-end testing framework in the cmdtest folder. It has some simple examples in func TestCmdTest in cmdtest_test.go. The tests should be readable by anybody familiar with rclone and look like this: // Test the rclone version command with debug logging (-vv) out, err = rclone("version", "-vv") if assert.NoError(t, err) { assert.Contains(t, out, "rclone v") assert.Contains(t, out, "os/version:") assert.Contains(t, out, " DEBUG : ") } The end-to-end tests are executed just like the Go unit tests, that is: go test ./cmdtest -v The change also contains a thorough test of environment variables in environment_test.go. Thanks to @ncw for encouragement and introduction to the TestMain trick. --- CONTRIBUTING.md | 1 + cmdtest/cmdtest.go | 19 +++ cmdtest/cmdtest_test.go | 228 +++++++++++++++++++++++++++++ cmdtest/environment_test.go | 276 ++++++++++++++++++++++++++++++++++++ docs/content/docs.md | 12 ++ 5 files changed, 536 insertions(+) create mode 100644 cmdtest/cmdtest.go create mode 100644 cmdtest/cmdtest_test.go create mode 100644 cmdtest/environment_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4123444fc..de57039ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -228,6 +228,7 @@ with modules beneath. * cmd - the rclone commands * all - import this to load all the commands * ...commands + * cmdtest - end-to-end tests of commands, flags, environment variables,... * docs - the documentation and website * content - adjust these docs only - everything else is autogenerated * command - these are auto generated - edit the corresponding .go file diff --git a/cmdtest/cmdtest.go b/cmdtest/cmdtest.go new file mode 100644 index 000000000..663178a1d --- /dev/null +++ b/cmdtest/cmdtest.go @@ -0,0 +1,19 @@ +// Package cmdtest creates a testable interface to rclone main +// +// The interface is used to perform end-to-end test of +// commands, flags, environment variables etc. +// +package cmdtest + +// The rest of this file is a 1:1 copy from rclone.go + +import ( + _ "github.com/rclone/rclone/backend/all" // import all backends + "github.com/rclone/rclone/cmd" + _ "github.com/rclone/rclone/cmd/all" // import all commands + _ "github.com/rclone/rclone/lib/plugin" // import plugins +) + +func main() { + cmd.Main() +} diff --git a/cmdtest/cmdtest_test.go b/cmdtest/cmdtest_test.go new file mode 100644 index 000000000..f8c6adbbb --- /dev/null +++ b/cmdtest/cmdtest_test.go @@ -0,0 +1,228 @@ +// cmdtest_test creates a testable interface to rclone main +// +// The interface is used to perform end-to-end test of +// commands, flags, environment variables etc. + +package cmdtest + +import ( + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMain is initially called by go test to initiate the testing. +// TestMain is also called during the tests to start rclone main in a fresh context (using exec.Command). +// The context is determined by setting/finding the environment variable RCLONE_TEST_MAIN +func TestMain(m *testing.M) { + _, found := os.LookupEnv(rcloneTestMain) + if !found { + // started by Go test => execute tests + err := os.Setenv(rcloneTestMain, "true") + if err != nil { + log.Fatalf("Unable to set %s: %s", rcloneTestMain, err.Error()) + } + os.Exit(m.Run()) + } else { + // started by func rcloneExecMain => call rclone main in cmdtest.go + err := os.Unsetenv(rcloneTestMain) + if err != nil { + log.Fatalf("Unable to unset %s: %s", rcloneTestMain, err.Error()) + } + main() + } +} + +const rcloneTestMain = "RCLONE_TEST_MAIN" + +// rcloneExecMain calls rclone with the given environment and arguments. +// The environment variables are in a single string separated by ; +// The terminal output is retuned as a string. +func rcloneExecMain(env string, args ...string) (string, error) { + _, found := os.LookupEnv(rcloneTestMain) + if !found { + log.Fatalf("Unexpected execution path: %s is missing.", rcloneTestMain) + } + // make a call to self to execute rclone main in a predefined environment (enters TestMain above) + command := exec.Command(os.Args[0], args...) + command.Env = getEnvInitial() + if env != "" { + command.Env = append(command.Env, strings.Split(env, ";")...) + } + out, err := command.CombinedOutput() + return string(out), err +} + +// rcloneEnv calls rclone with the given environment and arguments. +// The environment variables are in a single string separated by ; +// The test config file is automatically configured in RCLONE_CONFIG. +// The terminal output is retuned as a string. +func rcloneEnv(env string, args ...string) (string, error) { + envConfig := env + if testConfig != "" { + if envConfig != "" { + envConfig += ";" + } + envConfig += "RCLONE_CONFIG=" + testConfig + } + return rcloneExecMain(envConfig, args...) +} + +// rclone calls rclone with the given arguments, E.g. "version","--help". +// The test config file is automatically configured in RCLONE_CONFIG. +// The terminal output is retuned as a string. +func rclone(args ...string) (string, error) { + return rcloneEnv("", args...) +} + +// getEnvInitial returns the os environment variables cleaned for RCLONE_ vars (except RCLONE_TEST_MAIN). +func getEnvInitial() []string { + if envInitial == nil { + // Set initial environment variables + osEnv := os.Environ() + for i := range osEnv { + if !strings.HasPrefix(osEnv[i], "RCLONE_") || strings.HasPrefix(osEnv[i], rcloneTestMain) { + envInitial = append(envInitial, osEnv[i]) + } + } + } + return envInitial +} + +var envInitial []string + +// createTestEnvironment creates a temporary testFolder and +// sets testConfig to testFolder/rclone.config. +func createTestEnvironment(t *testing.T) { + //Set temporary folder for config and test data + tempFolder, err := ioutil.TempDir("", "rclone_cmdtest_") + require.NoError(t, err) + testFolder = filepath.ToSlash(tempFolder) + + // Set path to temporary config file + testConfig = testFolder + "/rclone.config" +} + +var testFolder string +var testConfig string + +// removeTestEnvironment removes the test environment created by createTestEnvironment +func removeTestEnvironment(t *testing.T) { + // Remove temporary folder with all contents + err := os.RemoveAll(testFolder) + require.NoError(t, err) +} + +// createTestFile creates the file testFolder/name +func createTestFile(name string, t *testing.T) string { + err := ioutil.WriteFile(testFolder+"/"+name, []byte("content_of_"+name), 0666) + require.NoError(t, err) + return testFolder + "/" + name +} + +// createTestFolder creates the folder testFolder/name +func createTestFolder(name string, t *testing.T) string { + err := os.Mkdir(testFolder+"/"+name, 0777) + require.NoError(t, err) + return testFolder + "/" + name +} + +// createSimpleTestData creates simple test data in testFolder/subFolder +func createSimpleTestData(t *testing.T) string { + createTestFolder("testdata", t) + createTestFile("testdata/file1.txt", t) + createTestFile("testdata/file2.txt", t) + createTestFolder("testdata/folderA", t) + createTestFile("testdata/folderA/fileA1.txt", t) + createTestFile("testdata/folderA/fileA2.txt", t) + createTestFolder("testdata/folderA/folderAA", t) + createTestFile("testdata/folderA/folderAA/fileAA1.txt", t) + createTestFile("testdata/folderA/folderAA/fileAA2.txt", t) + createTestFolder("testdata/folderB", t) + createTestFile("testdata/folderB/fileB1.txt", t) + createTestFile("testdata/folderB/fileB2.txt", t) + return testFolder + "/testdata" +} + +// removeSimpleTestData removes the test data created by createSimpleTestData +func removeSimpleTestData(t *testing.T) { + err := os.RemoveAll(testFolder + "/testdata") + require.NoError(t, err) +} + +// TestCmdTest demonstrates and verifies the test functions for end-to-end testing of rclone +func TestCmdTest(t *testing.T) { + createTestEnvironment(t) + defer removeTestEnvironment(t) + + // Test simple call and output from rclone + out, err := rclone("version") + t.Logf("rclone version\n" + out) + if assert.NoError(t, err) { + assert.Contains(t, out, "rclone v") + assert.Contains(t, out, "version: ") + assert.NotContains(t, out, "Error:") + assert.NotContains(t, out, "--help") + assert.NotContains(t, out, " DEBUG : ") + assert.Regexp(t, "rclone\\s+v\\d+\\.\\d+", out) // rclone v_.__ + } + + // Test multiple arguments and DEBUG output + out, err = rclone("version", "-vv") + if assert.NoError(t, err) { + assert.Contains(t, out, "rclone v") + assert.Contains(t, out, " DEBUG : ") + } + + // Test error and error output + out, err = rclone("version", "--provoke-an-error") + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "exit status 1") + assert.Contains(t, out, "Error: unknown flag") + } + + // Test effect of environment variable + env := "RCLONE_LOG_LEVEL=DEBUG" + out, err = rcloneEnv(env, "version") + if assert.NoError(t, err) { + assert.Contains(t, out, "rclone v") + assert.Contains(t, out, " DEBUG : ") + } + + // Test effect of multiple environment variables, including one with , + env = "RCLONE_LOG_LEVEL=DEBUG;RCLONE_LOG_FORMAT=date,shortfile;RCLONE_STATS=173ms" + out, err = rcloneEnv(env, "version") + if assert.NoError(t, err) { + assert.Contains(t, out, "rclone v") + assert.Contains(t, out, " DEBUG : ") + assert.Regexp(t, "[^\\s]+\\.go:\\d+:", out) // ___.go:__: + assert.Contains(t, out, "173ms") + } + + // Test setup of config file + out, err = rclone("config", "create", "myLocal", "local") + if assert.NoError(t, err) { + assert.Contains(t, out, "[myLocal]") + assert.Contains(t, out, "type = local") + } + + // Test creation of simple test data + createSimpleTestData(t) + defer removeSimpleTestData(t) + + // Test access to config file and simple test data + out, err = rclone("lsl", "myLocal:"+testFolder) + t.Logf("rclone lsl myLocal:testFolder\n" + out) + if assert.NoError(t, err) { + assert.Contains(t, out, "rclone.config") + assert.Contains(t, out, "testdata/folderA/fileA1.txt") + } + +} diff --git a/cmdtest/environment_test.go b/cmdtest/environment_test.go new file mode 100644 index 000000000..14538522e --- /dev/null +++ b/cmdtest/environment_test.go @@ -0,0 +1,276 @@ +// environment_test tests the use and precedence of environment variables +// +// The tests rely on functions defined in cmdtest_test.go + +package cmdtest + +import ( + "os" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestCmdTest demonstrates and verifies the test functions for end-to-end testing of rclone +func TestEnvironmentVariables(t *testing.T) { + + createTestEnvironment(t) + defer removeTestEnvironment(t) + + testdataPath := createSimpleTestData(t) + defer removeSimpleTestData(t) + + // Non backend flags + // ================= + + // First verify default behaviour of the implicit max_depth=-1 + env := "" + out, err := rcloneEnv(env, "lsl", testFolder) + //t.Logf("\n" + out) + if assert.NoError(t, err) { + assert.Contains(t, out, "rclone.config") // depth 1 + assert.Contains(t, out, "file1.txt") // depth 2 + assert.Contains(t, out, "fileA1.txt") // depth 3 + assert.Contains(t, out, "fileAA1.txt") // depth 4 + } + + // Test of flag.Value + env = "RCLONE_MAX_DEPTH=2" + out, err = rcloneEnv(env, "lsl", testFolder) + if assert.NoError(t, err) { + assert.Contains(t, out, "file1.txt") // depth 2 + assert.NotContains(t, out, "fileA1.txt") // depth 3 + } + + // Test of flag.Changed (tests #5341 Issue1) + env = "RCLONE_LOG_LEVEL=DEBUG" + out, err = rcloneEnv(env, "version", "--quiet") + if assert.Error(t, err) { + assert.Contains(t, out, " DEBUG : ") + assert.Contains(t, out, "Can't set -q and --log-level") + assert.Contains(t, "exit status 1", err.Error()) + } + + // Test of flag.DefValue + env = "RCLONE_STATS=173ms" + out, err = rcloneEnv(env, "help", "flags") + if assert.NoError(t, err) { + assert.Contains(t, out, "(default 173ms)") + } + + // Test of command line flags overriding environment flags + env = "RCLONE_MAX_DEPTH=2" + out, err = rcloneEnv(env, "lsl", testFolder, "--max-depth", "3") + if assert.NoError(t, err) { + assert.Contains(t, out, "fileA1.txt") // depth 3 + assert.NotContains(t, out, "fileAA1.txt") // depth 4 + } + + // Test of debug logging while initialising flags from environment (tests #5241 Enhance1) + env = "RCLONE_STATS=173ms" + out, err = rcloneEnv(env, "version", "-vv") + if assert.NoError(t, err) { + assert.Contains(t, out, " DEBUG : ") + assert.Contains(t, out, "--stats") + assert.Contains(t, out, "173ms") + assert.Contains(t, out, "RCLONE_STATS=") + } + + // Backend flags and option precedence + // =================================== + + // Test approach: + // Verify no symlink warning when skip_links=true one the level with highest precedence + // and skip_links=false on all levels with lower precedence + // + // Reference: https://rclone.org/docs/#precedence + + // Create a symlink in test data + err = os.Symlink(testdataPath+"/folderA", testdataPath+"/symlinkA") + if runtime.GOOS == "windows" { + errNote := "The policy settings on Windows often prohibit the creation of symlinks due to security issues.\n" + errNote += "You can safely ignore this test, if your change didn't affect environment variables." + require.NoError(t, err, errNote) + } else { + require.NoError(t, err) + } + + // Create a local remote with explicit skip_links=false + out, err = rclone("config", "create", "myLocal", "local", "skip_links", "false") + if assert.NoError(t, err) { + assert.Contains(t, out, "[myLocal]") + assert.Contains(t, out, "type = local") + assert.Contains(t, out, "skip_links = false") + } + + // Verify symlink warning when skip_links=false on all levels + env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false" + out, err = rcloneEnv(env, "lsd", "myLocal,skip_links=false:"+testdataPath, "--skip-links=false") + //t.Logf("\n" + out) + if assert.NoError(t, err) { + assert.Contains(t, out, "NOTICE: symlinkA:") + assert.Contains(t, out, "folderA") + } + + // Test precedence of connection strings + env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false" + out, err = rcloneEnv(env, "lsd", "myLocal,skip_links:"+testdataPath, "--skip-links=false") + if assert.NoError(t, err) { + assert.NotContains(t, out, "symlinkA") + assert.Contains(t, out, "folderA") + } + + // Test precedence of command line flags + env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false" + out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath, "--skip-links") + if assert.NoError(t, err) { + assert.NotContains(t, out, "symlinkA") + assert.Contains(t, out, "folderA") + } + + // Test precedence of remote specific environment variables (tests #5341 Issue2) + env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=true" + out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath) + if assert.NoError(t, err) { + assert.NotContains(t, out, "symlinkA") + assert.Contains(t, out, "folderA") + } + + // Test precedence of backend specific environment variables (tests #5341 Issue3) + env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=true" + out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath) + if assert.NoError(t, err) { + assert.NotContains(t, out, "symlinkA") + assert.Contains(t, out, "folderA") + } + + // Test precedence of backend generic environment variables + env = "RCLONE_SKIP_LINKS=true" + out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath) + if assert.NoError(t, err) { + assert.NotContains(t, out, "symlinkA") + assert.Contains(t, out, "folderA") + } + + // Recreate the test remote with explicit skip_links=true + out, err = rclone("config", "create", "myLocal", "local", "skip_links", "true") + if assert.NoError(t, err) { + assert.Contains(t, out, "[myLocal]") + assert.Contains(t, out, "type = local") + assert.Contains(t, out, "skip_links = true") + } + + // Test precedence of config file options + env = "" + out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath) + if assert.NoError(t, err) { + assert.NotContains(t, out, "symlinkA") + assert.Contains(t, out, "folderA") + } + + // Recreate the test remote with rclone defaults, that is implicit skip_links=false + out, err = rclone("config", "create", "myLocal", "local") + if assert.NoError(t, err) { + assert.Contains(t, out, "[myLocal]") + assert.Contains(t, out, "type = local") + assert.NotContains(t, out, "skip_links") + } + + // Verify the rclone default value (implicit skip_links=false) + env = "" + out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath) + if assert.NoError(t, err) { + assert.Contains(t, out, "NOTICE: symlinkA:") + assert.Contains(t, out, "folderA") + } + + // Display of backend defaults (tests #4659) + //------------------------------------------ + + env = "RCLONE_DRIVE_CHUNK_SIZE=111M" + out, err = rcloneEnv(env, "help", "flags") + if assert.NoError(t, err) { + assert.Regexp(t, "--drive-chunk-size[^\\(]+\\(default 111M\\)", out) + } + + // Options on referencing remotes (alias, crypt, etc.) + //---------------------------------------------------- + + // Create alias remote on myLocal having implicit skip_links=false + out, err = rclone("config", "create", "myAlias", "alias", "remote", "myLocal:"+testdataPath) + if assert.NoError(t, err) { + assert.Contains(t, out, "[myAlias]") + assert.Contains(t, out, "type = alias") + assert.Contains(t, out, "remote = myLocal:") + } + + // Verify symlink warnings on the alias + env = "" + out, err = rcloneEnv(env, "lsd", "myAlias:") + if assert.NoError(t, err) { + assert.Contains(t, out, "NOTICE: symlinkA:") + assert.Contains(t, out, "folderA") + } + + // Test backend generic flags + // having effect on the underlying local remote + env = "RCLONE_SKIP_LINKS=true" + out, err = rcloneEnv(env, "lsd", "myAlias:") + if assert.NoError(t, err) { + assert.NotContains(t, out, "symlinkA") + assert.Contains(t, out, "folderA") + } + + // Test backend specific flags + // having effect on the underlying local remote + env = "RCLONE_LOCAL_SKIP_LINKS=true" + out, err = rcloneEnv(env, "lsd", "myAlias:") + if assert.NoError(t, err) { + assert.NotContains(t, out, "symlinkA") + assert.Contains(t, out, "folderA") + } + + // Test remote specific flags + // having no effect unless supported by the immediate remote (alias) + env = "RCLONE_CONFIG_MYALIAS_SKIP_LINKS=true" + out, err = rcloneEnv(env, "lsd", "myAlias:") + if assert.NoError(t, err) { + assert.Contains(t, out, "NOTICE: symlinkA:") + assert.Contains(t, out, "folderA") + } + + env = "RCLONE_CONFIG_MYALIAS_REMOTE=" + "myLocal:" + testdataPath + "/folderA" + out, err = rcloneEnv(env, "lsl", "myAlias:") + if assert.NoError(t, err) { + assert.Contains(t, out, "fileA1.txt") + assert.NotContains(t, out, "fileB1.txt") + } + + // Test command line flags + // having effect on the underlying local remote + env = "" + out, err = rcloneEnv(env, "lsd", "myAlias:", "--skip-links") + if assert.NoError(t, err) { + assert.NotContains(t, out, "symlinkA") + assert.Contains(t, out, "folderA") + } + + // Test connection specific flags + // having no effect unless supported by the immediate remote (alias) + env = "" + out, err = rcloneEnv(env, "lsd", "myAlias,skip_links:") + if assert.NoError(t, err) { + assert.Contains(t, out, "NOTICE: symlinkA:") + assert.Contains(t, out, "folderA") + } + + env = "" + out, err = rcloneEnv(env, "lsl", "myAlias,remote='myLocal:"+testdataPath+"/folderA':", "-vv") + if assert.NoError(t, err) { + assert.Contains(t, out, "fileA1.txt") + assert.NotContains(t, out, "fileB1.txt") + } + +} diff --git a/docs/content/docs.md b/docs/content/docs.md index ea547b7aa..f18a12a58 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -239,6 +239,13 @@ However using the connection string syntax, this does work. rclone copy "gdrive,shared_with_me:shared-file.txt" gdrive: +Note that the connection string only affects the options of the immediate +backend. If for example gdriveCrypt is a crypt based on gdrive, then the +following command **will not work** as intended, because +`shared_with_me` is ignored by the crypt backend: + + rclone copy "gdriveCrypt,shared_with_me:shared-file.txt" gdriveCrypt: + The connection strings have the following syntax remote,parameter=value,parameter2=value2:path/to/dir @@ -2157,6 +2164,11 @@ mys3: Note that if you want to create a remote using environment variables you must create the `..._TYPE` variable as above. +Note that you can only set the options of the immediate backend, +so RCLONE_CONFIG_MYS3CRYPT_ACCESS_KEY_ID has no effect, if myS3Crypt is +a crypt remote based on an S3 remote. However RCLONE_S3_ACCESS_KEY_ID will +set the access key of all remotes using S3, including myS3Crypt. + Note also that now rclone has [connection strings](#connection-strings), it is probably easier to use those instead which makes the above example