mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 00:13:49 +01:00
874d66658e
After the config re-organisation, the setting of stringArray config
values (eg `--exclude` set with `RCLONE_EXCLUDE`) was broken and gave
a message like this for `RCLONE_EXCLUDE=*.jpg`:
Failed to load "filter" default values: failed to initialise "filter" options:
couldn't parse config item "exclude" = "*.jpg" as []string: parsing "*.jpg" as []string failed:
invalid character '/' looking for beginning of value
This was caused by the parser trying to parse the input string as a
JSON value.
When the config was re-organised it was thought that the internal
representation of stringArray values was not important as it was never
visible externally, however this turned out not to be true.
A defined representation was chosen - a comma separated string and
this was documented and tests were introduced in this patch.
This potentially introduces a very small backwards incompatibility. In
rclone v1.67.0
RCLONE_EXCLUDE=a,b
Would be interpreted as
--exclude "a,b"
Whereas this new code will interpret it as
--exclude "a" --exclude "b"
The benefit of being able to set multiple values with an environment
variable was deemed to outweigh the very small backwards compatibility
risk.
If a value with a `,` is needed, then use CSV escaping, eg
RCLONE_EXCLUDE="a,b"
(Note this needs to have the quotes in so at the unix shell that would be
RCLONE_EXCLUDE='"a,b"'
Fixes #8063
388 lines
13 KiB
Go
388 lines
13 KiB
Go
// environment_test tests the use and precedence of environment variables
|
|
//
|
|
// The tests rely on functions defined in cmdtest_test.go
|
|
|
|
package cmdtest
|
|
|
|
import (
|
|
"os"
|
|
"regexp"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// TestCmdTest demonstrates and verifies the test functions for end-to-end testing of rclone
|
|
func TestEnvironmentVariables(t *testing.T) {
|
|
|
|
createTestEnvironment(t)
|
|
|
|
testdataPath := createSimpleTestData(t)
|
|
|
|
// Non backend flags
|
|
// =================
|
|
|
|
// First verify default behaviour of the implicit max_depth=-1
|
|
env := ""
|
|
out, err := rcloneEnv(env, "lsl", testFolder)
|
|
//t.Logf("\n" + out)
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "rclone.config") // depth 1
|
|
assert.Contains(t, out, "file1.txt") // depth 2
|
|
assert.Contains(t, out, "fileA1.txt") // depth 3
|
|
assert.Contains(t, out, "fileAA1.txt") // depth 4
|
|
}
|
|
|
|
// Test of flag.Value
|
|
env = "RCLONE_MAX_DEPTH=2"
|
|
out, err = rcloneEnv(env, "lsl", testFolder)
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "file1.txt") // depth 2
|
|
assert.NotContains(t, out, "fileA1.txt") // depth 3
|
|
}
|
|
|
|
// Test of flag.Changed (tests #5341 Issue1)
|
|
env = "RCLONE_LOG_LEVEL=DEBUG"
|
|
out, err = rcloneEnv(env, "version", "--quiet")
|
|
if assert.Error(t, err) {
|
|
assert.Contains(t, out, " DEBUG : ")
|
|
assert.Contains(t, out, "Can't set -q and --log-level")
|
|
assert.Contains(t, "exit status 1", err.Error())
|
|
}
|
|
|
|
// Test of flag.DefValue
|
|
env = "RCLONE_STATS=173ms"
|
|
out, err = rcloneEnv(env, "help", "flags")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "(default 173ms)")
|
|
}
|
|
|
|
// Test of command line flags overriding environment flags
|
|
env = "RCLONE_MAX_DEPTH=2"
|
|
out, err = rcloneEnv(env, "lsl", testFolder, "--max-depth", "3")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "fileA1.txt") // depth 3
|
|
assert.NotContains(t, out, "fileAA1.txt") // depth 4
|
|
}
|
|
|
|
// Test of debug logging while initialising flags from environment (tests #5341 Enhance1)
|
|
env = "RCLONE_STATS=173ms"
|
|
out, err = rcloneEnv(env, "version", "-vv")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, " DEBUG : ")
|
|
assert.Contains(t, out, "--stats")
|
|
assert.Contains(t, out, "173ms")
|
|
assert.Contains(t, out, "RCLONE_STATS=")
|
|
}
|
|
|
|
// Backend flags and remote name
|
|
// - The listremotes command includes names from environment variables,
|
|
// the part between "RCLONE_CONFIG_" and "_TYPE", converted to lowercase.
|
|
// - When using a remote created from env, e.g. with lsd command,
|
|
// the name is case insensitive in contrast to remotes in config file
|
|
// (fs.ConfigToEnv converts to uppercase before checking environment).
|
|
// - Previously using a remote created from env, e.g. with lsd command,
|
|
// would not be possible for remotes with '-' in names, and remote names
|
|
// with '_' could be referred to with both '-' and '_', because any '-'
|
|
// were replaced with '_' before lookup.
|
|
// ===================================
|
|
|
|
env = "RCLONE_CONFIG_MY-LOCAL_TYPE=local"
|
|
out, err = rcloneEnv(env, "listremotes")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "my-local:")
|
|
}
|
|
out, err = rcloneEnv(env, "lsl", "my-local:"+testFolder)
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "rclone.config")
|
|
assert.Contains(t, out, "file1.txt")
|
|
assert.Contains(t, out, "fileA1.txt")
|
|
assert.Contains(t, out, "fileAA1.txt")
|
|
}
|
|
out, err = rcloneEnv(env, "lsl", "mY-LoCaL:"+testFolder)
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "rclone.config")
|
|
assert.Contains(t, out, "file1.txt")
|
|
assert.Contains(t, out, "fileA1.txt")
|
|
assert.Contains(t, out, "fileAA1.txt")
|
|
}
|
|
out, err = rcloneEnv(env, "lsl", "my_local:"+testFolder)
|
|
if assert.Error(t, err) {
|
|
assert.Contains(t, out, "Failed to create file system")
|
|
}
|
|
|
|
env = "RCLONE_CONFIG_MY_LOCAL_TYPE=local"
|
|
out, err = rcloneEnv(env, "listremotes")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "my_local:")
|
|
}
|
|
out, err = rcloneEnv(env, "lsl", "my_local:"+testFolder)
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "rclone.config")
|
|
assert.Contains(t, out, "file1.txt")
|
|
assert.Contains(t, out, "fileA1.txt")
|
|
assert.Contains(t, out, "fileAA1.txt")
|
|
}
|
|
out, err = rcloneEnv(env, "lsl", "my-local:"+testFolder)
|
|
if assert.Error(t, err) {
|
|
assert.Contains(t, out, "Failed to create file system")
|
|
}
|
|
|
|
// Backend flags and option precedence
|
|
// ===================================
|
|
|
|
// Test approach:
|
|
// Verify no symlink warning when skip_links=true one the level with highest precedence
|
|
// and skip_links=false on all levels with lower precedence
|
|
//
|
|
// Reference: https://rclone.org/docs/#precedence
|
|
// Create a symlink in test data
|
|
err = os.Symlink(testdataPath+"/folderA", testdataPath+"/symlinkA")
|
|
if runtime.GOOS == "windows" {
|
|
errNote := "The policy settings on Windows often prohibit the creation of symlinks due to security issues.\n"
|
|
errNote += "You can safely ignore this test, if your change didn't affect environment variables."
|
|
require.NoError(t, err, errNote)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
// Create a local remote with explicit skip_links=false
|
|
out, err = rclone("config", "create", "myLocal", "local", "skip_links", "false")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "[myLocal]")
|
|
assert.Contains(t, out, "type = local")
|
|
assert.Contains(t, out, "skip_links = false")
|
|
}
|
|
|
|
// Verify symlink warning when skip_links=false on all levels
|
|
env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false"
|
|
out, err = rcloneEnv(env, "lsd", "myLocal,skip_links=false:"+testdataPath, "--skip-links=false")
|
|
//t.Logf("\n" + out)
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "NOTICE: symlinkA:")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Test precedence of connection strings
|
|
env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false"
|
|
out, err = rcloneEnv(env, "lsd", "myLocal,skip_links:"+testdataPath, "--skip-links=false")
|
|
if assert.NoError(t, err) {
|
|
assert.NotContains(t, out, "symlinkA")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Test precedence of command line flags
|
|
env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=false"
|
|
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath, "--skip-links")
|
|
if assert.NoError(t, err) {
|
|
assert.NotContains(t, out, "symlinkA")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Test precedence of remote specific environment variables (tests #5341 Issue2)
|
|
env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=false;RCLONE_CONFIG_MYLOCAL_SKIP_LINKS=true"
|
|
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
|
|
if assert.NoError(t, err) {
|
|
assert.NotContains(t, out, "symlinkA")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Test precedence of backend specific environment variables (tests #5341 Issue3)
|
|
env = "RCLONE_SKIP_LINKS=false;RCLONE_LOCAL_SKIP_LINKS=true"
|
|
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
|
|
if assert.NoError(t, err) {
|
|
assert.NotContains(t, out, "symlinkA")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Test precedence of backend generic environment variables
|
|
env = "RCLONE_SKIP_LINKS=true"
|
|
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
|
|
if assert.NoError(t, err) {
|
|
assert.NotContains(t, out, "symlinkA")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Recreate the test remote with explicit skip_links=true
|
|
out, err = rclone("config", "create", "myLocal", "local", "skip_links", "true")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "[myLocal]")
|
|
assert.Contains(t, out, "type = local")
|
|
assert.Contains(t, out, "skip_links = true")
|
|
}
|
|
|
|
// Test precedence of config file options
|
|
env = ""
|
|
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
|
|
if assert.NoError(t, err) {
|
|
assert.NotContains(t, out, "symlinkA")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Recreate the test remote with rclone defaults, that is implicit skip_links=false
|
|
out, err = rclone("config", "create", "myLocal", "local")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "[myLocal]")
|
|
assert.Contains(t, out, "type = local")
|
|
assert.NotContains(t, out, "skip_links")
|
|
}
|
|
|
|
// Verify the rclone default value (implicit skip_links=false)
|
|
env = ""
|
|
out, err = rcloneEnv(env, "lsd", "myLocal:"+testdataPath)
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "NOTICE: symlinkA:")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Display of backend defaults (tests #4659)
|
|
//------------------------------------------
|
|
|
|
env = "RCLONE_DRIVE_CHUNK_SIZE=111M"
|
|
out, err = rcloneEnv(env, "help", "flags")
|
|
if assert.NoError(t, err) {
|
|
assert.Regexp(t, "--drive-chunk-size[^\\(]+\\(default 111M\\)", out)
|
|
}
|
|
|
|
// Options on referencing remotes (alias, crypt, etc.)
|
|
//----------------------------------------------------
|
|
|
|
// Create alias remote on myLocal having implicit skip_links=false
|
|
out, err = rclone("config", "create", "myAlias", "alias", "remote", "myLocal:"+testdataPath)
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "[myAlias]")
|
|
assert.Contains(t, out, "type = alias")
|
|
assert.Contains(t, out, "remote = myLocal:")
|
|
}
|
|
|
|
// Verify symlink warnings on the alias
|
|
env = ""
|
|
out, err = rcloneEnv(env, "lsd", "myAlias:")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "NOTICE: symlinkA:")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Test backend generic flags
|
|
// having effect on the underlying local remote
|
|
env = "RCLONE_SKIP_LINKS=true"
|
|
out, err = rcloneEnv(env, "lsd", "myAlias:")
|
|
if assert.NoError(t, err) {
|
|
assert.NotContains(t, out, "symlinkA")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Test backend specific flags
|
|
// having effect on the underlying local remote
|
|
env = "RCLONE_LOCAL_SKIP_LINKS=true"
|
|
out, err = rcloneEnv(env, "lsd", "myAlias:")
|
|
if assert.NoError(t, err) {
|
|
assert.NotContains(t, out, "symlinkA")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Test remote specific flags
|
|
// having no effect unless supported by the immediate remote (alias)
|
|
env = "RCLONE_CONFIG_MYALIAS_SKIP_LINKS=true"
|
|
out, err = rcloneEnv(env, "lsd", "myAlias:")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "NOTICE: symlinkA:")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
env = "RCLONE_CONFIG_MYALIAS_REMOTE=" + "myLocal:" + testdataPath + "/folderA"
|
|
out, err = rcloneEnv(env, "lsl", "myAlias:")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "fileA1.txt")
|
|
assert.NotContains(t, out, "fileB1.txt")
|
|
}
|
|
|
|
// Test command line flags
|
|
// having effect on the underlying local remote
|
|
env = ""
|
|
out, err = rcloneEnv(env, "lsd", "myAlias:", "--skip-links")
|
|
if assert.NoError(t, err) {
|
|
assert.NotContains(t, out, "symlinkA")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
// Test connection specific flags
|
|
// having no effect unless supported by the immediate remote (alias)
|
|
env = ""
|
|
out, err = rcloneEnv(env, "lsd", "myAlias,skip_links:")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "NOTICE: symlinkA:")
|
|
assert.Contains(t, out, "folderA")
|
|
}
|
|
|
|
env = ""
|
|
out, err = rcloneEnv(env, "lsl", "myAlias,remote='myLocal:"+testdataPath+"/folderA':", "-vv")
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, "fileA1.txt")
|
|
assert.NotContains(t, out, "fileB1.txt")
|
|
}
|
|
|
|
// Test --use-json-log and -vv combinations
|
|
jsonLogOK := func() {
|
|
t.Helper()
|
|
if assert.NoError(t, err) {
|
|
assert.Contains(t, out, `{"level":"debug",`)
|
|
assert.Contains(t, out, `"msg":"Version `)
|
|
assert.Contains(t, out, `"}`)
|
|
}
|
|
}
|
|
env = "RCLONE_USE_JSON_LOG=1;RCLONE_LOG_LEVEL=DEBUG"
|
|
out, err = rcloneEnv(env, "version")
|
|
jsonLogOK()
|
|
env = "RCLONE_USE_JSON_LOG=1"
|
|
out, err = rcloneEnv(env, "version", "-vv")
|
|
jsonLogOK()
|
|
env = "RCLONE_LOG_LEVEL=DEBUG"
|
|
out, err = rcloneEnv(env, "version", "--use-json-log")
|
|
jsonLogOK()
|
|
env = ""
|
|
out, err = rcloneEnv(env, "version", "-vv", "--use-json-log")
|
|
jsonLogOK()
|
|
|
|
// Find all the File filter lines in out and return them
|
|
parseFileFilters := func(out string) (extensions []string) {
|
|
// Match: - (^|/)[^/]*\.jpg$
|
|
find := regexp.MustCompile(`^- \(\^\|\/\)\[\^\/\]\*\\\.(.*?)\$$`)
|
|
for _, line := range strings.Split(out, "\n") {
|
|
if m := find.FindStringSubmatch(line); m != nil {
|
|
extensions = append(extensions, m[1])
|
|
}
|
|
}
|
|
return extensions
|
|
}
|
|
|
|
// Make sure that multiple valued (stringArray) environment variables are handled properly
|
|
env = ``
|
|
out, err = rcloneEnv(env, "version", "-vv", "--dump", "filters", "--exclude", "*.gif", "--exclude", "*.tif")
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"gif", "tif"}, parseFileFilters(out))
|
|
|
|
env = `RCLONE_EXCLUDE=*.jpg`
|
|
out, err = rcloneEnv(env, "version", "-vv", "--dump", "filters", "--exclude", "*.gif")
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"jpg", "gif"}, parseFileFilters(out))
|
|
|
|
env = `RCLONE_EXCLUDE=*.jpg,*.png`
|
|
out, err = rcloneEnv(env, "version", "-vv", "--dump", "filters", "--exclude", "*.gif", "--exclude", "*.tif")
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"jpg", "png", "gif", "tif"}, parseFileFilters(out))
|
|
|
|
env = `RCLONE_EXCLUDE="*.jpg","*.png"`
|
|
out, err = rcloneEnv(env, "version", "-vv", "--dump", "filters")
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"jpg", "png"}, parseFileFilters(out))
|
|
|
|
env = `RCLONE_EXCLUDE="*.,,,","*.png"`
|
|
out, err = rcloneEnv(env, "version", "-vv", "--dump", "filters")
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{",,,", "png"}, parseFileFilters(out))
|
|
}
|