config: delay load config file (#5258)

Restructuring of config code in v1.55 resulted in config
file being loaded early at process startup. If configuration
file is encrypted this means user will need to supply the password,
even when running commands that does not use config.
This also lead to an issue where mount with --deamon failed to
decrypt the config file when it had to prompt user for passord.

Fixes #5236
Fixes #5228
This commit is contained in:
albertony 2021-04-26 23:37:49 +02:00 committed by GitHub
parent 5d799431a7
commit f8d56bebaf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 117 additions and 101 deletions

View File

@ -20,7 +20,7 @@ var (
)
func prepare(t *testing.T, root string) {
configfile.LoadConfig(context.Background())
configfile.Install()
// Configure the remote
config.FileSet(remoteName, "type", "alias")

View File

@ -47,7 +47,7 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
ts := httptest.NewServer(handler)
// Configure the remote
configfile.LoadConfig(context.Background())
configfile.Install()
// fs.Config.LogLevel = fs.LogLevelDebug
// fs.Config.DumpHeaders = true
// fs.Config.DumpBodies = true

View File

@ -400,7 +400,7 @@ func initConfig() {
configflags.SetFlags(ci)
// Load the config
configfile.LoadConfig(ctx)
configfile.Install()
// Start accounting
accounting.Start(ctx)

View File

@ -21,7 +21,7 @@ import (
func TestRc(t *testing.T) {
ctx := context.Background()
configfile.LoadConfig(ctx)
configfile.Install()
mount := rc.Calls.Get("mount/mount")
assert.NotNil(t, mount)
unmount := rc.Calls.Get("mount/unmount")

View File

@ -41,7 +41,7 @@ func startServer(t *testing.T, f fs.Fs) {
}
func TestInit(t *testing.T) {
configfile.LoadConfig(context.Background())
configfile.Install()
f, err := fs.NewFs(context.Background(), "testdata/files")
l, _ := f.List(context.Background(), "")

View File

@ -61,7 +61,7 @@ var (
func TestInit(t *testing.T) {
ctx := context.Background()
// Configure the remote
configfile.LoadConfig(context.Background())
configfile.Install()
// fs.Config.LogLevel = fs.LogLevelDebug
// fs.Config.DumpHeaders = true
// fs.Config.DumpBodies = true

View File

@ -1,7 +1,6 @@
package restic
import (
"context"
"crypto/rand"
"encoding/hex"
"io"
@ -65,8 +64,7 @@ func createOverwriteDeleteSeq(t testing.TB, path string) []TestRequest {
// TestResticHandler runs tests on the restic handler code, especially in append-only mode.
func TestResticHandler(t *testing.T) {
ctx := context.Background()
configfile.LoadConfig(ctx)
configfile.Install()
buf := make([]byte, 32)
_, err := io.ReadFull(rand.Reader, buf)
require.NoError(t, err)

View File

@ -228,7 +228,7 @@ adding the `--drive-shared-with-me` parameter to the remote `gdrive:`.
rclone lsf "gdrive,shared_with_me:path/to/dir"
The major advantage to using the connection string style syntax is
that it only applies the the remote, not to all the remotes of that
that it only applies to the remote, not to all the remotes of that
type of the command line. A common confusion is this attempt to copy a
file shared on google drive to the normal drive which **does not
work** because the `--drive-shared-with-me` flag applies to both the
@ -1915,17 +1915,21 @@ password prompts. To do that, pass the parameter
of asking for a password if `RCLONE_CONFIG_PASS` doesn't contain
a valid password, and `--password-command` has not been supplied.
Some rclone commands, such as `genautocomplete`, do not require configuration.
Nevertheless, rclone will read any configuration file found
according to the rules described [above](https://rclone.org/docs/#config-config-file).
If an encrypted configuration file is found, this means you will be prompted for
password (unless using `--password-command`). To avoid this, you can bypass
the loading of the default configuration file by overriding the location,
e.g. with one of the documented special values for memory-only configuration:
```
rclone genautocomplete bash --config=""
```
Whenever running commands that may be affected by options in a
configuration file, rclone will look for an existing file according
to the rules described [above](#config-config-file), and load any it
finds. If an encrypted file is found, this includes decrypting it,
with the possible consequence of a password prompt. When executing
a command line that you know are not actually using anything from such
a configuration file, you can avoid it being loaded by overriding the
location, e.g. with one of the documented special values for
memory-only configuration. Since only backend options can be stored
in configuration files, this is normally unnecessary for commands
that do not operate on backends, e.g. `genautocomplete`. However,
it will be relevant for commands that do operate on backends in
general, but are used without referencing a stored remote, e.g.
listing local filesystem paths, or
[connection strings](#connection-strings): `rclone --config="" ls .`
Developer options
-----------------

View File

@ -64,7 +64,7 @@ const (
// load and save to a config file when this is imported
//
// import "github.com/rclone/rclone/fs/config/configfile"
// configfile.LoadConfig(ctx)
// configfile.Install()
type Storage interface {
// GetSectionList returns a slice of strings with names for all the
// sections
@ -101,9 +101,6 @@ type Storage interface {
// Global
var (
// Data is the global config data structure
Data Storage = defaultStorage{}
// CacheDir points to the cache directory. Users of this
// should make a subdirectory and use MkdirAll() to create it
// and any parents.
@ -113,13 +110,18 @@ var (
Password = random.Password
)
var configPath string
var (
configPath string
data Storage
dataLoaded bool
)
func init() {
// Set the function pointers up in fs
fs.ConfigFileGet = FileGetFlag
fs.ConfigFileSet = SetValueAndSave
configPath = makeConfigPath()
data = defaultStorage{}
}
// Join directory with filename, and check if exists
@ -327,23 +329,39 @@ func SetConfigPath(path string) (err error) {
return nil
}
// LoadConfig loads the config file
func LoadConfig(ctx context.Context) {
// Set RCLONE_CONFIG_DIR for backend config and subprocesses
// If empty configPath (in-memory only) the value will be "."
_ = os.Setenv("RCLONE_CONFIG_DIR", filepath.Dir(configPath))
// Load configuration from file (or initialize sensible default if no file or error)
if err := Data.Load(); err == ErrorConfigFileNotFound {
if configPath == "" {
fs.Debugf(nil, "Config is memory-only - using defaults")
// SetData sets new config file storage
func SetData(newData Storage) {
data = newData
dataLoaded = false
}
// Data returns current config file storage
func Data() Storage {
return data
}
// LoadedData ensures the config file storage is loaded and returns it
func LoadedData() Storage {
if !dataLoaded {
// Set RCLONE_CONFIG_DIR for backend config and subprocesses
// If empty configPath (in-memory only) the value will be "."
_ = os.Setenv("RCLONE_CONFIG_DIR", filepath.Dir(configPath))
// Load configuration from file (or initialize sensible default if no file or error)
if err := data.Load(); err == nil {
fs.Debugf(nil, "Using config file from %q", configPath)
dataLoaded = true
} else if err == ErrorConfigFileNotFound {
if configPath == "" {
fs.Debugf(nil, "Config is memory-only - using defaults")
} else {
fs.Logf(nil, "Config file %q not found - using defaults", configPath)
}
dataLoaded = true
} else {
fs.Logf(nil, "Config file %q not found - using defaults", configPath)
log.Fatalf("Failed to load config file %q: %v", configPath, err)
}
} else if err != nil {
log.Fatalf("Failed to load config file %q: %v", configPath, err)
} else {
fs.Debugf(nil, "Using config file from %q", configPath)
}
return data
}
// ErrorConfigFileNotFound is returned when the config file is not found
@ -360,7 +378,7 @@ func SaveConfig() {
ci := fs.GetConfig(ctx)
var err error
for i := 0; i < ci.LowLevelRetries+1; i++ {
if err = Data.Save(); err == nil {
if err = LoadedData().Save(); err == nil {
return
}
waitingTimeMs := mathrand.Intn(1000)
@ -374,7 +392,7 @@ func SaveConfig() {
// disk first and overwrites the given value only.
func SetValueAndSave(name, key, value string) error {
// Set the value in config in case we fail to reload it
Data.SetValue(name, key, value)
LoadedData().SetValue(name, key, value)
// Save it again
SaveConfig()
return nil
@ -383,7 +401,7 @@ func SetValueAndSave(name, key, value string) error {
// getWithDefault gets key out of section name returning defaultValue if not
// found.
func getWithDefault(name, key, defaultValue string) string {
value, found := Data.GetValue(name, key)
value, found := LoadedData().GetValue(name, key)
if !found {
return defaultValue
}
@ -435,7 +453,7 @@ func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, doObscu
}
}
}
Data.SetValue(name, k, vStr)
LoadedData().SetValue(name, k, vStr)
}
RemoteConfig(ctx, name)
SaveConfig()
@ -452,9 +470,9 @@ func CreateRemote(ctx context.Context, name string, provider string, keyValues r
return err
}
// Delete the old config if it exists
Data.DeleteSection(name)
LoadedData().DeleteSection(name)
// Set the type
Data.SetValue(name, "type", provider)
LoadedData().SetValue(name, "type", provider)
// Set the remaining values
return UpdateRemote(ctx, name, keyValues, doObscure, noObscure)
}
@ -507,7 +525,7 @@ func fsOption() *fs.Option {
// FileGetFlag gets the config key under section returning the
// the value and true if found and or ("", false) otherwise
func FileGetFlag(section, key string) (string, bool) {
return Data.GetValue(section, key)
return LoadedData().GetValue(section, key)
}
// FileGet gets the config key under section returning the default if not set.
@ -527,7 +545,7 @@ func FileGet(section, key string) string {
// the config file.
func FileSet(section, key, value string) {
if value != "" {
Data.SetValue(section, key, value)
LoadedData().SetValue(section, key, value)
} else {
FileDeleteKey(section, key)
}
@ -537,7 +555,7 @@ func FileSet(section, key, value string) {
// It returns true if the key was deleted,
// or returns false if the section or key didn't exist.
func FileDeleteKey(section, key string) bool {
return Data.DeleteKey(section, key)
return LoadedData().DeleteKey(section, key)
}
var matchEnv = regexp.MustCompile(`^RCLONE_CONFIG_(.*?)_TYPE=.*$`)
@ -545,7 +563,7 @@ var matchEnv = regexp.MustCompile(`^RCLONE_CONFIG_(.*?)_TYPE=.*$`)
// FileSections returns the sections in the config file
// including any defined by environment variables.
func FileSections() []string {
sections := Data.GetSectionList()
sections := LoadedData().GetSectionList()
for _, item := range os.Environ() {
matches := matchEnv.FindStringSubmatch(item)
if len(matches) == 2 {
@ -558,7 +576,7 @@ func FileSections() []string {
// DumpRcRemote dumps the config for a single remote
func DumpRcRemote(name string) (dump rc.Params) {
params := rc.Params{}
for _, key := range Data.GetKeyList(name) {
for _, key := range LoadedData().GetKeyList(name) {
params[key] = FileGet(name, key)
}
return params
@ -568,7 +586,7 @@ func DumpRcRemote(name string) (dump rc.Params) {
// for the rc
func DumpRcBlob() (dump rc.Params) {
dump = rc.Params{}
for _, name := range Data.GetSectionList() {
for _, name := range LoadedData().GetSectionList() {
dump[name] = DumpRcRemote(name)
}
return dump

View File

@ -3,7 +3,6 @@
package config_test
import (
"context"
"testing"
"github.com/rclone/rclone/fs/config"
@ -18,12 +17,12 @@ func TestConfigLoad(t *testing.T) {
assert.NoError(t, config.SetConfigPath(oldConfigPath))
}()
config.ClearConfigPassword()
configfile.LoadConfig(context.Background())
sections := config.Data.GetSectionList()
configfile.Install()
sections := config.Data().GetSectionList()
var expect = []string{"RCLONE_ENCRYPT_V0", "nounc", "unc"}
assert.Equal(t, expect, sections)
keys := config.Data.GetKeyList("nounc")
keys := config.Data().GetKeyList("nounc")
expect = []string{"type", "nounc"}
assert.Equal(t, expect, keys)
}

View File

@ -3,7 +3,6 @@ package configfile
import (
"bytes"
"context"
"io/ioutil"
"os"
"path/filepath"
@ -15,10 +14,9 @@ import (
"github.com/rclone/rclone/fs/config"
)
// LoadConfig installs the config file handler and calls config.LoadConfig
func LoadConfig(ctx context.Context) {
config.Data = &Storage{}
config.LoadConfig(ctx)
// Install installs the config file handler
func Install() {
config.SetData(&Storage{})
}
// Storage implements config.Storage for saving and loading config

View File

@ -26,13 +26,13 @@ func TestConfigLoadEncrypted(t *testing.T) {
// Set correct password
err = config.SetConfigPassword("asdf")
require.NoError(t, err)
err = config.Data.Load()
err = config.Data().Load()
require.NoError(t, err)
sections := config.Data.GetSectionList()
sections := config.Data().GetSectionList()
var expect = []string{"nounc", "unc"}
assert.Equal(t, expect, sections)
keys := config.Data.GetKeyList("nounc")
keys := config.Data().GetKeyList("nounc")
expect = []string{"type", "nounc"}
assert.Equal(t, expect, keys)
}
@ -54,14 +54,14 @@ func TestConfigLoadEncryptedWithValidPassCommand(t *testing.T) {
config.ClearConfigPassword()
err := config.Data.Load()
err := config.Data().Load()
require.NoError(t, err)
sections := config.Data.GetSectionList()
sections := config.Data().GetSectionList()
var expect = []string{"nounc", "unc"}
assert.Equal(t, expect, sections)
keys := config.Data.GetKeyList("nounc")
keys := config.Data().GetKeyList("nounc")
expect = []string{"type", "nounc"}
assert.Equal(t, expect, keys)
}
@ -83,7 +83,7 @@ func TestConfigLoadEncryptedWithInvalidPassCommand(t *testing.T) {
config.ClearConfigPassword()
err := config.Data.Load()
err := config.Data().Load()
require.Error(t, err)
assert.Contains(t, err.Error(), "using --password-command derived password")
}
@ -95,21 +95,21 @@ func TestConfigLoadEncryptedFailures(t *testing.T) {
oldConfigPath := config.GetConfigPath()
assert.NoError(t, config.SetConfigPath("./testdata/enc-short.conf"))
defer func() { assert.NoError(t, config.SetConfigPath(oldConfigPath)) }()
err = config.Data.Load()
err = config.Data().Load()
require.Error(t, err)
// This file contains invalid base64 characters.
assert.NoError(t, config.SetConfigPath("./testdata/enc-invalid.conf"))
err = config.Data.Load()
err = config.Data().Load()
require.Error(t, err)
// This file contains invalid base64 characters.
assert.NoError(t, config.SetConfigPath("./testdata/enc-too-new.conf"))
err = config.Data.Load()
err = config.Data().Load()
require.Error(t, err)
// This file does not exist.
assert.NoError(t, config.SetConfigPath("./testdata/filenotfound.conf"))
err = config.Data.Load()
err = config.Data().Load()
assert.Equal(t, config.ErrorConfigFileNotFound, err)
}

View File

@ -3,7 +3,7 @@ package config
// Default config.Storage which panics with a useful error when used
type defaultStorage struct{}
var noConfigStorage = "internal error: no config file system found. Did you call configfile.LoadConfig(ctx)?"
var noConfigStorage = "internal error: no config file system found. Did you call configfile.Install()?"
// GetSectionList returns a slice of strings with names for all the
// sections

View File

@ -72,7 +72,7 @@ See the [listremotes command](/commands/rclone_listremotes/) command for more in
// Return the a list of remotes in the config file
func rcListRemotes(ctx context.Context, in rc.Params) (out rc.Params, err error) {
var remotes = []string{}
for _, remote := range Data.GetSectionList() {
for _, remote := range LoadedData().GetSectionList() {
remotes = append(remotes, remote)
}
out = rc.Params{

View File

@ -18,7 +18,7 @@ const testName = "configTestNameForRc"
func TestRc(t *testing.T) {
ctx := context.Background()
configfile.LoadConfig(ctx)
configfile.Install()
// Create the test remote
call := rc.Calls.Get("config/create")
assert.NotNil(t, call)

View File

@ -200,7 +200,7 @@ func ChooseNumber(what string, min, max int) int {
// ShowRemotes shows an overview of the config file
func ShowRemotes() {
remotes := Data.GetSectionList()
remotes := LoadedData().GetSectionList()
if len(remotes) == 0 {
return
}
@ -214,7 +214,7 @@ func ShowRemotes() {
// ChooseRemote chooses a remote name
func ChooseRemote() string {
remotes := Data.GetSectionList()
remotes := LoadedData().GetSectionList()
sort.Strings(remotes)
return Choose("remote", remotes, nil, false)
}
@ -234,7 +234,7 @@ func ShowRemote(name string) {
fmt.Printf("--------------------\n")
fmt.Printf("[%s]\n", name)
fs := mustFindByName(name)
for _, key := range Data.GetKeyList(name) {
for _, key := range LoadedData().GetKeyList(name) {
isPassword := false
for _, option := range fs.Options {
if option.Name == key && option.IsPassword {
@ -261,7 +261,7 @@ func OkRemote(name string) bool {
case 'e':
return false
case 'd':
Data.DeleteSection(name)
LoadedData().DeleteSection(name)
return true
default:
fs.Errorf(nil, "Bad choice %c", i)
@ -401,7 +401,7 @@ func NewRemoteName() (name string) {
for {
fmt.Printf("name> ")
name = ReadLine()
if Data.HasSection(name) {
if LoadedData().HasSection(name) {
fmt.Printf("Remote %q already exists.\n", name)
continue
}
@ -473,7 +473,7 @@ func NewRemote(ctx context.Context, name string) {
}
break
}
Data.SetValue(name, "type", newType)
LoadedData().SetValue(name, "type", newType)
editOptions(ri, name, true)
RemoteConfig(ctx, name)
@ -500,7 +500,7 @@ func EditRemote(ctx context.Context, ri *fs.RegInfo, name string) {
// DeleteRemote gets the user to delete a remote
func DeleteRemote(name string) {
Data.DeleteSection(name)
LoadedData().DeleteSection(name)
SaveConfig()
}
@ -509,9 +509,9 @@ func DeleteRemote(name string) {
func copyRemote(name string) string {
newName := NewRemoteName()
// Copy the keys
for _, key := range Data.GetKeyList(name) {
for _, key := range LoadedData().GetKeyList(name) {
value := getWithDefault(name, key, "")
Data.SetValue(newName, key, value)
LoadedData().SetValue(newName, key, value)
}
return newName
}
@ -521,7 +521,7 @@ func RenameRemote(name string) {
fmt.Printf("Enter new name for %q remote.\n", name)
newName := copyRemote(name)
if name != newName {
Data.DeleteSection(name)
LoadedData().DeleteSection(name)
SaveConfig()
}
}
@ -549,7 +549,7 @@ func ShowConfigLocation() {
// ShowConfig prints the (unencrypted) config options
func ShowConfig() {
str, err := Data.Serialize()
str, err := LoadedData().Serialize()
if err != nil {
log.Fatalf("Failed to serialize config: %v", err)
}
@ -562,7 +562,7 @@ func ShowConfig() {
// EditConfig edits the config file interactively
func EditConfig(ctx context.Context) {
for {
haveRemotes := len(Data.GetSectionList()) != 0
haveRemotes := len(LoadedData().GetSectionList()) != 0
what := []string{"eEdit existing remote", "nNew remote", "dDelete remote", "rRename remote", "cCopy remote", "sSet configuration password", "qQuit config"}
if haveRemotes {
fmt.Printf("Current remotes:\n\n")

View File

@ -36,15 +36,15 @@ func testConfigFile(t *testing.T, configFileName string) func() {
oldOsStdout := os.Stdout
oldConfigPath := config.GetConfigPath()
oldConfig := *ci
oldConfigFile := config.Data
oldConfigFile := config.Data()
oldReadLine := config.ReadLine
oldPassword := config.Password
os.Stdout = nil
assert.NoError(t, config.SetConfigPath(path))
ci = &fs.ConfigInfo{}
configfile.LoadConfig(ctx)
assert.Equal(t, []string{}, config.Data.GetSectionList())
configfile.Install()
assert.Equal(t, []string{}, config.Data().GetSectionList())
// Fake a remote
fs.Register(&fs.RegInfo{
@ -73,7 +73,7 @@ func testConfigFile(t *testing.T, configFileName string) func() {
config.ReadLine = oldReadLine
config.Password = oldPassword
*ci = oldConfig
config.Data = oldConfigFile
config.SetData(oldConfigFile)
_ = os.Unsetenv("_RCLONE_CONFIG_KEY_FILE")
_ = os.Unsetenv("RCLONE_CONFIG_PASS")
@ -105,7 +105,7 @@ func TestCRUD(t *testing.T) {
})
config.NewRemote(ctx, "test")
assert.Equal(t, []string{"test"}, config.Data.GetSectionList())
assert.Equal(t, []string{"test"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test", "type"))
assert.Equal(t, "true", config.FileGet("test", "bool"))
assert.Equal(t, "secret", obscure.MustReveal(config.FileGet("test", "pass")))
@ -118,14 +118,14 @@ func TestCRUD(t *testing.T) {
})
config.RenameRemote("test")
assert.Equal(t, []string{"asdf"}, config.Data.GetSectionList())
assert.Equal(t, []string{"asdf"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("asdf", "type"))
assert.Equal(t, "true", config.FileGet("asdf", "bool"))
assert.Equal(t, "secret", obscure.MustReveal(config.FileGet("asdf", "pass")))
// delete remote
config.DeleteRemote("asdf")
assert.Equal(t, []string{}, config.Data.GetSectionList())
assert.Equal(t, []string{}, config.Data().GetSectionList())
}
func TestChooseOption(t *testing.T) {
@ -202,7 +202,7 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
"pass": "potato",
}, doObscure, noObscure))
assert.Equal(t, []string{"test2"}, config.Data.GetSectionList())
assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))
assert.Equal(t, "true", config.FileGet("test2", "bool"))
gotPw := config.FileGet("test2", "pass")
@ -218,7 +218,7 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
"spare": "spare",
}, doObscure, noObscure))
assert.Equal(t, []string{"test2"}, config.Data.GetSectionList())
assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))
assert.Equal(t, "false", config.FileGet("test2", "bool"))
gotPw = config.FileGet("test2", "pass")
@ -231,7 +231,7 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
"pass": "potato3",
}))
assert.Equal(t, []string{"test2"}, config.Data.GetSectionList())
assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))
assert.Equal(t, "false", config.FileGet("test2", "bool"))
assert.Equal(t, "potato3", obscure.MustReveal(config.FileGet("test2", "pass")))

View File

@ -103,7 +103,7 @@ type testRun struct {
// Run a suite of tests
func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
ctx := context.Background()
configfile.LoadConfig(ctx)
configfile.Install()
mux := http.NewServeMux()
opt.HTTPOptions.Template = testTemplate
rcServer := newServer(ctx, opt, mux)

View File

@ -71,7 +71,7 @@ func Initialise() {
if envConfig := os.Getenv("RCLONE_CONFIG"); envConfig != "" {
_ = config.SetConfigPath(envConfig)
}
configfile.LoadConfig(ctx)
configfile.Install()
accounting.Start(ctx)
if *Verbose {
ci.LogLevel = fs.LogLevelDebug

View File

@ -12,7 +12,6 @@ Make TesTrun have a []string of flags to try - that then makes it generic
*/
import (
"context"
"flag"
"log"
"math/rand"
@ -72,7 +71,7 @@ func main() {
log.Println("test_all should be run from the root of the rclone source code")
log.Fatal(err)
}
configfile.LoadConfig(context.Background())
configfile.Install()
// Seed the random number generator
rand.Seed(time.Now().UTC().UnixNano())