config: make getting config values more consistent

This commit is contained in:
albertony 2023-11-04 15:34:23 +01:00 committed by Nick Craig-Wood
parent e4749cf0d0
commit d6b0743cf4
7 changed files with 164 additions and 176 deletions

View File

@ -23,8 +23,8 @@ func prepare(t *testing.T, root string) {
configfile.Install()
// Configure the remote
config.FileSet(remoteName, "type", "alias")
config.FileSet(remoteName, "remote", root)
config.FileSetValue(remoteName, "type", "alias")
config.FileSetValue(remoteName, "remote", root)
}
func TestNewFS(t *testing.T) {

View File

@ -875,12 +875,12 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
cacheRemote := remote
if !remoteExists {
localRemote := remote + "-local"
config.FileSet(localRemote, "type", "local")
config.FileSet(localRemote, "nounc", "true")
config.FileSetValue(localRemote, "type", "local")
config.FileSetValue(localRemote, "nounc", "true")
m.Set("type", "cache")
m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
} else {
remoteType := config.FileGet(remote, "type")
remoteType := config.GetValue(remote, "type")
if remoteType == "" {
t.Skipf("skipped due to invalid remote type for %v", remote)
return nil, nil
@ -891,14 +891,14 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
m.Set("password", cryptPassword1)
m.Set("password2", cryptPassword2)
}
remoteRemote := config.FileGet(remote, "remote")
remoteRemote := config.GetValue(remote, "remote")
if remoteRemote == "" {
t.Skipf("skipped due to invalid remote wrapper for %v", remote)
return nil, nil
}
remoteRemoteParts := strings.Split(remoteRemote, ":")
remoteWrapping := remoteRemoteParts[0]
remoteType := config.FileGet(remoteWrapping, "type")
remoteType := config.GetValue(remoteWrapping, "type")
if remoteType != "cache" {
t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
return nil, nil

View File

@ -121,7 +121,7 @@ var (
func init() {
// Set the function pointers up in fs
fs.ConfigFileGet = FileGetFlag
fs.ConfigFileGet = FileGetValue
fs.ConfigFileSet = SetValueAndSave
fs.ConfigFileHasSection = func(section string) bool {
return LoadedData().HasSection(section)
@ -351,6 +351,9 @@ func Data() Storage {
return data
}
// ErrorConfigFileNotFound is returned when the config file is not found
var ErrorConfigFileNotFound = errors.New("config file not found")
// LoadedData ensures the config file storage is loaded and returns it
func LoadedData() Storage {
if !dataLoaded {
@ -375,9 +378,6 @@ func LoadedData() Storage {
return data
}
// ErrorConfigFileNotFound is returned when the config file is not found
var ErrorConfigFileNotFound = errors.New("config file not found")
// SaveConfig calling function which saves configuration file.
// if SaveConfig returns error trying again after sleep.
func SaveConfig() {
@ -394,25 +394,103 @@ func SaveConfig() {
fs.Errorf(nil, "Failed to save config after %d tries: %v", ci.LowLevelRetries, err)
}
// FileSections returns the sections in the config file
func FileSections() []string {
return LoadedData().GetSectionList()
}
// FileGetValue gets the config key under section returning the
// the value and true if found and or ("", false) otherwise
func FileGetValue(section, key string) (string, bool) {
return LoadedData().GetValue(section, key)
}
// FileSetValue sets the key in section to value.
// It doesn't save the config file.
func FileSetValue(section, key, value string) {
LoadedData().SetValue(section, key, value)
}
// FileDeleteKey deletes the config key in the config file.
// It returns true if the key was deleted,
// or returns false if the section or key didn't exist.
func FileDeleteKey(section, key string) bool {
return LoadedData().DeleteKey(section, key)
}
// GetValue gets the value for a config key from environment
// or config file under section returning the default if not set.
//
// Emulates the preference documented and normally used by rclone via
// configmap, which means environment variables before config file.
func GetValue(remote, key string) string {
envKey := fs.ConfigToEnv(remote, key)
value, found := os.LookupEnv(envKey)
if found {
return value
}
value, _ = LoadedData().GetValue(remote, key)
return value
}
// SetValueAndSave sets the key to the value and saves just that
// value in the config file. It loads the old config file in from
// disk first and overwrites the given value only.
func SetValueAndSave(name, key, value string) error {
func SetValueAndSave(remote, key, value string) error {
// Set the value in config in case we fail to reload it
LoadedData().SetValue(name, key, value)
FileSetValue(remote, key, value)
// Save it again
SaveConfig()
return nil
}
// getWithDefault gets key out of section name returning defaultValue if not
// found.
func getWithDefault(name, key, defaultValue string) string {
value, found := LoadedData().GetValue(name, key)
if !found {
return defaultValue
// Remote defines a remote with a name, type and source
type Remote struct {
Name string `json:"name"`
Type string `json:"type"`
Source string `json:"source"`
}
return value
var remoteEnvRe = regexp.MustCompile(`^RCLONE_CONFIG_(.+?)_TYPE=(.+)$`)
// GetRemotes returns the list of remotes defined in environment and config file.
//
// Emulates the preference documented and normally used by rclone via
// configmap, which means environment variables before config file.
func GetRemotes() []Remote {
var remotes []Remote
for _, item := range os.Environ() {
matches := remoteEnvRe.FindStringSubmatch(item)
if len(matches) == 3 {
remotes = append(remotes, Remote{
Name: strings.ToLower(matches[1]),
Type: strings.ToLower(matches[2]),
Source: "environment",
})
}
}
remoteExists := func(name string) bool {
for _, remote := range remotes {
if name == remote.Name {
return true
}
}
return false
}
sections := LoadedData().GetSectionList()
for _, section := range sections {
if !remoteExists(section) {
typeValue, found := LoadedData().GetValue(section, "type")
if found {
remotes = append(remotes, Remote{
Name: section,
Type: typeValue,
Source: "file",
})
}
}
}
return remotes
}
// GetRemoteNames returns the names of remotes defined in environment and config file.
@ -459,7 +537,7 @@ func updateRemote(ctx context.Context, name string, keyValues rc.Params, opt Upd
ctx = suppressConfirm(ctx)
}
fsType := FileGet(name, "type")
fsType := GetValue(name, "type")
if fsType == "" {
return nil, errors.New("couldn't find type field in config")
}
@ -611,101 +689,11 @@ func fsOption() *fs.Option {
return o
}
// FileGetFlag gets the config key under section returning the
// the value and true if found and or ("", false) otherwise
func FileGetFlag(section, key string) (string, bool) {
return LoadedData().GetValue(section, key)
}
// FileGet gets the config key under section returning the default if not set.
//
// It looks up defaults in the environment if they are present
func FileGet(section, key string) string {
var defaultVal string
envKey := fs.ConfigToEnv(section, key)
newValue, found := os.LookupEnv(envKey)
if found {
defaultVal = newValue
}
return getWithDefault(section, key, defaultVal)
}
// FileSet sets the key in section to value. It doesn't save
// the config file.
func FileSet(section, key, value string) {
if value != "" {
LoadedData().SetValue(section, key, value)
} else {
FileDeleteKey(section, key)
}
}
// FileDeleteKey deletes the config key in the config file.
// It returns true if the key was deleted,
// or returns false if the section or key didn't exist.
func FileDeleteKey(section, key string) bool {
return LoadedData().DeleteKey(section, key)
}
// FileSections returns the sections in the config file
func FileSections() []string {
return LoadedData().GetSectionList()
}
// Remote defines a remote with a name, type and source
type Remote struct {
Name string `json:"name"`
Type string `json:"type"`
Source string `json:"source"`
}
var remoteEnvRe = regexp.MustCompile(`^RCLONE_CONFIG_(.+?)_TYPE=(.+)$`)
// GetRemotes returns the list of remotes defined in environment and config file.
//
// Emulates the preference documented and normally used by rclone via configmap,
// which means environment variables before config file.
func GetRemotes() []Remote {
var remotes []Remote
for _, item := range os.Environ() {
matches := remoteEnvRe.FindStringSubmatch(item)
if len(matches) == 3 {
remotes = append(remotes, Remote{
Name: strings.ToLower(matches[1]),
Type: strings.ToLower(matches[2]),
Source: "environment",
})
}
}
remoteExists := func(name string) bool {
for _, remote := range remotes {
if name == remote.Name {
return true
}
}
return false
}
sections := LoadedData().GetSectionList()
for _, section := range sections {
if !remoteExists(section) {
typeValue, found := LoadedData().GetValue(section, "type")
if found {
remotes = append(remotes, Remote{
Name: section,
Type: typeValue,
Source: "file",
})
}
}
}
return remotes
}
// DumpRcRemote dumps the config for a single remote
func DumpRcRemote(name string) (dump rc.Params) {
params := rc.Params{}
for _, key := range LoadedData().GetKeyList(name) {
params[key] = FileGet(name, key)
params[key] = GetValue(name, key)
}
return params
}

View File

@ -33,8 +33,8 @@ func TestRc(t *testing.T) {
out, err := call.Fn(ctx, in)
require.NoError(t, err)
require.Nil(t, out)
assert.Equal(t, "local", config.FileGet(testName, "type"))
assert.Equal(t, "sausage", config.FileGet(testName, "test_key"))
assert.Equal(t, "local", config.GetValue(testName, "type"))
assert.Equal(t, "sausage", config.GetValue(testName, "test_key"))
// The sub tests rely on the remote created above but they can
// all be run independently
@ -102,9 +102,9 @@ func TestRc(t *testing.T) {
require.NoError(t, err)
assert.Nil(t, out)
assert.Equal(t, "local", config.FileGet(testName, "type"))
assert.Equal(t, "rutabaga", config.FileGet(testName, "test_key"))
assert.Equal(t, "cabbage", config.FileGet(testName, "test_key2"))
assert.Equal(t, "local", config.GetValue(testName, "type"))
assert.Equal(t, "rutabaga", config.GetValue(testName, "test_key"))
assert.Equal(t, "cabbage", config.GetValue(testName, "test_key2"))
})
t.Run("Password", func(t *testing.T) {
@ -122,9 +122,9 @@ func TestRc(t *testing.T) {
require.NoError(t, err)
assert.Nil(t, out)
assert.Equal(t, "local", config.FileGet(testName, "type"))
assert.Equal(t, "rutabaga", obscure.MustReveal(config.FileGet(testName, "test_key")))
assert.Equal(t, pw2, obscure.MustReveal(config.FileGet(testName, "test_key2")))
assert.Equal(t, "local", config.GetValue(testName, "type"))
assert.Equal(t, "rutabaga", obscure.MustReveal(config.GetValue(testName, "test_key")))
assert.Equal(t, pw2, obscure.MustReveal(config.GetValue(testName, "test_key2")))
})
// Delete the test remote
@ -136,8 +136,8 @@ func TestRc(t *testing.T) {
out, err = call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Nil(t, out)
assert.Equal(t, "", config.FileGet(testName, "type"))
assert.Equal(t, "", config.FileGet(testName, "test_key"))
assert.Equal(t, "", config.GetValue(testName, "type"))
assert.Equal(t, "", config.GetValue(testName, "test_key"))
}
func TestRcProviders(t *testing.T) {

View File

@ -280,7 +280,7 @@ func ShowRemotes() {
fmt.Printf("%-20s %s\n", "Name", "Type")
fmt.Printf("%-20s %s\n", "====", "====")
for _, remote := range remotes {
fmt.Printf("%-20s %s\n", remote, FileGet(remote, "type"))
fmt.Printf("%-20s %s\n", remote, GetValue(remote, "type"))
}
}
@ -295,7 +295,7 @@ func ChooseRemote() string {
// mustFindByName finds the RegInfo for the remote name passed in or
// exits with a fatal error.
func mustFindByName(name string) *fs.RegInfo {
fsType := FileGet(name, "type")
fsType := GetValue(name, "type")
if fsType == "" {
log.Fatalf("Couldn't find type of fs for %q", name)
}
@ -305,7 +305,7 @@ func mustFindByName(name string) *fs.RegInfo {
// findByName finds the RegInfo for the remote name passed in or
// returns an error
func findByName(name string) (*fs.RegInfo, error) {
fsType := FileGet(name, "type")
fsType := GetValue(name, "type")
if fsType == "" {
return nil, fmt.Errorf("couldn't find type of fs for %q", name)
}
@ -333,7 +333,7 @@ func printRemoteOptions(name string, prefix string, sep string, redacted bool) {
}
}
}
value := FileGet(name, key)
value := GetValue(name, key)
if redacted && (isSensitive || isPassword) && value != "" {
fmt.Printf("%s%s%sXXX\n", prefix, key, sep)
} else if isPassword && value != "" {
@ -613,7 +613,7 @@ func copyRemote(name string) string {
newName := NewRemoteName()
// Copy the keys
for _, key := range LoadedData().GetKeyList(name) {
value := getWithDefault(name, key, "")
value, _ := FileGetValue(name, key)
LoadedData().SetValue(newName, key, value)
}
return newName

View File

@ -110,9 +110,9 @@ func TestCRUD(t *testing.T) {
require.NoError(t, config.NewRemote(ctx, "test"))
assert.Equal(t, []string{"test"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test", "type"))
assert.Equal(t, "true", config.FileGet("test", "bool"))
assert.Equal(t, "secret", obscure.MustReveal(config.FileGet("test", "pass")))
assert.Equal(t, "config_test_remote", config.GetValue("test", "type"))
assert.Equal(t, "true", config.GetValue("test", "bool"))
assert.Equal(t, "secret", obscure.MustReveal(config.GetValue("test", "pass")))
// normal rename, test → asdf
config.ReadLine = makeReadLine([]string{
@ -123,9 +123,9 @@ func TestCRUD(t *testing.T) {
config.RenameRemote("test")
assert.Equal(t, []string{"asdf"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("asdf", "type"))
assert.Equal(t, "true", config.FileGet("asdf", "bool"))
assert.Equal(t, "secret", obscure.MustReveal(config.FileGet("asdf", "pass")))
assert.Equal(t, "config_test_remote", config.GetValue("asdf", "type"))
assert.Equal(t, "true", config.GetValue("asdf", "bool"))
assert.Equal(t, "secret", obscure.MustReveal(config.GetValue("asdf", "pass")))
// delete remote
config.DeleteRemote("asdf")
@ -152,8 +152,8 @@ func TestChooseOption(t *testing.T) {
}
require.NoError(t, config.NewRemote(ctx, "test"))
assert.Equal(t, "", config.FileGet("test", "bool")) // this is the default now
assert.Equal(t, "not very random password", obscure.MustReveal(config.FileGet("test", "pass")))
assert.Equal(t, "", config.GetValue("test", "bool")) // this is the default now
assert.Equal(t, "not very random password", obscure.MustReveal(config.GetValue("test", "pass")))
// script for creating remote
config.ReadLine = makeReadLine([]string{
@ -164,8 +164,8 @@ func TestChooseOption(t *testing.T) {
})
require.NoError(t, config.NewRemote(ctx, "test"))
assert.Equal(t, "true", config.FileGet("test", "bool"))
assert.Equal(t, "", config.FileGet("test", "pass"))
assert.Equal(t, "true", config.GetValue("test", "bool"))
assert.Equal(t, "", config.GetValue("test", "pass"))
}
func TestNewRemoteName(t *testing.T) {
@ -212,9 +212,9 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))
assert.Equal(t, "true", config.FileGet("test2", "bool"))
gotPw := config.FileGet("test2", "pass")
assert.Equal(t, "config_test_remote", config.GetValue("test2", "type"))
assert.Equal(t, "true", config.GetValue("test2", "bool"))
gotPw := config.GetValue("test2", "pass")
if !noObscure {
gotPw = obscure.MustReveal(gotPw)
}
@ -229,9 +229,9 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))
assert.Equal(t, "false", config.FileGet("test2", "bool"))
gotPw = config.FileGet("test2", "pass")
assert.Equal(t, "config_test_remote", config.GetValue("test2", "type"))
assert.Equal(t, "false", config.GetValue("test2", "bool"))
gotPw = config.GetValue("test2", "pass")
if doObscure {
gotPw = obscure.MustReveal(gotPw)
}
@ -242,9 +242,9 @@ func TestCreateUpdatePasswordRemote(t *testing.T) {
}))
assert.Equal(t, []string{"test2"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test2", "type"))
assert.Equal(t, "false", config.FileGet("test2", "bool"))
assert.Equal(t, "potato3", obscure.MustReveal(config.FileGet("test2", "pass")))
assert.Equal(t, "config_test_remote", config.GetValue("test2", "type"))
assert.Equal(t, "false", config.GetValue("test2", "bool"))
assert.Equal(t, "potato3", obscure.MustReveal(config.GetValue("test2", "pass")))
})
}
}
@ -280,10 +280,10 @@ func TestDefaultRequired(t *testing.T) {
require.NoError(t, config.NewRemote(ctx, "test"))
assert.Equal(t, []string{"test"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test", "type"))
assert.Equal(t, "111", config.FileGet("test", "string_required"))
assert.Equal(t, "222", config.FileGet("test", "string_default"))
assert.Equal(t, "333", config.FileGet("test", "string_required_default"))
assert.Equal(t, "config_test_remote", config.GetValue("test", "type"))
assert.Equal(t, "111", config.GetValue("test", "string_required"))
assert.Equal(t, "222", config.GetValue("test", "string_default"))
assert.Equal(t, "333", config.GetValue("test", "string_required_default"))
// delete remote
config.DeleteRemote("test")
@ -301,10 +301,10 @@ func TestDefaultRequired(t *testing.T) {
require.NoError(t, config.NewRemote(ctx, "test"))
assert.Equal(t, []string{"test"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test", "type"))
assert.Equal(t, "111", config.FileGet("test", "string_required"))
assert.Equal(t, "", config.FileGet("test", "string_default"))
assert.Equal(t, "", config.FileGet("test", "string_required_default"))
assert.Equal(t, "config_test_remote", config.GetValue("test", "type"))
assert.Equal(t, "111", config.GetValue("test", "string_required"))
assert.Equal(t, "", config.GetValue("test", "string_default"))
assert.Equal(t, "", config.GetValue("test", "string_required_default"))
}
func TestMultipleChoice(t *testing.T) {
@ -383,11 +383,11 @@ func TestMultipleChoice(t *testing.T) {
require.NoError(t, config.NewRemote(ctx, "test"))
assert.Equal(t, []string{"test"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test", "type"))
assert.Equal(t, "CCC", config.FileGet("test", "multiple_choice"))
assert.Equal(t, "CCC", config.FileGet("test", "multiple_choice_required"))
assert.Equal(t, "CCC", config.FileGet("test", "multiple_choice_default"))
assert.Equal(t, "CCC", config.FileGet("test", "multiple_choice_required_default"))
assert.Equal(t, "config_test_remote", config.GetValue("test", "type"))
assert.Equal(t, "CCC", config.GetValue("test", "multiple_choice"))
assert.Equal(t, "CCC", config.GetValue("test", "multiple_choice_required"))
assert.Equal(t, "CCC", config.GetValue("test", "multiple_choice_default"))
assert.Equal(t, "CCC", config.GetValue("test", "multiple_choice_required_default"))
// delete remote
config.DeleteRemote("test")
@ -405,11 +405,11 @@ func TestMultipleChoice(t *testing.T) {
require.NoError(t, config.NewRemote(ctx, "test"))
assert.Equal(t, []string{"test"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test", "type"))
assert.Equal(t, "XXX", config.FileGet("test", "multiple_choice"))
assert.Equal(t, "XXX", config.FileGet("test", "multiple_choice_required"))
assert.Equal(t, "XXX", config.FileGet("test", "multiple_choice_default"))
assert.Equal(t, "XXX", config.FileGet("test", "multiple_choice_required_default"))
assert.Equal(t, "config_test_remote", config.GetValue("test", "type"))
assert.Equal(t, "XXX", config.GetValue("test", "multiple_choice"))
assert.Equal(t, "XXX", config.GetValue("test", "multiple_choice_required"))
assert.Equal(t, "XXX", config.GetValue("test", "multiple_choice_default"))
assert.Equal(t, "XXX", config.GetValue("test", "multiple_choice_required_default"))
// delete remote
config.DeleteRemote("test")
@ -428,11 +428,11 @@ func TestMultipleChoice(t *testing.T) {
require.NoError(t, config.NewRemote(ctx, "test"))
assert.Equal(t, []string{"test"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test", "type"))
assert.Equal(t, "", config.FileGet("test", "multiple_choice"))
assert.Equal(t, "XXX", config.FileGet("test", "multiple_choice_required"))
assert.Equal(t, "", config.FileGet("test", "multiple_choice_default"))
assert.Equal(t, "", config.FileGet("test", "multiple_choice_required_default"))
assert.Equal(t, "config_test_remote", config.GetValue("test", "type"))
assert.Equal(t, "", config.GetValue("test", "multiple_choice"))
assert.Equal(t, "XXX", config.GetValue("test", "multiple_choice_required"))
assert.Equal(t, "", config.GetValue("test", "multiple_choice_default"))
assert.Equal(t, "", config.GetValue("test", "multiple_choice_required_default"))
}
func TestMultipleChoiceExclusive(t *testing.T) {
@ -483,9 +483,9 @@ func TestMultipleChoiceExclusive(t *testing.T) {
require.NoError(t, config.NewRemote(ctx, "test"))
assert.Equal(t, []string{"test"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test", "type"))
assert.Equal(t, "", config.FileGet("test", "multiple_choice_exclusive"))
assert.Equal(t, "", config.FileGet("test", "multiple_choice_exclusive_default"))
assert.Equal(t, "config_test_remote", config.GetValue("test", "type"))
assert.Equal(t, "", config.GetValue("test", "multiple_choice_exclusive"))
assert.Equal(t, "", config.GetValue("test", "multiple_choice_exclusive_default"))
}
func TestMultipleChoiceExclusiveRequired(t *testing.T) {
@ -539,7 +539,7 @@ func TestMultipleChoiceExclusiveRequired(t *testing.T) {
require.NoError(t, config.NewRemote(ctx, "test"))
assert.Equal(t, []string{"test"}, config.Data().GetSectionList())
assert.Equal(t, "config_test_remote", config.FileGet("test", "type"))
assert.Equal(t, "CCC", config.FileGet("test", "multiple_choice_exclusive_required"))
assert.Equal(t, "", config.FileGet("test", "multiple_choice_exclusive_required_default"))
assert.Equal(t, "config_test_remote", config.GetValue("test", "type"))
assert.Equal(t, "CCC", config.GetValue("test", "multiple_choice_exclusive_required"))
assert.Equal(t, "", config.GetValue("test", "multiple_choice_exclusive_required_default"))
}

View File

@ -429,7 +429,7 @@ func Run(t *testing.T, opt *Opt) {
// Set extra config if supplied
for _, item := range opt.ExtraConfig {
config.FileSet(item.Name, item.Key, item.Value)
config.FileSetValue(item.Name, item.Key, item.Value)
}
if *fstest.RemoteName != "" {
remoteName = *fstest.RemoteName