From 29b58dd4c5db413109cb380df75ad93560abff77 Mon Sep 17 00:00:00 2001 From: Dan McArdle Date: Thu, 11 Apr 2024 11:10:16 -0400 Subject: [PATCH] cmd/gitannex: Add support for different layouts This commit adds support for the same repo layouts supported by git-annex-remote-rclone. This should enable git-annex users with remotes of type "rclone" to switch to a "rclone-builtin" without needing to retransfer content. Issue #7625 --- cmd/gitannex/gitannex.go | 94 ++++++++++++++++++++++++++++++----- cmd/gitannex/gitannex_test.go | 13 +++-- cmd/gitannex/layout.go | 72 +++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 15 deletions(-) create mode 100644 cmd/gitannex/layout.go diff --git a/cmd/gitannex/gitannex.go b/cmd/gitannex/gitannex.go index a71b04871..0eb09da12 100644 --- a/cmd/gitannex/gitannex.go +++ b/cmd/gitannex/gitannex.go @@ -110,9 +110,10 @@ func (m *messageParser) finalParameter() string { // configDefinition describes a configuration value required by this command. We // use "GETCONFIG" messages to query git-annex for these values at runtime. type configDefinition struct { - name string - description string - destination *string + name string + description string + destination *string + defaultValue *string } // server contains this command's current state. @@ -132,6 +133,7 @@ type server struct { configsDone bool configPrefix string configRcloneRemoteName string + configRcloneLayout string } func (s *server) sendMsg(msg string) { @@ -267,6 +269,9 @@ func (s *server) handleInitRemote() error { // Get a list of configs with pointers to fields of `s`. func (s *server) getRequiredConfigs() []configDefinition { + defaultRclonePrefix := "git-annex-rclone" + defaultRcloneLayout := "nodir" + return []configDefinition{ { "rcloneremotename", @@ -274,13 +279,23 @@ func (s *server) getRequiredConfigs() []configDefinition { "Must match a remote known to rclone. " + "(Note that rclone remotes are a distinct concept from git-annex remotes.)", &s.configRcloneRemoteName, + nil, }, { "rcloneprefix", "Directory where rclone will write git-annex content. " + - "If not specified, defaults to \"git-annex-rclone\". " + - "This directory be created on init if it does not exist.", + fmt.Sprintf("If not specified, defaults to %q. ", defaultRclonePrefix) + + "This directory will be created on init if it does not exist.", &s.configPrefix, + &defaultRclonePrefix, + }, + { + "rclonelayout", + "Defines where, within the rcloneprefix directory, rclone will write git-annex content. " + + fmt.Sprintf("Must be one of %v. ", allLayoutModes()) + + fmt.Sprintf("If empty, defaults to %q.", defaultRcloneLayout), + &s.configRcloneLayout, + &defaultRcloneLayout, }, } } @@ -307,10 +322,13 @@ func (s *server) queryConfigs() error { } value := message.finalParameter() - if value == "" { + if value == "" && config.defaultValue == nil { return fmt.Errorf("config value of %q must not be empty", config.name) } - + if value == "" { + *config.destination = *config.defaultValue + continue + } *config.destination = value } @@ -358,7 +376,19 @@ func (s *server) handleTransfer(message *messageParser) error { return fmt.Errorf("error getting configs: %w", err) } - remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix)) + layout := parseLayoutMode(s.configRcloneLayout) + if layout == layoutModeUnknown { + s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey)) + return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout) + } + + remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix) + if err != nil { + s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey)) + return fmt.Errorf("error building fs string: %w", err) + } + + remoteFs, err := cache.Get(context.TODO(), remoteFsString) if err != nil { s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey)) return err @@ -415,7 +445,19 @@ func (s *server) handleCheckPresent(message *messageParser) error { return fmt.Errorf("error getting configs: %s", err) } - remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix)) + layout := parseLayoutMode(s.configRcloneLayout) + if layout == layoutModeUnknown { + s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey)) + return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout) + } + + remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix) + if err != nil { + s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey)) + return fmt.Errorf("error building fs string: %w", err) + } + + remoteFs, err := cache.Get(context.TODO(), remoteFsString) if err != nil { s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey)) return err @@ -435,17 +477,45 @@ func (s *server) handleCheckPresent(message *messageParser) error { return nil } +func (s *server) queryDirhash(msg string) (string, error) { + s.sendMsg(msg) + parser, err := s.getMsg() + if err != nil { + return "", err + } + keyword, err := parser.nextSpaceDelimitedParameter() + if err != nil { + return "", err + } + if keyword != "VALUE" { + return "", fmt.Errorf("expected VALUE keyword, but got %q", keyword) + } + dirhash, err := parser.nextSpaceDelimitedParameter() + if err != nil { + return "", fmt.Errorf("failed to parse dirhash: %w", err) + } + return dirhash, nil +} + func (s *server) handleRemove(message *messageParser) error { argKey := message.finalParameter() if argKey == "" { return errors.New("failed to parse key for REMOVE") } - if err != nil { - return err + layout := parseLayoutMode(s.configRcloneLayout) + if layout == layoutModeUnknown { + s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey)) + return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout) } - remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix)) + remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix) + if err != nil { + s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey)) + return fmt.Errorf("error building fs string: %w", err) + } + + remoteFs, err := cache.Get(context.TODO(), remoteFsString) if err != nil { s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey)) return fmt.Errorf("error getting remote fs: %w", err) diff --git a/cmd/gitannex/gitannex_test.go b/cmd/gitannex/gitannex_test.go index b3846221d..2b023b8c0 100644 --- a/cmd/gitannex/gitannex_test.go +++ b/cmd/gitannex/gitannex_test.go @@ -234,6 +234,7 @@ func (h *testState) requireWriteLine(line string) { func (h *testState) preconfigureServer() { h.server.configPrefix = h.localFsDir h.server.configRcloneRemoteName = h.remoteName + h.server.configRcloneLayout = string(layoutModeNodir) h.server.configsDone = true } @@ -285,6 +286,8 @@ var localBackendTestCases = []testCase{ h.requireWriteLine("VALUE " + h.remoteName) h.requireReadLineExact("GETCONFIG rcloneprefix") h.requireWriteLine("VALUE " + h.localFsDir) + h.requireReadLineExact("GETCONFIG rclonelayout") + h.requireWriteLine("VALUE foo") h.requireReadLineExact("PREPARE-SUCCESS") require.Equal(t, h.server.configRcloneRemoteName, h.remoteName) @@ -313,9 +316,13 @@ var localBackendTestCases = []testCase{ localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir) h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces)) - h.requireReadLineExact("GETCONFIG rcloneprefix") + h.requireReadLineExact("GETCONFIG rcloneprefix") h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces)) + + h.requireReadLineExact("GETCONFIG rclonelayout") + h.requireWriteLine("VALUE") + h.requireReadLineExact("PREPARE-SUCCESS") require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces) @@ -367,11 +374,11 @@ var localBackendTestCases = []testCase{ // Note the whitespace following the key. h.requireWriteLine("TRANSFER STORE Key ") - h.requireReadLineExact("TRANSFER-FAILURE failed to parse file") + h.requireReadLineExact("TRANSFER-FAILURE failed to parse file path") require.NoError(t, h.mockStdinW.Close()) }, - expectedError: "malformed arguments for TRANSFER: nothing remains to parse", + expectedError: "failed to parse file", }, // Repeated EXTENSIONS messages add to each other rather than overriding // prior advertised extensions. This behavior is not mandated by the diff --git a/cmd/gitannex/layout.go b/cmd/gitannex/layout.go new file mode 100644 index 000000000..81b18bfb6 --- /dev/null +++ b/cmd/gitannex/layout.go @@ -0,0 +1,72 @@ +package gitannex + +import ( + "fmt" + "strings" +) + +type layoutMode string + +// All layout modes from git-annex-remote-rclone are supported. +const ( + layoutModeLower layoutMode = "lower" + layoutModeDirectory layoutMode = "directory" + layoutModeNodir layoutMode = "nodir" + layoutModeMixed layoutMode = "mixed" + layoutModeFrankencase layoutMode = "frankencase" + layoutModeUnknown layoutMode = "" +) + +func allLayoutModes() []layoutMode { + return []layoutMode{ + layoutModeLower, + layoutModeDirectory, + layoutModeNodir, + layoutModeMixed, + layoutModeFrankencase, + } +} + +func parseLayoutMode(mode string) layoutMode { + for _, knownMode := range allLayoutModes() { + if mode == string(knownMode) { + return knownMode + } + } + return layoutModeUnknown +} + +type queryDirhashFunc func(msg string) (string, error) + +func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) { + if mode == layoutModeNodir { + return fmt.Sprintf("%s:%s", remoteName, prefix), nil + } + + var dirhash string + var err error + switch mode { + case layoutModeLower, layoutModeDirectory: + dirhash, err = queryDirhash("DIRHASH-LOWER " + key) + case layoutModeMixed, layoutModeFrankencase: + dirhash, err = queryDirhash("DIRHASH " + key) + default: + panic("unreachable") + } + if err != nil { + return "", fmt.Errorf("buildFsString failed to query dirhash: %w", err) + } + + switch mode { + case layoutModeLower: + return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil + case layoutModeDirectory: + return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil + case layoutModeMixed: + return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil + case layoutModeFrankencase: + return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil + default: + panic("unreachable") + } +}