cmd/gitannex: Add support for different layouts

This commit adds support for the same repo layouts supported by
git-annex-remote-rclone. This should enable git-annex users with remotes
of type "rclone" to switch to a "rclone-builtin" without needing to
retransfer content.

Issue #7625
This commit is contained in:
Dan McArdle 2024-04-11 11:10:16 -04:00 committed by Nick Craig-Wood
parent 36ad4eb145
commit 29b58dd4c5
3 changed files with 164 additions and 15 deletions

View File

@ -110,9 +110,10 @@ func (m *messageParser) finalParameter() string {
// configDefinition describes a configuration value required by this command. We
// use "GETCONFIG" messages to query git-annex for these values at runtime.
type configDefinition struct {
name string
description string
destination *string
name string
description string
destination *string
defaultValue *string
}
// server contains this command's current state.
@ -132,6 +133,7 @@ type server struct {
configsDone bool
configPrefix string
configRcloneRemoteName string
configRcloneLayout string
}
func (s *server) sendMsg(msg string) {
@ -267,6 +269,9 @@ func (s *server) handleInitRemote() error {
// Get a list of configs with pointers to fields of `s`.
func (s *server) getRequiredConfigs() []configDefinition {
defaultRclonePrefix := "git-annex-rclone"
defaultRcloneLayout := "nodir"
return []configDefinition{
{
"rcloneremotename",
@ -274,13 +279,23 @@ func (s *server) getRequiredConfigs() []configDefinition {
"Must match a remote known to rclone. " +
"(Note that rclone remotes are a distinct concept from git-annex remotes.)",
&s.configRcloneRemoteName,
nil,
},
{
"rcloneprefix",
"Directory where rclone will write git-annex content. " +
"If not specified, defaults to \"git-annex-rclone\". " +
"This directory be created on init if it does not exist.",
fmt.Sprintf("If not specified, defaults to %q. ", defaultRclonePrefix) +
"This directory will be created on init if it does not exist.",
&s.configPrefix,
&defaultRclonePrefix,
},
{
"rclonelayout",
"Defines where, within the rcloneprefix directory, rclone will write git-annex content. " +
fmt.Sprintf("Must be one of %v. ", allLayoutModes()) +
fmt.Sprintf("If empty, defaults to %q.", defaultRcloneLayout),
&s.configRcloneLayout,
&defaultRcloneLayout,
},
}
}
@ -307,10 +322,13 @@ func (s *server) queryConfigs() error {
}
value := message.finalParameter()
if value == "" {
if value == "" && config.defaultValue == nil {
return fmt.Errorf("config value of %q must not be empty", config.name)
}
if value == "" {
*config.destination = *config.defaultValue
continue
}
*config.destination = value
}
@ -358,7 +376,19 @@ func (s *server) handleTransfer(message *messageParser) error {
return fmt.Errorf("error getting configs: %w", err)
}
remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
layout := parseLayoutMode(s.configRcloneLayout)
if layout == layoutModeUnknown {
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey))
return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
}
remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
if err != nil {
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey))
return fmt.Errorf("error building fs string: %w", err)
}
remoteFs, err := cache.Get(context.TODO(), remoteFsString)
if err != nil {
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey))
return err
@ -415,7 +445,19 @@ func (s *server) handleCheckPresent(message *messageParser) error {
return fmt.Errorf("error getting configs: %s", err)
}
remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
layout := parseLayoutMode(s.configRcloneLayout)
if layout == layoutModeUnknown {
s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
}
remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
if err != nil {
s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
return fmt.Errorf("error building fs string: %w", err)
}
remoteFs, err := cache.Get(context.TODO(), remoteFsString)
if err != nil {
s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey))
return err
@ -435,17 +477,45 @@ func (s *server) handleCheckPresent(message *messageParser) error {
return nil
}
func (s *server) queryDirhash(msg string) (string, error) {
s.sendMsg(msg)
parser, err := s.getMsg()
if err != nil {
return "", err
}
keyword, err := parser.nextSpaceDelimitedParameter()
if err != nil {
return "", err
}
if keyword != "VALUE" {
return "", fmt.Errorf("expected VALUE keyword, but got %q", keyword)
}
dirhash, err := parser.nextSpaceDelimitedParameter()
if err != nil {
return "", fmt.Errorf("failed to parse dirhash: %w", err)
}
return dirhash, nil
}
func (s *server) handleRemove(message *messageParser) error {
argKey := message.finalParameter()
if argKey == "" {
return errors.New("failed to parse key for REMOVE")
}
if err != nil {
return err
layout := parseLayoutMode(s.configRcloneLayout)
if layout == layoutModeUnknown {
s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
}
remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
if err != nil {
s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
return fmt.Errorf("error building fs string: %w", err)
}
remoteFs, err := cache.Get(context.TODO(), remoteFsString)
if err != nil {
s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
return fmt.Errorf("error getting remote fs: %w", err)

View File

@ -234,6 +234,7 @@ func (h *testState) requireWriteLine(line string) {
func (h *testState) preconfigureServer() {
h.server.configPrefix = h.localFsDir
h.server.configRcloneRemoteName = h.remoteName
h.server.configRcloneLayout = string(layoutModeNodir)
h.server.configsDone = true
}
@ -285,6 +286,8 @@ var localBackendTestCases = []testCase{
h.requireWriteLine("VALUE " + h.remoteName)
h.requireReadLineExact("GETCONFIG rcloneprefix")
h.requireWriteLine("VALUE " + h.localFsDir)
h.requireReadLineExact("GETCONFIG rclonelayout")
h.requireWriteLine("VALUE foo")
h.requireReadLineExact("PREPARE-SUCCESS")
require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
@ -313,9 +316,13 @@ var localBackendTestCases = []testCase{
localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)
h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))
h.requireReadLineExact("GETCONFIG rcloneprefix")
h.requireReadLineExact("GETCONFIG rcloneprefix")
h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))
h.requireReadLineExact("GETCONFIG rclonelayout")
h.requireWriteLine("VALUE")
h.requireReadLineExact("PREPARE-SUCCESS")
require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)
@ -367,11 +374,11 @@ var localBackendTestCases = []testCase{
// Note the whitespace following the key.
h.requireWriteLine("TRANSFER STORE Key ")
h.requireReadLineExact("TRANSFER-FAILURE failed to parse file")
h.requireReadLineExact("TRANSFER-FAILURE failed to parse file path")
require.NoError(t, h.mockStdinW.Close())
},
expectedError: "malformed arguments for TRANSFER: nothing remains to parse",
expectedError: "failed to parse file",
},
// Repeated EXTENSIONS messages add to each other rather than overriding
// prior advertised extensions. This behavior is not mandated by the

72
cmd/gitannex/layout.go Normal file
View File

@ -0,0 +1,72 @@
package gitannex
import (
"fmt"
"strings"
)
type layoutMode string
// All layout modes from git-annex-remote-rclone are supported.
const (
layoutModeLower layoutMode = "lower"
layoutModeDirectory layoutMode = "directory"
layoutModeNodir layoutMode = "nodir"
layoutModeMixed layoutMode = "mixed"
layoutModeFrankencase layoutMode = "frankencase"
layoutModeUnknown layoutMode = ""
)
func allLayoutModes() []layoutMode {
return []layoutMode{
layoutModeLower,
layoutModeDirectory,
layoutModeNodir,
layoutModeMixed,
layoutModeFrankencase,
}
}
func parseLayoutMode(mode string) layoutMode {
for _, knownMode := range allLayoutModes() {
if mode == string(knownMode) {
return knownMode
}
}
return layoutModeUnknown
}
type queryDirhashFunc func(msg string) (string, error)
func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) {
if mode == layoutModeNodir {
return fmt.Sprintf("%s:%s", remoteName, prefix), nil
}
var dirhash string
var err error
switch mode {
case layoutModeLower, layoutModeDirectory:
dirhash, err = queryDirhash("DIRHASH-LOWER " + key)
case layoutModeMixed, layoutModeFrankencase:
dirhash, err = queryDirhash("DIRHASH " + key)
default:
panic("unreachable")
}
if err != nil {
return "", fmt.Errorf("buildFsString failed to query dirhash: %w", err)
}
switch mode {
case layoutModeLower:
return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
case layoutModeDirectory:
return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil
case layoutModeMixed:
return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
case layoutModeFrankencase:
return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil
default:
panic("unreachable")
}
}