mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 08:23:47 +01:00
cmd/gitannex: Refactor e2e tests, add layout compat tests
TestEndToEndRepoLayoutCompat exercises git-annex-remote-rclone-builtin and git-annex-remote-rclone on the same rclone remote to ensure they are compatible. It repeats the same test for all known layout modes. Issue #7625
This commit is contained in:
Dan McArdle
Nick Craig-Wood
parent
29b58dd4c5
commit
9a2b85d71c
@ -1,6 +1,7 @@
|
||||
package gitannex
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -50,25 +51,137 @@ func checkRcloneBinaryVersion() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// This end-to-end test runs `git annex testremote` in a temporary git repo.
|
||||
// This test will be skipped unless the `rclone` binary on PATH reports the
|
||||
// expected version.
|
||||
// countFilesRecursively returns the number of files nested underneath `dir`. It
|
||||
// counts files only and excludes directories.
|
||||
func countFilesRecursively(t *testing.T, dir string) int {
|
||||
remoteFiles, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
var count int
|
||||
for _, f := range remoteFiles {
|
||||
if f.IsDir() {
|
||||
subdir := filepath.Join(dir, f.Name())
|
||||
count += countFilesRecursively(t, subdir)
|
||||
} else {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func findFileWithContents(t *testing.T, dir string, wantContents []byte) bool {
|
||||
remoteFiles, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, f := range remoteFiles {
|
||||
fPath := filepath.Join(dir, f.Name())
|
||||
if f.IsDir() {
|
||||
if findFileWithContents(t, fPath, wantContents) {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
contents, err := os.ReadFile(fPath)
|
||||
require.NoError(t, err)
|
||||
if bytes.Equal(contents, wantContents) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type e2eTestingContext struct {
|
||||
tempDir string
|
||||
binDir string
|
||||
homeDir string
|
||||
configDir string
|
||||
rcloneConfigDir string
|
||||
ephemeralRepoDir string
|
||||
}
|
||||
|
||||
// makeE2eTestingContext sets up a new e2eTestingContext rooted under
|
||||
// `t.TempDir()`. It creates the skeleton directory structure shown below in the
|
||||
// temp directory without creating any files.
|
||||
//
|
||||
// When run on CI, an rclone binary built from HEAD will be on the PATH. When
|
||||
// running locally, you will likely need to ensure the current binary is on the
|
||||
// PATH like so:
|
||||
//
|
||||
// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/...
|
||||
//
|
||||
// In the future, this test will probably be extended to test a number of
|
||||
// parameters like repo layouts, and runtime may suffer from a combinatorial
|
||||
// explosion.
|
||||
func TestEndToEnd(t *testing.T) {
|
||||
// .
|
||||
// |-- bin
|
||||
// | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}
|
||||
// |-- ephemeralRepo
|
||||
// `-- user
|
||||
// `-- .config
|
||||
// `-- rclone
|
||||
// `-- rclone.conf
|
||||
func makeE2eTestingContext(t *testing.T) e2eTestingContext {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
binDir := filepath.Join(tempDir, "bin")
|
||||
homeDir := filepath.Join(tempDir, "user")
|
||||
configDir := filepath.Join(homeDir, ".config")
|
||||
rcloneConfigDir := filepath.Join(configDir, "rclone")
|
||||
ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")
|
||||
|
||||
for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {
|
||||
require.NoError(t, os.Mkdir(dir, 0700))
|
||||
}
|
||||
|
||||
return e2eTestingContext{tempDir, binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir}
|
||||
}
|
||||
|
||||
// Install the symlink that enables git-annex to invoke "rclone gitannex"
|
||||
// without explicitly specifying the subcommand.
|
||||
func (e *e2eTestingContext) installRcloneGitannexSymlink(t *testing.T) {
|
||||
rcloneBinaryPath, err := exec.LookPath("rclone")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Symlink(
|
||||
rcloneBinaryPath,
|
||||
filepath.Join(e.binDir, "git-annex-remote-rclone-builtin")))
|
||||
}
|
||||
|
||||
// Install a rclone.conf file in an appropriate location in the fake home
|
||||
// directory. The config defines an rclone remote named "MyRcloneRemote" using
|
||||
// the local backend.
|
||||
func (e *e2eTestingContext) installRcloneConfig(t *testing.T) {
|
||||
// Install the rclone.conf file that defines the remote.
|
||||
rcloneConfigPath := filepath.Join(e.rcloneConfigDir, "rclone.conf")
|
||||
rcloneConfigContents := "[MyRcloneRemote]\ntype = local"
|
||||
require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))
|
||||
}
|
||||
|
||||
// runInRepo runs the given command from within the ephemeral repo directory. To
|
||||
// prevent accidental changes in the real home directory, it sets the HOME
|
||||
// variable to a subdirectory of the temp directory. It also ensures that the
|
||||
// git-annex-remote-rclone-builtin symlink will be found by extending the PATH.
|
||||
func (e *e2eTestingContext) runInRepo(t *testing.T, command string, args ...string) {
|
||||
fmt.Printf("+ %s %v\n", command, args)
|
||||
cmd := exec.Command(command, args...)
|
||||
cmd.Dir = e.ephemeralRepoDir
|
||||
cmd.Env = []string{
|
||||
"HOME=" + e.homeDir,
|
||||
"PATH=" + os.Getenv("PATH") + ":" + e.binDir,
|
||||
}
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
require.NoError(t, cmd.Run())
|
||||
}
|
||||
|
||||
// createGitRepo creates an empty git repository in the ephemeral repo
|
||||
// directory. It makes "global" config changes that are ultimately scoped to the
|
||||
// calling test thanks to runInRepo() overriding the HOME environment variable.
|
||||
func (e *e2eTestingContext) createGitRepo(t *testing.T) {
|
||||
e.runInRepo(t, "git", "annex", "version")
|
||||
e.runInRepo(t, "git", "config", "--global", "user.name", "User Name")
|
||||
e.runInRepo(t, "git", "config", "--global", "user.email", "user@example.com")
|
||||
e.runInRepo(t, "git", "config", "--global", "init.defaultBranch", "main")
|
||||
e.runInRepo(t, "git", "init")
|
||||
e.runInRepo(t, "git", "annex", "init")
|
||||
}
|
||||
|
||||
func skipE2eTestIfNecessary(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping due to short mode.")
|
||||
}
|
||||
|
||||
// TODO: Support this test on Windows. Need to evaluate the semantics of the
|
||||
// TODO: Support e2e tests on Windows. Need to evaluate the semantics of the
|
||||
// HOME and PATH environment variables.
|
||||
switch runtime.GOOS {
|
||||
case "darwin",
|
||||
@ -89,6 +202,23 @@ func TestEndToEnd(t *testing.T) {
|
||||
if _, err := exec.LookPath("git-annex"); err != nil {
|
||||
t.Skipf("Skipping because git-annex was not found: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// This end-to-end test runs `git annex testremote` in a temporary git repo.
|
||||
// This test will be skipped unless the `rclone` binary on PATH reports the
|
||||
// expected version.
|
||||
//
|
||||
// When run on CI, an rclone binary built from HEAD will be on the PATH. When
|
||||
// running locally, you will likely need to ensure the current binary is on the
|
||||
// PATH like so:
|
||||
//
|
||||
// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/...
|
||||
//
|
||||
// In the future, this test will probably be extended to test a number of
|
||||
// parameters like repo layouts, and runtime may suffer from a combinatorial
|
||||
// explosion.
|
||||
func TestEndToEnd(t *testing.T) {
|
||||
skipE2eTestIfNecessary(t)
|
||||
|
||||
// Create a temp directory and chdir there, just in case.
|
||||
originalWd, err := os.Getwd()
|
||||
@ -97,63 +227,114 @@ func TestEndToEnd(t *testing.T) {
|
||||
require.NoError(t, os.Chdir(tempDir))
|
||||
defer func() { require.NoError(t, os.Chdir(originalWd)) }()
|
||||
|
||||
// Flesh out subdirectories of the temp directory:
|
||||
//
|
||||
// .
|
||||
// |-- bin
|
||||
// | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}
|
||||
// |-- ephemeralRepo
|
||||
// `-- user
|
||||
// `-- .config
|
||||
// `-- rclone
|
||||
// `-- rclone.conf
|
||||
testingContext := makeE2eTestingContext(t)
|
||||
testingContext.installRcloneGitannexSymlink(t)
|
||||
testingContext.installRcloneConfig(t)
|
||||
testingContext.createGitRepo(t)
|
||||
|
||||
binDir := filepath.Join(tempDir, "bin")
|
||||
homeDir := filepath.Join(tempDir, "user")
|
||||
configDir := filepath.Join(homeDir, ".config")
|
||||
rcloneConfigDir := filepath.Join(configDir, "rclone")
|
||||
ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")
|
||||
for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {
|
||||
require.NoError(t, os.Mkdir(dir, 0700))
|
||||
testingContext.runInRepo(t, "git", "annex", "initremote", "MyTestRemote",
|
||||
"type=external", "externaltype=rclone-builtin", "encryption=none",
|
||||
"rcloneremotename=MyRcloneRemote", "rcloneprefix="+testingContext.ephemeralRepoDir)
|
||||
|
||||
testingContext.runInRepo(t, "git", "annex", "testremote", "MyTestRemote")
|
||||
}
|
||||
|
||||
// For each layout mode, ensure that we're compatible with data written by
|
||||
// git-annex-remote-rclone.
|
||||
func TestEndToEndRepoLayoutCompat(t *testing.T) {
|
||||
skipE2eTestIfNecessary(t)
|
||||
|
||||
if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil {
|
||||
t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err)
|
||||
}
|
||||
|
||||
// Install the symlink that enables git-annex to invoke "rclone gitannex"
|
||||
// without explicitly specifying the subcommand.
|
||||
rcloneBinaryPath, err := exec.LookPath("rclone")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Symlink(
|
||||
rcloneBinaryPath,
|
||||
filepath.Join(binDir, "git-annex-remote-rclone-builtin")))
|
||||
for _, mode := range allLayoutModes() {
|
||||
mode := mode
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Install the rclone.conf file that defines the remote.
|
||||
rcloneConfigPath := filepath.Join(rcloneConfigDir, "rclone.conf")
|
||||
rcloneConfigContents := "[MyRcloneRemote]\ntype = local"
|
||||
require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))
|
||||
// Create a temp directory and chdir there, just in case.
|
||||
originalWd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
tempDir := t.TempDir()
|
||||
require.NoError(t, os.Chdir(tempDir))
|
||||
defer func() { require.NoError(t, os.Chdir(originalWd)) }()
|
||||
|
||||
// NOTE: These commands must be run with HOME pointing at an ephemeral
|
||||
// directory, rather than the real home directory.
|
||||
cmds := [][]string{
|
||||
{"git", "annex", "version"},
|
||||
{"git", "config", "--global", "user.name", "User Name"},
|
||||
{"git", "config", "--global", "user.email", "user@example.com"},
|
||||
{"git", "init"},
|
||||
{"git", "annex", "init"},
|
||||
{"git", "annex", "initremote", "MyTestRemote",
|
||||
"type=external", "externaltype=rclone-builtin", "encryption=none",
|
||||
"rcloneremotename=MyRcloneRemote", "rcloneprefix=" + ephemeralRepoDir},
|
||||
{"git", "annex", "testremote", "MyTestRemote"},
|
||||
}
|
||||
tc := makeE2eTestingContext(t)
|
||||
tc.installRcloneGitannexSymlink(t)
|
||||
tc.installRcloneConfig(t)
|
||||
tc.createGitRepo(t)
|
||||
|
||||
for _, args := range cmds {
|
||||
fmt.Printf("+ %v\n", args)
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
cmd.Dir = ephemeralRepoDir
|
||||
cmd.Env = []string{
|
||||
"HOME=" + homeDir,
|
||||
"PATH=" + os.Getenv("PATH") + ":" + binDir,
|
||||
}
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
require.NoError(t, cmd.Run())
|
||||
remoteStorage := filepath.Join(tc.tempDir, "remotePrefix")
|
||||
require.NoError(t, os.Mkdir(remoteStorage, 0777))
|
||||
|
||||
tc.runInRepo(t,
|
||||
"git", "annex", "initremote", "Control",
|
||||
"type=external", "externaltype=rclone", "encryption=none",
|
||||
"target=MyRcloneRemote",
|
||||
"rclone_layout="+string(mode),
|
||||
"prefix="+remoteStorage)
|
||||
|
||||
tc.runInRepo(t,
|
||||
"git", "annex", "initremote", "Experiment",
|
||||
"type=external", "externaltype=rclone-builtin", "encryption=none",
|
||||
"rcloneremotename=MyRcloneRemote",
|
||||
"rclonelayout="+string(mode),
|
||||
"rcloneprefix="+remoteStorage)
|
||||
|
||||
fooFileContents := []byte{1, 2, 3, 4}
|
||||
fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo")
|
||||
require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700))
|
||||
tc.runInRepo(t, "git", "annex", "add", "foo")
|
||||
tc.runInRepo(t, "git", "commit", "-m", "Add foo file")
|
||||
// Git-annex objects are not writable, which prevents `testing` from
|
||||
// cleaning up the temp directory. We can work around this by
|
||||
// explicitly dropping any files we add to the annex.
|
||||
t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") })
|
||||
|
||||
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
|
||||
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
||||
|
||||
// Copy the file to Control and verify it's present on Experiment.
|
||||
|
||||
tc.runInRepo(t, "git", "annex", "copy", "--to=Control", "foo")
|
||||
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
||||
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
||||
|
||||
tc.runInRepo(t, "git", "annex", "fsck", "--from=Experiment", "foo")
|
||||
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
||||
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
||||
|
||||
// Drop the file locally and verify we can copy it back from Experiment.
|
||||
|
||||
tc.runInRepo(t, "git", "annex", "drop", "--force", "foo")
|
||||
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
||||
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
||||
|
||||
tc.runInRepo(t, "git", "annex", "copy", "--from=Experiment", "foo")
|
||||
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
||||
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
||||
|
||||
// Drop the file from Experiment, copy it back to Experiment, and
|
||||
// verify it's still present on Control.
|
||||
|
||||
tc.runInRepo(t, "git", "annex", "drop", "--from=Experiment", "--force", "foo")
|
||||
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
|
||||
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
||||
|
||||
tc.runInRepo(t, "git", "annex", "copy", "--to=Experiment", "foo")
|
||||
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
||||
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
||||
|
||||
tc.runInRepo(t, "git", "annex", "fsck", "--from=Control", "foo")
|
||||
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
||||
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
||||
|
||||
// Drop the file from Control.
|
||||
|
||||
tc.runInRepo(t, "git", "annex", "drop", "--from=Control", "--force", "foo")
|
||||
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
|
||||
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user