mirror of
https://github.com/rclone/rclone.git
synced 2024-11-25 01:44:41 +01:00
4ab235c06c
I'm hopeful that running these in parallel will not impact CI runtime very much, but that likely depends on the number of CPU cores and whether the tmp filesystem is backed by memory vs a physical disk. Issue #7625
349 lines
12 KiB
Go
349 lines
12 KiB
Go
package gitannex
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"runtime"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/lib/buildinfo"
|
|
)
|
|
|
|
// checkRcloneBinaryVersion runs whichever rclone is on the PATH and checks
|
|
// whether it reports a version that matches the test's expectations. Returns
|
|
// nil when the version is the expected version, otherwise returns an error.
|
|
func checkRcloneBinaryVersion() error {
|
|
// versionInfo is a subset of information produced by "core/version".
|
|
type versionInfo struct {
|
|
Version string
|
|
IsGit bool
|
|
GoTags string
|
|
}
|
|
|
|
cmd := exec.Command("rclone", "rc", "--loopback", "core/version")
|
|
stdout, err := cmd.Output()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get rclone version: %w", err)
|
|
}
|
|
|
|
var parsed versionInfo
|
|
if err := json.Unmarshal(stdout, &parsed); err != nil {
|
|
return fmt.Errorf("failed to parse rclone version: %w", err)
|
|
}
|
|
if parsed.Version != fs.Version {
|
|
return fmt.Errorf("expected version %q, but got %q", fs.Version, parsed.Version)
|
|
}
|
|
if !parsed.IsGit {
|
|
return errors.New("expected rclone to be a dev build")
|
|
}
|
|
_, tagString := buildinfo.GetLinkingAndTags()
|
|
if parsed.GoTags != tagString {
|
|
return fmt.Errorf("expected tag string %q, but got %q", tagString, parsed.GoTags)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// countFilesRecursively returns the number of files nested underneath `dir`. It
|
|
// counts files only and excludes directories.
|
|
func countFilesRecursively(t *testing.T, dir string) int {
|
|
remoteFiles, err := os.ReadDir(dir)
|
|
require.NoError(t, err)
|
|
|
|
var count int
|
|
for _, f := range remoteFiles {
|
|
if f.IsDir() {
|
|
subdir := filepath.Join(dir, f.Name())
|
|
count += countFilesRecursively(t, subdir)
|
|
} else {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func findFileWithContents(t *testing.T, dir string, wantContents []byte) bool {
|
|
remoteFiles, err := os.ReadDir(dir)
|
|
require.NoError(t, err)
|
|
|
|
for _, f := range remoteFiles {
|
|
fPath := filepath.Join(dir, f.Name())
|
|
if f.IsDir() {
|
|
if findFileWithContents(t, fPath, wantContents) {
|
|
return true
|
|
}
|
|
} else {
|
|
contents, err := os.ReadFile(fPath)
|
|
require.NoError(t, err)
|
|
if bytes.Equal(contents, wantContents) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
type e2eTestingContext struct {
|
|
tempDir string
|
|
binDir string
|
|
homeDir string
|
|
configDir string
|
|
rcloneConfigDir string
|
|
ephemeralRepoDir string
|
|
}
|
|
|
|
// makeE2eTestingContext sets up a new e2eTestingContext rooted under
|
|
// `t.TempDir()`. It creates the skeleton directory structure shown below in the
|
|
// temp directory without creating any files.
|
|
//
|
|
// .
|
|
// |-- bin
|
|
// | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}
|
|
// |-- ephemeralRepo
|
|
// `-- user
|
|
// `-- .config
|
|
// `-- rclone
|
|
// `-- rclone.conf
|
|
func makeE2eTestingContext(t *testing.T) e2eTestingContext {
|
|
tempDir := t.TempDir()
|
|
|
|
binDir := filepath.Join(tempDir, "bin")
|
|
homeDir := filepath.Join(tempDir, "user")
|
|
configDir := filepath.Join(homeDir, ".config")
|
|
rcloneConfigDir := filepath.Join(configDir, "rclone")
|
|
ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")
|
|
|
|
for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {
|
|
require.NoError(t, os.Mkdir(dir, 0700))
|
|
}
|
|
|
|
return e2eTestingContext{tempDir, binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir}
|
|
}
|
|
|
|
// Install the symlink that enables git-annex to invoke "rclone gitannex"
|
|
// without explicitly specifying the subcommand.
|
|
func (e *e2eTestingContext) installRcloneGitannexSymlink(t *testing.T) {
|
|
rcloneBinaryPath, err := exec.LookPath("rclone")
|
|
require.NoError(t, err)
|
|
require.NoError(t, os.Symlink(
|
|
rcloneBinaryPath,
|
|
filepath.Join(e.binDir, "git-annex-remote-rclone-builtin")))
|
|
}
|
|
|
|
// Install a rclone.conf file in an appropriate location in the fake home
|
|
// directory. The config defines an rclone remote named "MyRcloneRemote" using
|
|
// the local backend.
|
|
func (e *e2eTestingContext) installRcloneConfig(t *testing.T) {
|
|
// Install the rclone.conf file that defines the remote.
|
|
rcloneConfigPath := filepath.Join(e.rcloneConfigDir, "rclone.conf")
|
|
rcloneConfigContents := "[MyRcloneRemote]\ntype = local"
|
|
require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))
|
|
}
|
|
|
|
// runInRepo runs the given command from within the ephemeral repo directory. To
|
|
// prevent accidental changes in the real home directory, it sets the HOME
|
|
// variable to a subdirectory of the temp directory. It also ensures that the
|
|
// git-annex-remote-rclone-builtin symlink will be found by extending the PATH.
|
|
func (e *e2eTestingContext) runInRepo(t *testing.T, command string, args ...string) {
|
|
fmt.Printf("+ %s %v\n", command, args)
|
|
cmd := exec.Command(command, args...)
|
|
cmd.Dir = e.ephemeralRepoDir
|
|
cmd.Env = []string{
|
|
"HOME=" + e.homeDir,
|
|
"PATH=" + os.Getenv("PATH") + ":" + e.binDir,
|
|
}
|
|
cmd.Stdout = os.Stdout
|
|
cmd.Stderr = os.Stderr
|
|
require.NoError(t, cmd.Run())
|
|
}
|
|
|
|
// createGitRepo creates an empty git repository in the ephemeral repo
|
|
// directory. It makes "global" config changes that are ultimately scoped to the
|
|
// calling test thanks to runInRepo() overriding the HOME environment variable.
|
|
func (e *e2eTestingContext) createGitRepo(t *testing.T) {
|
|
e.runInRepo(t, "git", "annex", "version")
|
|
e.runInRepo(t, "git", "config", "--global", "user.name", "User Name")
|
|
e.runInRepo(t, "git", "config", "--global", "user.email", "user@example.com")
|
|
e.runInRepo(t, "git", "config", "--global", "init.defaultBranch", "main")
|
|
e.runInRepo(t, "git", "init")
|
|
e.runInRepo(t, "git", "annex", "init")
|
|
}
|
|
|
|
func skipE2eTestIfNecessary(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping due to short mode.")
|
|
}
|
|
|
|
// TODO: Support e2e tests on Windows. Need to evaluate the semantics of the
|
|
// HOME and PATH environment variables.
|
|
switch runtime.GOOS {
|
|
case "darwin",
|
|
"freebsd",
|
|
"linux",
|
|
"netbsd",
|
|
"openbsd",
|
|
"plan9",
|
|
"solaris":
|
|
default:
|
|
t.Skipf("GOOS %q is not supported.", runtime.GOOS)
|
|
}
|
|
|
|
if err := checkRcloneBinaryVersion(); err != nil {
|
|
t.Skipf("Skipping due to rclone version: %s", err)
|
|
}
|
|
|
|
if _, err := exec.LookPath("git-annex"); err != nil {
|
|
t.Skipf("Skipping because git-annex was not found: %s", err)
|
|
}
|
|
}
|
|
|
|
// This end-to-end test runs `git annex testremote` in a temporary git repo.
|
|
// This test will be skipped unless the `rclone` binary on PATH reports the
|
|
// expected version.
|
|
//
|
|
// When run on CI, an rclone binary built from HEAD will be on the PATH. When
|
|
// running locally, you will likely need to ensure the current binary is on the
|
|
// PATH like so:
|
|
//
|
|
// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/...
|
|
//
|
|
// In the future, this test will probably be extended to test a number of
|
|
// parameters like repo layouts, and runtime may suffer from a combinatorial
|
|
// explosion.
|
|
func TestEndToEnd(t *testing.T) {
|
|
skipE2eTestIfNecessary(t)
|
|
|
|
for _, mode := range allLayoutModes() {
|
|
mode := mode
|
|
t.Run(string(mode), func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Create a temp directory and chdir there, just in case.
|
|
originalWd, err := os.Getwd()
|
|
require.NoError(t, err)
|
|
tempDir := t.TempDir()
|
|
require.NoError(t, os.Chdir(tempDir))
|
|
t.Cleanup(func() { require.NoError(t, os.Chdir(originalWd)) })
|
|
|
|
testingContext := makeE2eTestingContext(t)
|
|
testingContext.installRcloneGitannexSymlink(t)
|
|
testingContext.installRcloneConfig(t)
|
|
testingContext.createGitRepo(t)
|
|
|
|
testingContext.runInRepo(t, "git", "annex", "initremote", "MyTestRemote",
|
|
"type=external", "externaltype=rclone-builtin", "encryption=none",
|
|
"rcloneremotename=MyRcloneRemote", "rcloneprefix="+testingContext.ephemeralRepoDir,
|
|
"rclonelayout="+string(mode))
|
|
|
|
testingContext.runInRepo(t, "git", "annex", "testremote", "MyTestRemote")
|
|
})
|
|
}
|
|
}
|
|
|
|
// For each layout mode, ensure that we're compatible with data written by
|
|
// git-annex-remote-rclone.
|
|
func TestEndToEndRepoLayoutCompat(t *testing.T) {
|
|
skipE2eTestIfNecessary(t)
|
|
|
|
if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil {
|
|
t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err)
|
|
}
|
|
|
|
for _, mode := range allLayoutModes() {
|
|
mode := mode
|
|
t.Run(string(mode), func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Create a temp directory and chdir there, just in case.
|
|
originalWd, err := os.Getwd()
|
|
require.NoError(t, err)
|
|
tempDir := t.TempDir()
|
|
require.NoError(t, os.Chdir(tempDir))
|
|
defer func() { require.NoError(t, os.Chdir(originalWd)) }()
|
|
|
|
tc := makeE2eTestingContext(t)
|
|
tc.installRcloneGitannexSymlink(t)
|
|
tc.installRcloneConfig(t)
|
|
tc.createGitRepo(t)
|
|
|
|
remoteStorage := filepath.Join(tc.tempDir, "remotePrefix")
|
|
require.NoError(t, os.Mkdir(remoteStorage, 0777))
|
|
|
|
tc.runInRepo(t,
|
|
"git", "annex", "initremote", "Control",
|
|
"type=external", "externaltype=rclone", "encryption=none",
|
|
"target=MyRcloneRemote",
|
|
"rclone_layout="+string(mode),
|
|
"prefix="+remoteStorage)
|
|
|
|
tc.runInRepo(t,
|
|
"git", "annex", "initremote", "Experiment",
|
|
"type=external", "externaltype=rclone-builtin", "encryption=none",
|
|
"rcloneremotename=MyRcloneRemote",
|
|
"rclonelayout="+string(mode),
|
|
"rcloneprefix="+remoteStorage)
|
|
|
|
fooFileContents := []byte{1, 2, 3, 4}
|
|
fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo")
|
|
require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700))
|
|
tc.runInRepo(t, "git", "annex", "add", "foo")
|
|
tc.runInRepo(t, "git", "commit", "-m", "Add foo file")
|
|
// Git-annex objects are not writable, which prevents `testing` from
|
|
// cleaning up the temp directory. We can work around this by
|
|
// explicitly dropping any files we add to the annex.
|
|
t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") })
|
|
|
|
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
|
|
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
|
|
|
// Copy the file to Control and verify it's present on Experiment.
|
|
|
|
tc.runInRepo(t, "git", "annex", "copy", "--to=Control", "foo")
|
|
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
|
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
|
|
|
tc.runInRepo(t, "git", "annex", "fsck", "--from=Experiment", "foo")
|
|
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
|
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
|
|
|
// Drop the file locally and verify we can copy it back from Experiment.
|
|
|
|
tc.runInRepo(t, "git", "annex", "drop", "--force", "foo")
|
|
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
|
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
|
|
|
tc.runInRepo(t, "git", "annex", "copy", "--from=Experiment", "foo")
|
|
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
|
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
|
|
|
// Drop the file from Experiment, copy it back to Experiment, and
|
|
// verify it's still present on Control.
|
|
|
|
tc.runInRepo(t, "git", "annex", "drop", "--from=Experiment", "--force", "foo")
|
|
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
|
|
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
|
|
|
tc.runInRepo(t, "git", "annex", "copy", "--to=Experiment", "foo")
|
|
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
|
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
|
|
|
tc.runInRepo(t, "git", "annex", "fsck", "--from=Control", "foo")
|
|
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
|
|
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
|
|
|
// Drop the file from Control.
|
|
|
|
tc.runInRepo(t, "git", "annex", "drop", "--from=Control", "--force", "foo")
|
|
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
|
|
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
|
|
})
|
|
}
|
|
}
|