mirror of
https://github.com/rclone/rclone.git
synced 2024-11-25 09:54:44 +01:00
2bebbfaded
This change officially adds bisync to the nightly integration tests for all backends. This will be part of giving us the confidence to take bisync out of beta. A number of fixes have been added to account for features which can differ on different backends -- for example, hash types / modtime support, empty directories, unicode normalization, and unimportant differences in log output. We will likely find that more of these are needed once we start running these with the full set of remotes. Additionally, bisync's extremely sensitive tests revealed a few bugs in other backends that weren't previously covered by other tests. Fixes for those issues have been submitted on the following separate PRs (and bisync test failures will be expected until they are merged): - #7670 memory: fix deadlock in operations.Purge - #7688 memory: fix incorrect list entries when rooted at subdirectory - #7690 memory: fix dst mutating src after server-side copy - #7692 dropbox: fix chunked uploads when size <= chunkSize Relatedly, workarounds have been put in place for the following backend limitations that are unsolvable for the time being: - #3262 drive is sometimes aware of trashed files/folders when it shouldn't be - #6199 dropbox can't handle emojis and certain other characters - #4590 onedrive API has longstanding bug for conflictBehavior=replace in server-side copy/move
131 lines
3.8 KiB
Go
131 lines
3.8 KiB
Go
package bisync_test
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/rclone/rclone/cmd/bisync/bilib"
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/stretchr/testify/assert"
|
|
"gopkg.in/yaml.v2"
|
|
)
|
|
|
|
const configFile = "../../fstest/test_all/config.yaml"
|
|
|
|
// Config describes the config for this program
|
|
type Config struct {
|
|
Tests []Test
|
|
Backends []Backend
|
|
}
|
|
|
|
// Test describes an integration test to run with `go test`
|
|
type Test struct {
|
|
Path string // path to the source directory
|
|
FastList bool // if it is possible to add -fast-list to tests
|
|
Short bool // if it is possible to run the test with -short
|
|
AddBackend bool // set if Path needs the current backend appending
|
|
NoRetries bool // set if no retries should be performed
|
|
NoBinary bool // set to not build a binary in advance
|
|
LocalOnly bool // if set only run with the local backend
|
|
}
|
|
|
|
// Backend describes a backend test
|
|
//
|
|
// FIXME make bucket-based remotes set sub-dir automatically???
|
|
type Backend struct {
|
|
Backend string // name of the backend directory
|
|
Remote string // name of the test remote
|
|
FastList bool // set to test with -fast-list
|
|
Short bool // set to test with -short
|
|
OneOnly bool // set to run only one backend test at once
|
|
MaxFile string // file size limit
|
|
CleanUp bool // when running clean, run cleanup first
|
|
Ignore []string // test names to ignore the failure of
|
|
Tests []string // paths of tests to run, blank for all
|
|
ListRetries int // -list-retries if > 0
|
|
ExtraTime float64 // factor to multiply the timeout by
|
|
}
|
|
|
|
func parseConfig() (*Config, error) {
|
|
d, err := os.ReadFile(configFile)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read config file: %w", err)
|
|
}
|
|
config := &Config{}
|
|
err = yaml.Unmarshal(d, &config)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse config file: %w", err)
|
|
}
|
|
return config, nil
|
|
}
|
|
|
|
const debugFormat = ` {
|
|
"name": %q,
|
|
"type": "go",
|
|
"request": "launch",
|
|
"mode": "test",
|
|
"program": "./cmd/bisync",
|
|
"args": ["-remote", %q, "-remote2", %q, "-case", %q, "-no-cleanup"]
|
|
},
|
|
`
|
|
|
|
const docFormat = `{
|
|
"version": "0.2.0",
|
|
"configurations": [
|
|
%s
|
|
]
|
|
}`
|
|
|
|
// generates a launch.json file for debugging in VS Code.
|
|
// note: just copy the ones you need into your real launch.json file, as VS Code will crash if there are too many!
|
|
func (b *bisyncTest) generateDebuggers() {
|
|
config, err := parseConfig()
|
|
if err != nil {
|
|
fs.Errorf(config, "failed to parse config: %v", err)
|
|
}
|
|
|
|
testList := []string{}
|
|
for _, testCase := range b.listDir(b.dataRoot) {
|
|
if strings.HasPrefix(testCase, "test_") {
|
|
// if dir is empty, skip it (can happen due to gitignored files/dirs when checking out branch)
|
|
if len(b.listDir(filepath.Join(b.dataRoot, testCase))) == 0 {
|
|
continue
|
|
}
|
|
testList = append(testList, testCase)
|
|
}
|
|
}
|
|
|
|
variations := []string{"LocalRemote", "RemoteLocal", "RemoteRemote"}
|
|
debuggers := ""
|
|
|
|
for _, backend := range config.Backends {
|
|
if backend.Remote == "" {
|
|
backend.Remote = "local"
|
|
}
|
|
for _, testcase := range testList {
|
|
for _, variation := range variations {
|
|
if variation != "RemoteRemote" && backend.Remote == "local" {
|
|
continue
|
|
}
|
|
|
|
name := fmt.Sprintf("Test %s %s %s", backend.Remote, testcase, variation)
|
|
switch variation {
|
|
case "LocalRemote":
|
|
debuggers += fmt.Sprintf(debugFormat, name, "local", backend.Remote, testcase)
|
|
case "RemoteLocal":
|
|
debuggers += fmt.Sprintf(debugFormat, name, backend.Remote, "local", testcase)
|
|
case "RemoteRemote":
|
|
debuggers += fmt.Sprintf(debugFormat, name, backend.Remote, backend.Remote, testcase)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
out := fmt.Sprintf(docFormat, debuggers)
|
|
outpath := "./testdata/bisync_vscode_debuggers_launch.json"
|
|
err = os.WriteFile(outpath, []byte(out), bilib.PermSecure)
|
|
assert.NoError(b.t, err, "writing golden file %s", outpath)
|
|
}
|