mirror of
https://github.com/rclone/rclone.git
synced 2024-11-22 16:34:30 +01:00
519 lines
15 KiB
Go
519 lines
15 KiB
Go
// Package info provides the info test command.
|
||
package info
|
||
|
||
// FIXME once translations are implemented will need a no-escape
|
||
// option for Put so we can make these tests work again
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"os"
|
||
"path"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/rclone/rclone/cmd"
|
||
"github.com/rclone/rclone/cmd/test"
|
||
"github.com/rclone/rclone/cmd/test/info/internal"
|
||
"github.com/rclone/rclone/fs"
|
||
"github.com/rclone/rclone/fs/config/flags"
|
||
"github.com/rclone/rclone/fs/hash"
|
||
"github.com/rclone/rclone/fs/object"
|
||
"github.com/rclone/rclone/fs/operations"
|
||
"github.com/rclone/rclone/lib/random"
|
||
"github.com/spf13/cobra"
|
||
)
|
||
|
||
var (
|
||
writeJSON string
|
||
keepTestFiles bool
|
||
checkNormalization bool
|
||
checkControl bool
|
||
checkLength bool
|
||
checkStreaming bool
|
||
checkBase32768 bool
|
||
all bool
|
||
uploadWait time.Duration
|
||
positionLeftRe = regexp.MustCompile(`(?s)^(.*)-position-left-([[:xdigit:]]+)$`)
|
||
positionMiddleRe = regexp.MustCompile(`(?s)^position-middle-([[:xdigit:]]+)-(.*)-$`)
|
||
positionRightRe = regexp.MustCompile(`(?s)^position-right-([[:xdigit:]]+)-(.*)$`)
|
||
)
|
||
|
||
func init() {
|
||
test.Command.AddCommand(commandDefinition)
|
||
cmdFlags := commandDefinition.Flags()
|
||
flags.StringVarP(cmdFlags, &writeJSON, "write-json", "", "", "Write results to file", "")
|
||
flags.BoolVarP(cmdFlags, &checkNormalization, "check-normalization", "", false, "Check UTF-8 Normalization", "")
|
||
flags.BoolVarP(cmdFlags, &checkControl, "check-control", "", false, "Check control characters", "")
|
||
flags.DurationVarP(cmdFlags, &uploadWait, "upload-wait", "", 0, "Wait after writing a file", "")
|
||
flags.BoolVarP(cmdFlags, &checkLength, "check-length", "", false, "Check max filename length", "")
|
||
flags.BoolVarP(cmdFlags, &checkStreaming, "check-streaming", "", false, "Check uploads with indeterminate file size", "")
|
||
flags.BoolVarP(cmdFlags, &checkBase32768, "check-base32768", "", false, "Check can store all possible base32768 characters", "")
|
||
flags.BoolVarP(cmdFlags, &all, "all", "", false, "Run all tests", "")
|
||
flags.BoolVarP(cmdFlags, &keepTestFiles, "keep-test-files", "", false, "Keep test files after execution", "")
|
||
}
|
||
|
||
var commandDefinition = &cobra.Command{
|
||
Use: "info [remote:path]+",
|
||
Short: `Discovers file name or other limitations for paths.`,
|
||
Long: `rclone info discovers what filenames and upload methods are possible
|
||
to write to the paths passed in and how long they can be. It can take some
|
||
time. It will write test files into the remote:path passed in. It outputs
|
||
a bit of go code for each one.
|
||
|
||
**NB** this can create undeletable files and other hazards - use with care
|
||
`,
|
||
Annotations: map[string]string{
|
||
"versionIntroduced": "v1.55",
|
||
},
|
||
Run: func(command *cobra.Command, args []string) {
|
||
cmd.CheckArgs(1, 1e6, command, args)
|
||
if !checkNormalization && !checkControl && !checkLength && !checkStreaming && !checkBase32768 && !all {
|
||
log.Fatalf("no tests selected - select a test or use --all")
|
||
}
|
||
if all {
|
||
checkNormalization = true
|
||
checkControl = true
|
||
checkLength = true
|
||
checkStreaming = true
|
||
checkBase32768 = true
|
||
}
|
||
for i := range args {
|
||
tempDirName := "rclone-test-info-" + random.String(8)
|
||
tempDirPath := path.Join(args[i], tempDirName)
|
||
f := cmd.NewFsDir([]string{tempDirPath})
|
||
fs.Infof(f, "Created temporary directory for test files: %s", tempDirPath)
|
||
err := f.Mkdir(context.Background(), "")
|
||
if err != nil {
|
||
log.Fatalf("couldn't create temporary directory: %v", err)
|
||
}
|
||
|
||
cmd.Run(false, false, command, func() error {
|
||
return readInfo(context.Background(), f)
|
||
})
|
||
}
|
||
},
|
||
}
|
||
|
||
type results struct {
|
||
ctx context.Context
|
||
f fs.Fs
|
||
mu sync.Mutex
|
||
stringNeedsEscaping map[string]internal.Position
|
||
controlResults map[string]internal.ControlResult
|
||
maxFileLength [4]int
|
||
canWriteUnnormalized bool
|
||
canReadUnnormalized bool
|
||
canReadRenormalized bool
|
||
canStream bool
|
||
canBase32768 bool
|
||
}
|
||
|
||
func newResults(ctx context.Context, f fs.Fs) *results {
|
||
return &results{
|
||
ctx: ctx,
|
||
f: f,
|
||
stringNeedsEscaping: make(map[string]internal.Position),
|
||
controlResults: make(map[string]internal.ControlResult),
|
||
}
|
||
}
|
||
|
||
// Print the results to stdout
|
||
func (r *results) Print() {
|
||
fmt.Printf("// %s\n", r.f.Name())
|
||
if checkControl {
|
||
escape := []string{}
|
||
for c, needsEscape := range r.stringNeedsEscaping {
|
||
if needsEscape != internal.PositionNone {
|
||
k := strconv.Quote(c)
|
||
k = k[1 : len(k)-1]
|
||
escape = append(escape, fmt.Sprintf("'%s'", k))
|
||
}
|
||
}
|
||
sort.Strings(escape)
|
||
fmt.Printf("stringNeedsEscaping = []rune{\n")
|
||
fmt.Printf("\t%s\n", strings.Join(escape, ", "))
|
||
fmt.Printf("}\n")
|
||
}
|
||
if checkLength {
|
||
for i := range r.maxFileLength {
|
||
fmt.Printf("maxFileLength = %d // for %d byte unicode characters\n", r.maxFileLength[i], i+1)
|
||
}
|
||
}
|
||
if checkNormalization {
|
||
fmt.Printf("canWriteUnnormalized = %v\n", r.canWriteUnnormalized)
|
||
fmt.Printf("canReadUnnormalized = %v\n", r.canReadUnnormalized)
|
||
fmt.Printf("canReadRenormalized = %v\n", r.canReadRenormalized)
|
||
}
|
||
if checkStreaming {
|
||
fmt.Printf("canStream = %v\n", r.canStream)
|
||
}
|
||
if checkBase32768 {
|
||
fmt.Printf("base32768isOK = %v // make sure maxFileLength for 2 byte unicode chars is the same as for 1 byte characters\n", r.canBase32768)
|
||
}
|
||
}
|
||
|
||
// WriteJSON writes the results to a JSON file when requested
|
||
func (r *results) WriteJSON() {
|
||
if writeJSON == "" {
|
||
return
|
||
}
|
||
|
||
report := internal.InfoReport{
|
||
Remote: r.f.Name(),
|
||
}
|
||
if checkControl {
|
||
report.ControlCharacters = &r.controlResults
|
||
}
|
||
if checkLength {
|
||
report.MaxFileLength = &r.maxFileLength[0]
|
||
}
|
||
if checkNormalization {
|
||
report.CanWriteUnnormalized = &r.canWriteUnnormalized
|
||
report.CanReadUnnormalized = &r.canReadUnnormalized
|
||
report.CanReadRenormalized = &r.canReadRenormalized
|
||
}
|
||
if checkStreaming {
|
||
report.CanStream = &r.canStream
|
||
}
|
||
|
||
if f, err := os.Create(writeJSON); err != nil {
|
||
fs.Errorf(r.f, "Creating JSON file failed: %s", err)
|
||
} else {
|
||
defer fs.CheckClose(f, &err)
|
||
enc := json.NewEncoder(f)
|
||
enc.SetIndent("", " ")
|
||
err := enc.Encode(report)
|
||
if err != nil {
|
||
fs.Errorf(r.f, "Writing JSON file failed: %s", err)
|
||
}
|
||
}
|
||
fs.Infof(r.f, "Wrote JSON file: %s", writeJSON)
|
||
}
|
||
|
||
// writeFile writes a file with some random contents
|
||
func (r *results) writeFile(path string) (fs.Object, error) {
|
||
contents := random.String(50)
|
||
src := object.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f)
|
||
obj, err := r.f.Put(r.ctx, bytes.NewBufferString(contents), src)
|
||
if uploadWait > 0 {
|
||
time.Sleep(uploadWait)
|
||
}
|
||
return obj, err
|
||
}
|
||
|
||
// check whether normalization is enforced and check whether it is
|
||
// done on the files anyway
|
||
func (r *results) checkUTF8Normalization() {
|
||
unnormalized := "Héroique"
|
||
normalized := "Héroique"
|
||
_, err := r.writeFile(unnormalized)
|
||
if err != nil {
|
||
r.canWriteUnnormalized = false
|
||
return
|
||
}
|
||
r.canWriteUnnormalized = true
|
||
_, err = r.f.NewObject(r.ctx, unnormalized)
|
||
if err == nil {
|
||
r.canReadUnnormalized = true
|
||
}
|
||
_, err = r.f.NewObject(r.ctx, normalized)
|
||
if err == nil {
|
||
r.canReadRenormalized = true
|
||
}
|
||
}
|
||
|
||
func (r *results) checkStringPositions(k, s string) {
|
||
fs.Infof(r.f, "Writing position file 0x%0X", s)
|
||
positionError := internal.PositionNone
|
||
res := internal.ControlResult{
|
||
Text: s,
|
||
WriteError: make(map[internal.Position]string, 3),
|
||
GetError: make(map[internal.Position]string, 3),
|
||
InList: make(map[internal.Position]internal.Presence, 3),
|
||
}
|
||
|
||
for _, pos := range internal.PositionList {
|
||
path := ""
|
||
switch pos {
|
||
case internal.PositionMiddle:
|
||
path = fmt.Sprintf("position-middle-%0X-%s-", s, s)
|
||
case internal.PositionLeft:
|
||
path = fmt.Sprintf("%s-position-left-%0X", s, s)
|
||
case internal.PositionRight:
|
||
path = fmt.Sprintf("position-right-%0X-%s", s, s)
|
||
default:
|
||
panic("invalid position: " + pos.String())
|
||
}
|
||
_, writeError := r.writeFile(path)
|
||
if writeError != nil {
|
||
res.WriteError[pos] = writeError.Error()
|
||
fs.Infof(r.f, "Writing %s position file 0x%0X Error: %s", pos.String(), s, writeError)
|
||
} else {
|
||
fs.Infof(r.f, "Writing %s position file 0x%0X OK", pos.String(), s)
|
||
}
|
||
obj, getErr := r.f.NewObject(r.ctx, path)
|
||
if getErr != nil {
|
||
res.GetError[pos] = getErr.Error()
|
||
fs.Infof(r.f, "Getting %s position file 0x%0X Error: %s", pos.String(), s, getErr)
|
||
} else {
|
||
if obj.Size() != 50 {
|
||
res.GetError[pos] = fmt.Sprintf("invalid size %d", obj.Size())
|
||
fs.Infof(r.f, "Getting %s position file 0x%0X Invalid Size: %d", pos.String(), s, obj.Size())
|
||
} else {
|
||
fs.Infof(r.f, "Getting %s position file 0x%0X OK", pos.String(), s)
|
||
}
|
||
}
|
||
if writeError != nil || getErr != nil {
|
||
positionError += pos
|
||
}
|
||
}
|
||
|
||
r.mu.Lock()
|
||
r.stringNeedsEscaping[k] = positionError
|
||
r.controlResults[k] = res
|
||
r.mu.Unlock()
|
||
}
|
||
|
||
// check we can write a file with the control chars
|
||
func (r *results) checkControls() {
|
||
fs.Infof(r.f, "Trying to create control character file names")
|
||
ci := fs.GetConfig(context.Background())
|
||
|
||
// Concurrency control
|
||
tokens := make(chan struct{}, ci.Checkers)
|
||
for i := 0; i < ci.Checkers; i++ {
|
||
tokens <- struct{}{}
|
||
}
|
||
var wg sync.WaitGroup
|
||
for i := rune(0); i < 128; i++ {
|
||
s := string(i)
|
||
if i == 0 || i == '/' {
|
||
// We're not even going to check NULL or /
|
||
r.stringNeedsEscaping[s] = internal.PositionAll
|
||
continue
|
||
}
|
||
wg.Add(1)
|
||
go func(s string) {
|
||
defer wg.Done()
|
||
token := <-tokens
|
||
k := s
|
||
r.checkStringPositions(k, s)
|
||
tokens <- token
|
||
}(s)
|
||
}
|
||
for _, s := range []string{"＼", "\u00A0", "\xBF", "\xFE"} {
|
||
wg.Add(1)
|
||
go func(s string) {
|
||
defer wg.Done()
|
||
token := <-tokens
|
||
k := s
|
||
r.checkStringPositions(k, s)
|
||
tokens <- token
|
||
}(s)
|
||
}
|
||
wg.Wait()
|
||
r.checkControlsList()
|
||
fs.Infof(r.f, "Done trying to create control character file names")
|
||
}
|
||
|
||
func (r *results) checkControlsList() {
|
||
l, err := r.f.List(context.TODO(), "")
|
||
if err != nil {
|
||
fs.Errorf(r.f, "Listing control character file names failed: %s", err)
|
||
return
|
||
}
|
||
|
||
namesMap := make(map[string]struct{}, len(l))
|
||
for _, s := range l {
|
||
namesMap[path.Base(s.Remote())] = struct{}{}
|
||
}
|
||
|
||
for path := range namesMap {
|
||
var pos internal.Position
|
||
var hex, value string
|
||
if g := positionLeftRe.FindStringSubmatch(path); g != nil {
|
||
pos, hex, value = internal.PositionLeft, g[2], g[1]
|
||
} else if g := positionMiddleRe.FindStringSubmatch(path); g != nil {
|
||
pos, hex, value = internal.PositionMiddle, g[1], g[2]
|
||
} else if g := positionRightRe.FindStringSubmatch(path); g != nil {
|
||
pos, hex, value = internal.PositionRight, g[1], g[2]
|
||
} else {
|
||
fs.Infof(r.f, "Unknown path %q", path)
|
||
continue
|
||
}
|
||
var hexValue []byte
|
||
for ; len(hex) >= 2; hex = hex[2:] {
|
||
if b, err := strconv.ParseUint(hex[:2], 16, 8); err != nil {
|
||
fs.Infof(r.f, "Invalid path %q: %s", path, err)
|
||
continue
|
||
} else {
|
||
hexValue = append(hexValue, byte(b))
|
||
}
|
||
}
|
||
if hex != "" {
|
||
fs.Infof(r.f, "Invalid path %q", path)
|
||
continue
|
||
}
|
||
|
||
hexStr := string(hexValue)
|
||
k := hexStr
|
||
switch r.controlResults[k].InList[pos] {
|
||
case internal.Absent:
|
||
if hexStr == value {
|
||
r.controlResults[k].InList[pos] = internal.Present
|
||
} else {
|
||
r.controlResults[k].InList[pos] = internal.Renamed
|
||
}
|
||
case internal.Present:
|
||
r.controlResults[k].InList[pos] = internal.Multiple
|
||
case internal.Renamed:
|
||
r.controlResults[k].InList[pos] = internal.Multiple
|
||
}
|
||
delete(namesMap, path)
|
||
}
|
||
|
||
if len(namesMap) > 0 {
|
||
fs.Infof(r.f, "Found additional control character file names:")
|
||
for name := range namesMap {
|
||
fs.Infof(r.f, "%q", name)
|
||
}
|
||
}
|
||
}
|
||
|
||
// find the max file name size we can use
|
||
func (r *results) findMaxLength(characterLength int) {
|
||
var character rune
|
||
switch characterLength {
|
||
case 1:
|
||
character = 'a'
|
||
case 2:
|
||
character = 'á'
|
||
case 3:
|
||
character = '世'
|
||
case 4:
|
||
character = '🙂'
|
||
default:
|
||
panic("Bad characterLength")
|
||
}
|
||
if characterLength != len(string(character)) {
|
||
panic(fmt.Sprintf("Chose the wrong character length %q is %d not %d", character, len(string(character)), characterLength))
|
||
}
|
||
const maxLen = 16 * 1024
|
||
name := make([]rune, maxLen)
|
||
for i := range name {
|
||
name[i] = character
|
||
}
|
||
// Find the first size of filename we can't write
|
||
i := sort.Search(len(name), func(i int) (fail bool) {
|
||
defer func() {
|
||
if err := recover(); err != nil {
|
||
fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
|
||
fail = true
|
||
}
|
||
}()
|
||
|
||
path := string(name[:i])
|
||
o, err := r.writeFile(path)
|
||
if err != nil {
|
||
fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
|
||
return true
|
||
}
|
||
fs.Infof(r.f, "Wrote file with name length %d", i)
|
||
err = o.Remove(context.Background())
|
||
if err != nil {
|
||
fs.Errorf(o, "Failed to remove test file")
|
||
}
|
||
return false
|
||
})
|
||
r.maxFileLength[characterLength-1] = i - 1
|
||
fs.Infof(r.f, "Max file length is %d when writing %d byte characters %q", r.maxFileLength[characterLength-1], characterLength, character)
|
||
}
|
||
|
||
func (r *results) checkStreaming() {
|
||
putter := r.f.Put
|
||
if r.f.Features().PutStream != nil {
|
||
fs.Infof(r.f, "Given remote has specialized streaming function. Using that to test streaming.")
|
||
putter = r.f.Features().PutStream
|
||
}
|
||
|
||
contents := "thinking of test strings is hard"
|
||
buf := bytes.NewBufferString(contents)
|
||
hashIn := hash.NewMultiHasher()
|
||
in := io.TeeReader(buf, hashIn)
|
||
|
||
objIn := object.NewStaticObjectInfo("checkStreamingTest", time.Now(), -1, true, nil, r.f)
|
||
objR, err := putter(r.ctx, in, objIn)
|
||
if err != nil {
|
||
fs.Infof(r.f, "Streamed file failed to upload (%v)", err)
|
||
r.canStream = false
|
||
return
|
||
}
|
||
|
||
hashes := hashIn.Sums()
|
||
types := objR.Fs().Hashes().Array()
|
||
for _, Hash := range types {
|
||
sum, err := objR.Hash(r.ctx, Hash)
|
||
if err != nil {
|
||
fs.Infof(r.f, "Streamed file failed when getting hash %v (%v)", Hash, err)
|
||
r.canStream = false
|
||
return
|
||
}
|
||
if !hash.Equals(hashes[Hash], sum) {
|
||
fs.Infof(r.f, "Streamed file has incorrect hash %v: expecting %q got %q", Hash, hashes[Hash], sum)
|
||
r.canStream = false
|
||
return
|
||
}
|
||
}
|
||
if int64(len(contents)) != objR.Size() {
|
||
fs.Infof(r.f, "Streamed file has incorrect file size: expecting %d got %d", len(contents), objR.Size())
|
||
r.canStream = false
|
||
return
|
||
}
|
||
r.canStream = true
|
||
}
|
||
|
||
func readInfo(ctx context.Context, f fs.Fs) error {
|
||
// Ensure cleanup unless --keep-test-files is specified
|
||
if !keepTestFiles {
|
||
defer func() {
|
||
err := operations.Purge(ctx, f, "")
|
||
if err != nil {
|
||
fs.Errorf(f, "Failed to purge temporary directory: %v", err)
|
||
} else {
|
||
fs.Infof(f, "Removed temporary directory for test files: %s", f.Root())
|
||
}
|
||
}()
|
||
}
|
||
|
||
r := newResults(ctx, f)
|
||
if checkControl {
|
||
r.checkControls()
|
||
}
|
||
if checkLength {
|
||
for i := range r.maxFileLength {
|
||
r.findMaxLength(i + 1)
|
||
}
|
||
}
|
||
if checkNormalization {
|
||
r.checkUTF8Normalization()
|
||
}
|
||
if checkStreaming {
|
||
r.checkStreaming()
|
||
}
|
||
if checkBase32768 {
|
||
r.checkBase32768()
|
||
}
|
||
r.Print()
|
||
r.WriteJSON()
|
||
return nil
|
||
}
|