mirror of
https://github.com/rclone/rclone.git
synced 2024-11-30 04:15:26 +01:00
57d5de6fba
git grep -l github.com/ncw/rclone | xargs -d'\n' perl -i~ -lpe 's|github.com/ncw/rclone|github.com/rclone/rclone|g' goimports -w `find . -name \*.go`
351 lines
9.0 KiB
Go
351 lines
9.0 KiB
Go
package info
|
||
|
||
// FIXME once translations are implemented will need a no-escape
|
||
// option for Put so we can make these tests work agaig
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"fmt"
|
||
"io"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/pkg/errors"
|
||
"github.com/rclone/rclone/cmd"
|
||
"github.com/rclone/rclone/fs"
|
||
"github.com/rclone/rclone/fs/hash"
|
||
"github.com/rclone/rclone/fs/object"
|
||
"github.com/rclone/rclone/fstest"
|
||
"github.com/spf13/cobra"
|
||
)
|
||
|
||
type position int
|
||
|
||
const (
|
||
positionMiddle position = 1 << iota
|
||
positionLeft
|
||
positionRight
|
||
positionNone position = 0
|
||
positionAll position = positionRight<<1 - 1
|
||
)
|
||
|
||
var (
|
||
checkNormalization bool
|
||
checkControl bool
|
||
checkLength bool
|
||
checkStreaming bool
|
||
positionList = []position{positionMiddle, positionLeft, positionRight}
|
||
)
|
||
|
||
func init() {
|
||
cmd.Root.AddCommand(commandDefintion)
|
||
commandDefintion.Flags().BoolVarP(&checkNormalization, "check-normalization", "", true, "Check UTF-8 Normalization.")
|
||
commandDefintion.Flags().BoolVarP(&checkControl, "check-control", "", true, "Check control characters.")
|
||
commandDefintion.Flags().BoolVarP(&checkLength, "check-length", "", true, "Check max filename length.")
|
||
commandDefintion.Flags().BoolVarP(&checkStreaming, "check-streaming", "", true, "Check uploads with indeterminate file size.")
|
||
}
|
||
|
||
var commandDefintion = &cobra.Command{
|
||
Use: "info [remote:path]+",
|
||
Short: `Discovers file name or other limitations for paths.`,
|
||
Long: `rclone info discovers what filenames and upload methods are possible
|
||
to write to the paths passed in and how long they can be. It can take some
|
||
time. It will write test files into the remote:path passed in. It outputs
|
||
a bit of go code for each one.
|
||
`,
|
||
Hidden: true,
|
||
Run: func(command *cobra.Command, args []string) {
|
||
cmd.CheckArgs(1, 1E6, command, args)
|
||
for i := range args {
|
||
f := cmd.NewFsDir(args[i : i+1])
|
||
cmd.Run(false, false, command, func() error {
|
||
return readInfo(context.Background(), f)
|
||
})
|
||
}
|
||
},
|
||
}
|
||
|
||
type results struct {
|
||
ctx context.Context
|
||
f fs.Fs
|
||
mu sync.Mutex
|
||
stringNeedsEscaping map[string]position
|
||
maxFileLength int
|
||
canWriteUnnormalized bool
|
||
canReadUnnormalized bool
|
||
canReadRenormalized bool
|
||
canStream bool
|
||
}
|
||
|
||
func newResults(ctx context.Context, f fs.Fs) *results {
|
||
return &results{
|
||
ctx: ctx,
|
||
f: f,
|
||
stringNeedsEscaping: make(map[string]position),
|
||
}
|
||
}
|
||
|
||
// Print the results to stdout
|
||
func (r *results) Print() {
|
||
fmt.Printf("// %s\n", r.f.Name())
|
||
if checkControl {
|
||
escape := []string{}
|
||
for c, needsEscape := range r.stringNeedsEscaping {
|
||
if needsEscape != positionNone {
|
||
escape = append(escape, fmt.Sprintf("0x%02X", c))
|
||
}
|
||
}
|
||
sort.Strings(escape)
|
||
fmt.Printf("stringNeedsEscaping = []byte{\n")
|
||
fmt.Printf("\t%s\n", strings.Join(escape, ", "))
|
||
fmt.Printf("}\n")
|
||
}
|
||
if checkLength {
|
||
fmt.Printf("maxFileLength = %d\n", r.maxFileLength)
|
||
}
|
||
if checkNormalization {
|
||
fmt.Printf("canWriteUnnormalized = %v\n", r.canWriteUnnormalized)
|
||
fmt.Printf("canReadUnnormalized = %v\n", r.canReadUnnormalized)
|
||
fmt.Printf("canReadRenormalized = %v\n", r.canReadRenormalized)
|
||
}
|
||
if checkStreaming {
|
||
fmt.Printf("canStream = %v\n", r.canStream)
|
||
}
|
||
}
|
||
|
||
// writeFile writes a file with some random contents
|
||
func (r *results) writeFile(path string) (fs.Object, error) {
|
||
contents := fstest.RandomString(50)
|
||
src := object.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f)
|
||
return r.f.Put(r.ctx, bytes.NewBufferString(contents), src)
|
||
}
|
||
|
||
// check whether normalization is enforced and check whether it is
|
||
// done on the files anyway
|
||
func (r *results) checkUTF8Normalization() {
|
||
unnormalized := "Héroique"
|
||
normalized := "Héroique"
|
||
_, err := r.writeFile(unnormalized)
|
||
if err != nil {
|
||
r.canWriteUnnormalized = false
|
||
return
|
||
}
|
||
r.canWriteUnnormalized = true
|
||
_, err = r.f.NewObject(r.ctx, unnormalized)
|
||
if err == nil {
|
||
r.canReadUnnormalized = true
|
||
}
|
||
_, err = r.f.NewObject(r.ctx, normalized)
|
||
if err == nil {
|
||
r.canReadRenormalized = true
|
||
}
|
||
}
|
||
|
||
func (r *results) checkStringPositions(s string) {
|
||
fs.Infof(r.f, "Writing position file 0x%0X", s)
|
||
positionError := positionNone
|
||
|
||
for _, pos := range positionList {
|
||
path := ""
|
||
switch pos {
|
||
case positionMiddle:
|
||
path = fmt.Sprintf("position-middle-%0X-%s-", s, s)
|
||
case positionLeft:
|
||
path = fmt.Sprintf("%s-position-left-%0X", s, s)
|
||
case positionRight:
|
||
path = fmt.Sprintf("position-right-%0X-%s", s, s)
|
||
default:
|
||
panic("invalid position: " + pos.String())
|
||
}
|
||
_, writeErr := r.writeFile(path)
|
||
if writeErr != nil {
|
||
fs.Infof(r.f, "Writing %s position file 0x%0X Error: %s", pos.String(), s, writeErr)
|
||
} else {
|
||
fs.Infof(r.f, "Writing %s position file 0x%0X OK", pos.String(), s)
|
||
}
|
||
obj, getErr := r.f.NewObject(r.ctx, path)
|
||
if getErr != nil {
|
||
fs.Infof(r.f, "Getting %s position file 0x%0X Error: %s", pos.String(), s, getErr)
|
||
} else {
|
||
if obj.Size() != 50 {
|
||
fs.Infof(r.f, "Getting %s position file 0x%0X Invalid Size: %d", pos.String(), s, obj.Size())
|
||
} else {
|
||
fs.Infof(r.f, "Getting %s position file 0x%0X OK", pos.String(), s)
|
||
}
|
||
}
|
||
if writeErr != nil || getErr != nil {
|
||
positionError += pos
|
||
}
|
||
}
|
||
|
||
r.mu.Lock()
|
||
r.stringNeedsEscaping[s] = positionError
|
||
r.mu.Unlock()
|
||
}
|
||
|
||
// check we can write a file with the control chars
|
||
func (r *results) checkControls() {
|
||
fs.Infof(r.f, "Trying to create control character file names")
|
||
// Concurrency control
|
||
tokens := make(chan struct{}, fs.Config.Checkers)
|
||
for i := 0; i < fs.Config.Checkers; i++ {
|
||
tokens <- struct{}{}
|
||
}
|
||
var wg sync.WaitGroup
|
||
for i := rune(0); i < 128; i++ {
|
||
s := string(i)
|
||
if i == 0 || i == '/' {
|
||
// We're not even going to check NULL or /
|
||
r.stringNeedsEscaping[s] = positionAll
|
||
continue
|
||
}
|
||
wg.Add(1)
|
||
go func(s string) {
|
||
defer wg.Done()
|
||
token := <-tokens
|
||
r.checkStringPositions(s)
|
||
tokens <- token
|
||
}(s)
|
||
}
|
||
for _, s := range []string{"＼", "\xBF", "\xFE"} {
|
||
wg.Add(1)
|
||
go func(s string) {
|
||
defer wg.Done()
|
||
token := <-tokens
|
||
r.checkStringPositions(s)
|
||
tokens <- token
|
||
}(s)
|
||
}
|
||
wg.Wait()
|
||
fs.Infof(r.f, "Done trying to create control character file names")
|
||
}
|
||
|
||
// find the max file name size we can use
|
||
func (r *results) findMaxLength() {
|
||
const maxLen = 16 * 1024
|
||
name := make([]byte, maxLen)
|
||
for i := range name {
|
||
name[i] = 'a'
|
||
}
|
||
// Find the first size of filename we can't write
|
||
i := sort.Search(len(name), func(i int) (fail bool) {
|
||
defer func() {
|
||
if err := recover(); err != nil {
|
||
fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
|
||
fail = true
|
||
}
|
||
}()
|
||
|
||
path := string(name[:i])
|
||
_, err := r.writeFile(path)
|
||
if err != nil {
|
||
fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
|
||
return true
|
||
}
|
||
fs.Infof(r.f, "Wrote file with name length %d", i)
|
||
return false
|
||
})
|
||
r.maxFileLength = i - 1
|
||
fs.Infof(r.f, "Max file length is %d", r.maxFileLength)
|
||
}
|
||
|
||
func (r *results) checkStreaming() {
|
||
putter := r.f.Put
|
||
if r.f.Features().PutStream != nil {
|
||
fs.Infof(r.f, "Given remote has specialized streaming function. Using that to test streaming.")
|
||
putter = r.f.Features().PutStream
|
||
}
|
||
|
||
contents := "thinking of test strings is hard"
|
||
buf := bytes.NewBufferString(contents)
|
||
hashIn := hash.NewMultiHasher()
|
||
in := io.TeeReader(buf, hashIn)
|
||
|
||
objIn := object.NewStaticObjectInfo("checkStreamingTest", time.Now(), -1, true, nil, r.f)
|
||
objR, err := putter(r.ctx, in, objIn)
|
||
if err != nil {
|
||
fs.Infof(r.f, "Streamed file failed to upload (%v)", err)
|
||
r.canStream = false
|
||
return
|
||
}
|
||
|
||
hashes := hashIn.Sums()
|
||
types := objR.Fs().Hashes().Array()
|
||
for _, Hash := range types {
|
||
sum, err := objR.Hash(r.ctx, Hash)
|
||
if err != nil {
|
||
fs.Infof(r.f, "Streamed file failed when getting hash %v (%v)", Hash, err)
|
||
r.canStream = false
|
||
return
|
||
}
|
||
if !hash.Equals(hashes[Hash], sum) {
|
||
fs.Infof(r.f, "Streamed file has incorrect hash %v: expecting %q got %q", Hash, hashes[Hash], sum)
|
||
r.canStream = false
|
||
return
|
||
}
|
||
}
|
||
if int64(len(contents)) != objR.Size() {
|
||
fs.Infof(r.f, "Streamed file has incorrect file size: expecting %d got %d", len(contents), objR.Size())
|
||
r.canStream = false
|
||
return
|
||
}
|
||
r.canStream = true
|
||
}
|
||
|
||
func readInfo(ctx context.Context, f fs.Fs) error {
|
||
err := f.Mkdir(ctx, "")
|
||
if err != nil {
|
||
return errors.Wrap(err, "couldn't mkdir")
|
||
}
|
||
r := newResults(ctx, f)
|
||
if checkControl {
|
||
r.checkControls()
|
||
}
|
||
if checkLength {
|
||
r.findMaxLength()
|
||
}
|
||
if checkNormalization {
|
||
r.checkUTF8Normalization()
|
||
}
|
||
if checkStreaming {
|
||
r.checkStreaming()
|
||
}
|
||
r.Print()
|
||
return nil
|
||
}
|
||
|
||
func (e position) String() string {
|
||
switch e {
|
||
case positionNone:
|
||
return "none"
|
||
case positionAll:
|
||
return "all"
|
||
}
|
||
var buf bytes.Buffer
|
||
if e&positionMiddle != 0 {
|
||
buf.WriteString("middle")
|
||
e &= ^positionMiddle
|
||
}
|
||
if e&positionLeft != 0 {
|
||
if buf.Len() != 0 {
|
||
buf.WriteRune(',')
|
||
}
|
||
buf.WriteString("left")
|
||
e &= ^positionLeft
|
||
}
|
||
if e&positionRight != 0 {
|
||
if buf.Len() != 0 {
|
||
buf.WriteRune(',')
|
||
}
|
||
buf.WriteString("right")
|
||
e &= ^positionRight
|
||
}
|
||
if e != positionNone {
|
||
panic("invalid position")
|
||
}
|
||
return buf.String()
|
||
}
|