mirror of
https://github.com/rclone/rclone.git
synced 2024-11-30 12:25:29 +01:00
268 lines
6.9 KiB
Go
268 lines
6.9 KiB
Go
package info
|
|
|
|
// FIXME once translations are implemented will need a no-escape
|
|
// option for Put so we can make these tests work agaig
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/ncw/rclone/cmd"
|
|
"github.com/ncw/rclone/fs"
|
|
"github.com/ncw/rclone/fstest"
|
|
"github.com/pkg/errors"
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
var (
|
|
checkNormalization bool
|
|
checkControl bool
|
|
checkLength bool
|
|
checkStreaming bool
|
|
)
|
|
|
|
func init() {
|
|
cmd.Root.AddCommand(commandDefintion)
|
|
commandDefintion.Flags().BoolVarP(&checkNormalization, "check-normalization", "", true, "Check UTF-8 Normalization.")
|
|
commandDefintion.Flags().BoolVarP(&checkControl, "check-control", "", true, "Check control characters.")
|
|
commandDefintion.Flags().BoolVarP(&checkLength, "check-length", "", true, "Check max filename length.")
|
|
commandDefintion.Flags().BoolVarP(&checkStreaming, "check-streaming", "", true, "Check uploads with indeterminate file size.")
|
|
}
|
|
|
|
var commandDefintion = &cobra.Command{
|
|
Use: "info [remote:path]+",
|
|
Short: `Discovers file name or other limitations for paths.`,
|
|
Long: `rclone info discovers what filenames and upload methods are possible
|
|
to write to the paths passed in and how long they can be. It can take some
|
|
time. It will write test files into the remote:path passed in. It outputs
|
|
a bit of go code for each one.
|
|
`,
|
|
Hidden: true,
|
|
Run: func(command *cobra.Command, args []string) {
|
|
cmd.CheckArgs(1, 1E6, command, args)
|
|
for i := range args {
|
|
f := cmd.NewFsDst(args[i : i+1])
|
|
cmd.Run(false, false, command, func() error {
|
|
return readInfo(f)
|
|
})
|
|
}
|
|
},
|
|
}
|
|
|
|
type results struct {
|
|
f fs.Fs
|
|
mu sync.Mutex
|
|
charNeedsEscaping map[rune]bool
|
|
maxFileLength int
|
|
canWriteUnnormalized bool
|
|
canReadUnnormalized bool
|
|
canReadRenormalized bool
|
|
canStream bool
|
|
}
|
|
|
|
func newResults(f fs.Fs) *results {
|
|
return &results{
|
|
f: f,
|
|
charNeedsEscaping: make(map[rune]bool),
|
|
}
|
|
}
|
|
|
|
// Print the results to stdout
|
|
func (r *results) Print() {
|
|
fmt.Printf("// %s\n", r.f.Name())
|
|
if checkControl {
|
|
escape := []string{}
|
|
for c, needsEscape := range r.charNeedsEscaping {
|
|
if needsEscape {
|
|
escape = append(escape, fmt.Sprintf("0x%02X", c))
|
|
}
|
|
}
|
|
sort.Strings(escape)
|
|
fmt.Printf("charNeedsEscaping = []byte{\n")
|
|
fmt.Printf("\t%s\n", strings.Join(escape, ", "))
|
|
fmt.Printf("}\n")
|
|
}
|
|
if checkLength {
|
|
fmt.Printf("maxFileLength = %d\n", r.maxFileLength)
|
|
}
|
|
if checkNormalization {
|
|
fmt.Printf("canWriteUnnormalized = %v\n", r.canWriteUnnormalized)
|
|
fmt.Printf("canReadUnnormalized = %v\n", r.canReadUnnormalized)
|
|
fmt.Printf("canReadRenormalized = %v\n", r.canReadRenormalized)
|
|
}
|
|
if checkStreaming {
|
|
fmt.Printf("canStream = %v\n", r.canStream)
|
|
}
|
|
}
|
|
|
|
// writeFile writes a file with some random contents
|
|
func (r *results) writeFile(path string) (fs.Object, error) {
|
|
contents := fstest.RandomString(50)
|
|
src := fs.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f)
|
|
return r.f.Put(bytes.NewBufferString(contents), src)
|
|
}
|
|
|
|
// check whether normalization is enforced and check whether it is
|
|
// done on the files anyway
|
|
func (r *results) checkUTF8Normalization() {
|
|
unnormalized := "Héroique"
|
|
normalized := "Héroique"
|
|
_, err := r.writeFile(unnormalized)
|
|
if err != nil {
|
|
r.canWriteUnnormalized = false
|
|
return
|
|
}
|
|
r.canWriteUnnormalized = true
|
|
_, err = r.f.NewObject(unnormalized)
|
|
if err == nil {
|
|
r.canReadUnnormalized = true
|
|
}
|
|
_, err = r.f.NewObject(normalized)
|
|
if err == nil {
|
|
r.canReadRenormalized = true
|
|
}
|
|
}
|
|
|
|
// check we can write file with the rune passed in
|
|
func (r *results) checkChar(c rune) {
|
|
fs.Infof(r.f, "Writing file 0x%02X", c)
|
|
path := fmt.Sprintf("0x%02X-%c-", c, c)
|
|
_, err := r.writeFile(path)
|
|
escape := false
|
|
if err != nil {
|
|
fs.Infof(r.f, "Couldn't write file 0x%02X", c)
|
|
} else {
|
|
fs.Infof(r.f, "OK writing file 0x%02X", c)
|
|
}
|
|
r.mu.Lock()
|
|
r.charNeedsEscaping[c] = escape
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// check we can write a file with the control chars
|
|
func (r *results) checkControls() {
|
|
fs.Infof(r.f, "Trying to create control character file names")
|
|
// Concurrency control
|
|
tokens := make(chan struct{}, fs.Config.Checkers)
|
|
for i := 0; i < fs.Config.Checkers; i++ {
|
|
tokens <- struct{}{}
|
|
}
|
|
var wg sync.WaitGroup
|
|
for i := rune(0); i < 128; i++ {
|
|
if i == 0 || i == '/' {
|
|
// We're not even going to check NULL or /
|
|
r.charNeedsEscaping[i] = true
|
|
continue
|
|
}
|
|
wg.Add(1)
|
|
c := i
|
|
go func() {
|
|
defer wg.Done()
|
|
token := <-tokens
|
|
r.checkChar(c)
|
|
tokens <- token
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
fs.Infof(r.f, "Done trying to create control character file names")
|
|
}
|
|
|
|
// find the max file name size we can use
|
|
func (r *results) findMaxLength() {
|
|
const maxLen = 16 * 1024
|
|
name := make([]byte, maxLen)
|
|
for i := range name {
|
|
name[i] = 'a'
|
|
}
|
|
// Find the first size of filename we can't write
|
|
i := sort.Search(len(name), func(i int) (fail bool) {
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
|
|
fail = true
|
|
}
|
|
}()
|
|
|
|
path := string(name[:i])
|
|
_, err := r.writeFile(path)
|
|
if err != nil {
|
|
fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err)
|
|
return true
|
|
}
|
|
fs.Infof(r.f, "Wrote file with name length %d", i)
|
|
return false
|
|
})
|
|
r.maxFileLength = i - 1
|
|
fs.Infof(r.f, "Max file length is %d", r.maxFileLength)
|
|
}
|
|
|
|
func (r *results) checkStreaming() {
|
|
putter := r.f.Put
|
|
if r.f.Features().PutStream != nil {
|
|
fs.Infof(r.f, "Given remote has specialized streaming function. Using that to test streaming.")
|
|
putter = r.f.Features().PutStream
|
|
}
|
|
|
|
contents := "thinking of test strings is hard"
|
|
buf := bytes.NewBufferString(contents)
|
|
hashIn := fs.NewMultiHasher()
|
|
in := io.TeeReader(buf, hashIn)
|
|
|
|
objIn := fs.NewStaticObjectInfo("checkStreamingTest", time.Now(), -1, true, nil, r.f)
|
|
objR, err := putter(in, objIn)
|
|
if err != nil {
|
|
fs.Infof(r.f, "Streamed file failed to upload (%v)", err)
|
|
r.canStream = false
|
|
return
|
|
}
|
|
|
|
hashes := hashIn.Sums()
|
|
types := objR.Fs().Hashes().Array()
|
|
for _, hash := range types {
|
|
sum, err := objR.Hash(hash)
|
|
if err != nil {
|
|
fs.Infof(r.f, "Streamed file failed when getting hash %v (%v)", hash, err)
|
|
r.canStream = false
|
|
return
|
|
}
|
|
if !fs.HashEquals(hashes[hash], sum) {
|
|
fs.Infof(r.f, "Streamed file has incorrect hash %v: expecting %q got %q", hash, hashes[hash], sum)
|
|
r.canStream = false
|
|
return
|
|
}
|
|
}
|
|
if int64(len(contents)) != objR.Size() {
|
|
fs.Infof(r.f, "Streamed file has incorrect file size: expecting %d got %d", len(contents), objR.Size())
|
|
r.canStream = false
|
|
return
|
|
}
|
|
r.canStream = true
|
|
}
|
|
|
|
func readInfo(f fs.Fs) error {
|
|
err := f.Mkdir("")
|
|
if err != nil {
|
|
return errors.Wrap(err, "couldn't mkdir")
|
|
}
|
|
r := newResults(f)
|
|
if checkControl {
|
|
r.checkControls()
|
|
}
|
|
if checkLength {
|
|
r.findMaxLength()
|
|
}
|
|
if checkNormalization {
|
|
r.checkUTF8Normalization()
|
|
}
|
|
if checkStreaming {
|
|
r.checkStreaming()
|
|
}
|
|
r.Print()
|
|
return nil
|
|
}
|